agentx-python 0.6.6__tar.gz → 0.6.8__tar.gz

{agentx_python-0.6.6/agentx_python.egg-info → agentx_python-0.6.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.6
+Version: 0.6.8
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team
@@ -85,7 +85,7 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
 - **MCP support** — connect any Model Context Protocol server.
 - **Multi-agent orchestration** — workforces of agents with a designated manager, across LLM vendors.
 - **Production tracing** — one decorator or context manager records every agent run (input, output, latency, tool calls, token usage) into your workspace, for any framework.
-- **Agent Evaluations** — score any agent (LangChain, CrewAI, OpenAI, Anthropic, HTTP, …) with LLM-as-a-judge ratings plus optional cosine and Jaccard similarity metrics.
+- **Agent Evaluations** — score any agent (LangChain, CrewAI, OpenAI, Anthropic, HTTP, …) with LLM-as-a-judge ratings plus optional cosine and Jaccard similarity metrics. Configurable judge prompt/model (OpenAI or Anthropic), per-question judge guidelines, and smoke testing for phrasing robustness.
 - **A2A** — Each agent can be published with agent-to-agent protocol compatible.
 
 ---
@@ -266,6 +266,13 @@ with client.tracer.trace("support-agent", monitor=True, pattern_ids=[pattern.id]
 
 This works independently of the dashboard's per-agent monitoring toggle (Governance > Observe > Agents), which still auto-checks every trace from an agent once enabled there, with no code changes needed either way.
 
+Read the resulting alerts back with `client.monitor.signals.list()`/`.get()`, no dashboard required:
+
+```python
+for signal in client.monitor.signals.list(severity="high"):
+    print(signal.summary, signal.occurrence_count)
+```
+
 See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
 
 ---
@@ -286,9 +293,16 @@ report = (
 print(report.average_rating)       # LLM-graded score, 0–10
 print(report.cosine_similarity)    # embedding cosine, 0–1 (None if not enabled)
 print(report.jaccard_similarity)   # token-set overlap, 0–1 (None if not enabled)
+
+print(report.summary)              # AI-generated narrative from .analyze()
+print(report.recommendations)      # list of prioritized, actionable fixes
 ```
 
-See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, and the complete API reference.
+`.analyze()` also generates a full qualitative report (strengths, weaknesses, instruction adherence, reasoning quality, and recommendations), running the same durable, multi-judge pipeline as the dashboard's "Analyze" button. `analyze(mode=..., quality_mode=..., judges=[...])` controls how items are scored and by which models. See [AI analysis report](EVALUATIONS.md#ai-analysis-report) in the full guide for the complete field and parameter reference.
+
+Ask a case's question several extra ways each run, LLM-paraphrased server-side, to catch agents that break on phrasing rather than substance, and override the judge's prompt/model per config, see [Smoke testing](EVALUATIONS.md#smoke-testing-phrasing-robustness) and [Configuring the judge](EVALUATIONS.md#configuring-the-judge) in the full guide.
+
+See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, smoke testing, judge configuration, and the complete API reference.
 
 ---
 

{agentx_python-0.6.6 → agentx_python-0.6.8}/README.md

@@ -37,7 +37,7 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
 - **MCP support** — connect any Model Context Protocol server.
 - **Multi-agent orchestration** — workforces of agents with a designated manager, across LLM vendors.
 - **Production tracing** — one decorator or context manager records every agent run (input, output, latency, tool calls, token usage) into your workspace, for any framework.
-- **Agent Evaluations** — score any agent (LangChain, CrewAI, OpenAI, Anthropic, HTTP, …) with LLM-as-a-judge ratings plus optional cosine and Jaccard similarity metrics.
+- **Agent Evaluations** — score any agent (LangChain, CrewAI, OpenAI, Anthropic, HTTP, …) with LLM-as-a-judge ratings plus optional cosine and Jaccard similarity metrics. Configurable judge prompt/model (OpenAI or Anthropic), per-question judge guidelines, and smoke testing for phrasing robustness.
 - **A2A** — Each agent can be published with agent-to-agent protocol compatible.
 
 ---
@@ -218,6 +218,13 @@ with client.tracer.trace("support-agent", monitor=True, pattern_ids=[pattern.id]
 
 This works independently of the dashboard's per-agent monitoring toggle (Governance > Observe > Agents), which still auto-checks every trace from an agent once enabled there, with no code changes needed either way.
 
+Read the resulting alerts back with `client.monitor.signals.list()`/`.get()`, no dashboard required:
+
+```python
+for signal in client.monitor.signals.list(severity="high"):
+    print(signal.summary, signal.occurrence_count)
+```
+
 See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
 
 ---
@@ -238,9 +245,16 @@ report = (
 print(report.average_rating)       # LLM-graded score, 0–10
 print(report.cosine_similarity)    # embedding cosine, 0–1 (None if not enabled)
 print(report.jaccard_similarity)   # token-set overlap, 0–1 (None if not enabled)
+
+print(report.summary)              # AI-generated narrative from .analyze()
+print(report.recommendations)      # list of prioritized, actionable fixes
 ```
 
-See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, and the complete API reference.
+`.analyze()` also generates a full qualitative report (strengths, weaknesses, instruction adherence, reasoning quality, and recommendations), running the same durable, multi-judge pipeline as the dashboard's "Analyze" button. `analyze(mode=..., quality_mode=..., judges=[...])` controls how items are scored and by which models. See [AI analysis report](EVALUATIONS.md#ai-analysis-report) in the full guide for the complete field and parameter reference.
+
+Ask a case's question several extra ways each run, LLM-paraphrased server-side, to catch agents that break on phrasing rather than substance, and override the judge's prompt/model per config, see [Smoke testing](EVALUATIONS.md#smoke-testing-phrasing-robustness) and [Configuring the judge](EVALUATIONS.md#configuring-the-judge) in the full guide.
+
+See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, smoke testing, judge configuration, and the complete API reference.
 
 ---
 

{agentx_python-0.6.6 → agentx_python-0.6.8}/agentx/agentx.py

@@ -43,7 +43,8 @@ class AgentX:
         self.evaluations = EvaluationsRunner(_eval_client)
 
         # Monitor: create/reuse patterns (client.monitor.patterns) that a trace can be checked
-        # against at send time via tracer.trace(..., monitor=True, pattern_ids=[...]).
+        # against at send time via tracer.trace(..., monitor=True, pattern_ids=[...]), then read
+        # back the resulting alerts/findings with client.monitor.signals.
         self.monitor = MonitorClient(
             api_key=self.api_key,
             sdk_version=VERSION,

{agentx_python-0.6.6 → agentx_python-0.6.8}/agentx/evaluations/_term.py

@@ -71,6 +71,12 @@ class Spinner:
         self._thread.start()
         return self
 
+    def update(self, message: str) -> None:
+        """Change the displayed message while the spinner keeps running."""
+        self._message = message
+        if not _IS_TTY:
+            print(f"  {message}...", flush=True)
+
     def __exit__(self, *_) -> None:
         if not _IS_TTY:
             return

{agentx_python-0.6.6 → agentx_python-0.6.8}/agentx/evaluations/client.py

@@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional
 import requests
 
 from agentx.evaluations.models import (
+    AnalysisStatus,
     BatchAppendResponse,
     Dataset,
     EvaluationResult,
@@ -238,8 +239,28 @@ class EvaluationsClient:
             "POST", f"/runs/{run_id}/finalize", json={"status": "completed"}
         )
 
-    def analyze_run(self, run_id: str) -> Dict[str, Any]:
-        return self._request("POST", f"/runs/{run_id}/analyze", json={}, timeout=300)
+    def analyze_run(
+        self,
+        run_id: str,
+        mode: Optional[str] = None,
+        quality_mode: Optional[str] = None,
+        judges: Optional[List[str]] = None,
+    ) -> Dict[str, Any]:
+        # Starts the durable analysis job and returns immediately (e.g. {"jobId": ..., "status":
+        # "pending"}); poll get_analysis_status() until it reaches a terminal status, then call
+        # get_report(). mode/quality_mode/judges mirror the dashboard's AnalyzeEvaluationRequest.
+        payload: Dict[str, Any] = {}
+        if mode is not None:
+            payload["mode"] = mode
+        if quality_mode is not None:
+            payload["qualityMode"] = quality_mode
+        if judges is not None:
+            payload["judges"] = [{"model": m} for m in judges]
+        return self._request("POST", f"/runs/{run_id}/analyze", json=payload, timeout=30)
+
+    def get_analysis_status(self, run_id: str) -> AnalysisStatus:
+        data = self._request("GET", f"/runs/{run_id}/analyze-status")
+        return AnalysisStatus(**data)
 
     def get_run(self, run_id: str) -> Dict[str, Any]:
         return self._request("GET", f"/runs/{run_id}")

{agentx_python-0.6.6 → agentx_python-0.6.8}/agentx/evaluations/datasets.py

@@ -27,6 +27,8 @@ class DatasetBuilder:
         acceptance_criteria: Optional[str] = None,
         rejection_criteria: Optional[str] = None,
         evaluation_criteria: Optional[str] = None,
+        judge_prompt: Optional[str] = None,
+        judge_model: Optional[str] = None,
         vector_similarity: bool = False,
         jaccard_similarity: bool = False,
         bleu_score: bool = False,
@@ -44,6 +46,13 @@ class DatasetBuilder:
             "evaluationCriteria": evaluation_criteria,
             "questions": [],
         }
+        # LLM-as-judge overrides for this dataset's own grading config. Omit either to keep the
+        # server default (raw prompt template / OpenAI gpt-5.5, see EVALUATIONS.md). judge_model
+        # must be one of client.evaluations.list_models() (OpenAI or Anthropic).
+        if judge_prompt is not None:
+            self._payload["judgePrompt"] = judge_prompt
+        if judge_model is not None:
+            self._payload["judgeModel"] = judge_model
         # Opt-in similarity metrics, surfaced on the report as cosine_similarity /
         # jaccard_similarity / bleu_score / rouge_score (computed against each
         # case's expected_results).
@@ -74,8 +83,20 @@ class DatasetBuilder:
         expected_knowledge_base: Optional[List[str]] = None,
         expected_delegations: Optional[List[str]] = None,
         follow_up_questions: Optional[List[Dict[str, Any]]] = None,
+        judge_guideline: Optional[str] = None,
+        smoke_test_count: Optional[int] = None,
+        smoke_test_guidance: Optional[str] = None,
     ) -> "DatasetBuilder":
-        main = {"query": query}
+        """Add a case. `judge_guideline` is optional grading guidance specific to this question.
+
+        `smoke_test_count`, when set (1-10), asks this question that many extra ways each
+        evaluation run, LLM-paraphrased server-side, to catch agents that are brittle to phrasing
+        rather than genuinely wrong. `smoke_test_guidance` optionally steers what kind of variants
+        get generated (e.g. tone, adversarial phrasing, different languages); the SDK never
+        generates or counts variants itself, both fields are only ever consumed server-side.
+        Ignored on `follow_up_questions`, only the opening question of a case can be smoke-tested.
+        """
+        main: Dict[str, Any] = {"query": query}
         if expected_results:
             main["expectedResults"] = expected_results
         if expected_capabilities:
@@ -84,6 +105,12 @@ class DatasetBuilder:
             main["expectedKnowledgeBase"] = expected_knowledge_base
         if expected_delegations:
             main["expectedDelegations"] = expected_delegations
+        if judge_guideline:
+            main["judgeGuideline"] = judge_guideline
+        if smoke_test_count:
+            main["smokeTest"] = {"enabled": True, "count": smoke_test_count}
+            if smoke_test_guidance:
+                main["smokeTest"]["guidance"] = smoke_test_guidance
         self._payload["questions"].append(
             {
                 "main_question": main,
@@ -217,6 +244,8 @@ class DatasetClient:
         acceptance_criteria: Optional[str] = None,
         rejection_criteria: Optional[str] = None,
         evaluation_criteria: Optional[str] = None,
+        judge_prompt: Optional[str] = None,
+        judge_model: Optional[str] = None,
         vector_similarity: bool = False,
         jaccard_similarity: bool = False,
         bleu_score: bool = False,
@@ -232,6 +261,8 @@ class DatasetClient:
             acceptance_criteria=acceptance_criteria,
             rejection_criteria=rejection_criteria,
             evaluation_criteria=evaluation_criteria,
+            judge_prompt=judge_prompt,
+            judge_model=judge_model,
             vector_similarity=vector_similarity,
             jaccard_similarity=jaccard_similarity,
             bleu_score=bleu_score,

{agentx_python-0.6.6 → agentx_python-0.6.8}/agentx/evaluations/evaluation_settings.py

@@ -24,6 +24,8 @@ class EvaluationSettingsBuilder:
         acceptance_criteria: Optional[str] = None,
         rejection_criteria: Optional[str] = None,
         evaluation_criteria: Optional[str] = None,
+        judge_prompt: Optional[str] = None,
+        judge_model: Optional[str] = None,
         vector_similarity: bool = False,
         jaccard_similarity: bool = False,
         bleu_score: bool = False,
@@ -40,6 +42,13 @@ class EvaluationSettingsBuilder:
             "rejectionCriteria": rejection_criteria,
             "evaluationCriteria": evaluation_criteria,
         }
+        # LLM-as-judge overrides. Omit either to keep the server default (raw prompt template /
+        # OpenAI gpt-5.5, see EVALUATIONS.md). judge_model must be one of
+        # client.evaluations.list_models() (OpenAI or Anthropic).
+        if judge_prompt is not None:
+            self._payload["judgePrompt"] = judge_prompt
+        if judge_model is not None:
+            self._payload["judgeModel"] = judge_model
         # Opt-in similarity metrics, mirrors DatasetBuilder's config kwargs.
         if vector_similarity:
             vs: Dict[str, Any] = {"enabled": True}
@@ -79,6 +88,8 @@ class EvaluationSettingsClient:
         acceptance_criteria: Optional[str] = None,
         rejection_criteria: Optional[str] = None,
         evaluation_criteria: Optional[str] = None,
+        judge_prompt: Optional[str] = None,
+        judge_model: Optional[str] = None,
         vector_similarity: bool = False,
         jaccard_similarity: bool = False,
         bleu_score: bool = False,
@@ -94,6 +105,8 @@ class EvaluationSettingsClient:
             acceptance_criteria=acceptance_criteria,
             rejection_criteria=rejection_criteria,
             evaluation_criteria=evaluation_criteria,
+            judge_prompt=judge_prompt,
+            judge_model=judge_model,
             vector_similarity=vector_similarity,
             jaccard_similarity=jaccard_similarity,
             bleu_score=bleu_score,

{agentx_python-0.6.6 → agentx_python-0.6.8}/agentx/evaluations/models.py

@@ -31,14 +31,30 @@ class ObservableTrace(BaseModel):
 # ---------------------------------------------------------------------------
 
 
+class SmokeTestSettings(BaseModel):
+    """Only meaningful on a question's main_question. See DatasetBuilder.add_case's
+    smoke_test_count/smoke_test_guidance for how to set this."""
+
+    enabled: bool = False
+    count: int = 1
+    guidance: Optional[str] = None
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
 class TestCase(BaseModel):
     query: str
-    expected_results: Optional[str] = None
-    expected_capabilities: Optional[List[str]] = None
-    expected_knowledge_base: Optional[List[str]] = None
-    expected_delegations: Optional[List[str]] = None
+    expected_results: Optional[str] = Field(default=None, alias="expectedResults")
+    expected_capabilities: Optional[List[str]] = Field(default=None, alias="expectedCapabilities")
+    expected_knowledge_base: Optional[List[str]] = Field(default=None, alias="expectedKnowledgeBase")
+    expected_delegations: Optional[List[str]] = Field(default=None, alias="expectedDelegations")
+    judge_guideline: Optional[str] = Field(default=None, alias="judgeGuideline")
+    smoke_test: Optional[SmokeTestSettings] = Field(default=None, alias="smokeTest")
 
     class Config:
+        populate_by_name = True
         extra = "ignore"
 
 
@@ -89,6 +105,10 @@ class EvaluationSettings(BaseModel):
     acceptance_criteria: Optional[str] = Field(default=None, alias="acceptanceCriteria")
     rejection_criteria: Optional[str] = Field(default=None, alias="rejectionCriteria")
     evaluation_criteria: Optional[str] = Field(default=None, alias="evaluationCriteria")
+    # LLM-as-judge overrides. None means "use the server default" (raw prompt template / OpenAI
+    # gpt-5.5). See client.evaluations.settings.builder(judge_prompt=..., judge_model=...).
+    judge_prompt: Optional[str] = Field(default=None, alias="judgePrompt")
+    judge_model: Optional[str] = Field(default=None, alias="judgeModel")
     status: str = "published"
     # Sovereignty & Portability — models selected to compare when this config runs.
     # Hoisted from the nested ``sovereigntyIndex`` object when enabled.
@@ -202,12 +222,30 @@ class LiveStatistics(BaseModel):
         extra = "ignore"
 
 
+class SmokeTestVariantGroup(BaseModel):
+    """Paraphrased variants for one question, generated server-side (reusing the same
+    generation the dashboard's native runs use) and frozen for the lifetime of the run.
+    The SDK never generates or counts these itself, it only consumes what's returned here."""
+
+    question_index: int = Field(alias="questionIndex")
+    variants: List[str] = Field(default_factory=list)
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
 class EvaluationRun(BaseModel):
     run_id: str = Field(alias="runId")
     dataset_id: str = Field(alias="datasetId")
     dataset_version_id: Optional[str] = Field(default=None, alias="datasetVersionId")
     status: str = "in_progress"
     limits: ServerLimits = Field(default_factory=ServerLimits)
+    # Present only when at least one question in the dataset has smokeTest.enabled. See
+    # SmokeTestVariantGroup.
+    smoke_test_variants: Optional[List[SmokeTestVariantGroup]] = Field(
+        default=None, alias="smokeTestVariants"
+    )
 
     class Config:
         populate_by_name = True
@@ -232,6 +270,12 @@ class EvaluationCase(BaseModel):
     # dataset selects comparison models; your callable can read it to pick the
     # model. The SDK also tags the submitted result with it.
     model: Optional[str] = None
+    # Smoke test: True when `query` is a server-generated paraphrase variant rather than the
+    # dataset's original question text (see SmokeTestVariantGroup). Your callable doesn't need
+    # to branch on this, `query` is already the text to ask, but it's available if you want to
+    # log or handle variants differently.
+    is_smoke_test_variant: bool = False
+    smoke_test_variant_text: Optional[str] = None
 
     class Config:
         extra = "ignore"
@@ -278,6 +322,10 @@ class EvaluationResult(BaseModel):
     # the dashboard's "Message Trace Details -> Execution Timeline" render the full execution
     # trace for this case, not just the lightweight observable_trace events above.
     trace_id: Optional[str] = Field(default=None, alias="traceId")
+    # Smoke test: set by execute() from the originating EvaluationCase, not something you need to
+    # set yourself when returning a plain str/dict from your callable.
+    is_smoke_test_variant: Optional[bool] = Field(default=None, alias="isSmokeTestVariant")
+    smoke_test_variant_text: Optional[str] = Field(default=None, alias="smokeTestVariantText")
 
     class Config:
         populate_by_name = True
@@ -438,11 +486,9 @@ class SovereigntyIndex(BaseModel):
         extra = "ignore"
 
 
-class Report(BaseModel):
-    run_id: str = Field(alias="runId")
-    dataset_id: str = Field(alias="datasetId")
-    status: str = "completed"
-    statistics: Optional[ReportStatistics] = None
+class AnalysisResult(BaseModel):
+    """Shared qualitative-report fields, produced by ``client.evaluations.run(...).analyze()``."""
+
     summary: Optional[str] = None
     consistency_score: Optional[float] = Field(default=None, alias="consistencyScore")
     instruction_adherence: Optional[ReportInstructionAdherence] = Field(
@@ -461,6 +507,17 @@ class Report(BaseModel):
     weaknesses: List[str] = Field(default_factory=list)
     overall_rating: Optional[str] = Field(default=None, alias="overallRating")
     recommendations: List[ReportRecommendation] = Field(default_factory=list)
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class Report(AnalysisResult):
+    run_id: str = Field(alias="runId")
+    dataset_id: str = Field(alias="datasetId")
+    status: str = "completed"
+    statistics: Optional[ReportStatistics] = None
     low_scoring_cases: List[Dict[str, Any]] = Field(
         default_factory=list, alias="lowScoringCases"
     )
@@ -548,3 +605,52 @@ class Report(BaseModel):
     def average_rating(self) -> Optional[float]:
         """Convenience accessor matching cosine_similarity / jaccard_similarity."""
         return self.statistics.average_rating if self.statistics is not None else None
+
+
+class AnalysisLevelProgress(BaseModel):
+    total: int = 0
+    completed: int = 0
+    failed: int = 0
+    percentage: int = 0
+
+    class Config:
+        extra = "ignore"
+
+
+class AnalysisProgress(BaseModel):
+    overall_percentage: int = Field(default=0, alias="overallPercentage")
+    current_level: Optional[str] = Field(default=None, alias="currentLevel")
+    levels: Dict[str, AnalysisLevelProgress] = Field(default_factory=dict)
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class AnalysisFailureReason(BaseModel):
+    code: str
+    message: str
+    retryable: bool = False
+
+    class Config:
+        extra = "ignore"
+
+
+class AnalysisStatus(BaseModel):
+    """Returned by ``client.evaluations.run(...).analyze()``'s polling loop
+    (``EvaluationsClient.get_analysis_status``). ``status`` is terminal once it's one of
+    "completed", "partially_failed", or "failed"."""
+
+    job_id: Optional[str] = Field(default=None, alias="jobId")
+    status: str = "not_started"
+    progress: AnalysisProgress = Field(default_factory=AnalysisProgress)
+    failure_reason: Optional[AnalysisFailureReason] = Field(default=None, alias="failureReason")
+    warnings: List[Dict[str, Any]] = Field(default_factory=list)
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+    @property
+    def is_terminal(self) -> bool:
+        return self.status in ("completed", "partially_failed", "failed")

{agentx_python-0.6.6 → agentx_python-0.6.8}/agentx/evaluations/results.py

@@ -30,6 +30,8 @@ def normalize_result(
         raw.case_id = case.case_id
         raw.question_index = case.question_index
         raw.run_number = case.run_number
+        raw.is_smoke_test_variant = case.is_smoke_test_variant
+        raw.smoke_test_variant_text = case.smoke_test_variant_text
         return raw
 
     output: Optional[dict] = None
@@ -106,6 +108,8 @@ def normalize_result(
         ),
         metadata=metadata,
         traceId=trace_id,
+        isSmokeTestVariant=case.is_smoke_test_variant,
+        smokeTestVariantText=case.smoke_test_variant_text,
     )
 
 
@@ -126,4 +130,6 @@ def normalize_error(
             retryable=False,
         ),
         timings=ResultTimings(latencyMs=latency_ms) if latency_ms is not None else None,
+        isSmokeTestVariant=case.is_smoke_test_variant,
+        smokeTestVariantText=case.smoke_test_variant_text,
     )

{agentx_python-0.6.6 → agentx_python-0.6.8}/agentx/evaluations/runner.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import time
 import uuid
 from typing import Any, Callable, Dict, List, Optional, Set, Union
 
@@ -9,6 +10,7 @@ from agentx.evaluations.adapters.precomputed import PrecomputedAdapter
 from agentx.evaluations.adapters.http_endpoint import HttpEndpointAdapter
 from agentx.evaluations.client import EvaluationsClient
 from agentx.evaluations.models import (
+    AnalysisStatus,
     Dataset,
     EvaluationCase,
     EvaluationResult,
@@ -43,6 +45,15 @@ AdapterLike = Union[
     HttpEndpointAdapter,
 ]
 
+_ANALYSIS_LEVEL_LABELS = {
+    "l1_score": "scoring responses",
+    "l2_question_reduce": "reducing questions",
+    "l3_cluster_reduce": "reducing clusters",
+    "l4_final_reduce": "writing final report",
+}
+
+_DEFAULT_JUDGE_MODEL = "gpt-5.5"
+
 
 class EvaluationRunContext:
     """
@@ -82,7 +93,7 @@ class EvaluationRunContext:
     def execute(self, adapter: AdapterLike) -> "EvaluationRunContext":
         """Run all cases locally and submit batches to AgentX."""
         normalized = _wrap_adapter(adapter)
-        cases = _build_cases(self._dataset, self._evaluation_settings)
+        cases = _build_cases(self._dataset, self._run, self._evaluation_settings)
         max_batch = self._run.limits.max_batch_size
 
         # Banner
@@ -97,6 +108,7 @@ class EvaluationRunContext:
             if self._evaluation_settings
             else self._dataset.number_of_requests
         )
+        n_smoke = sum(1 for c in cases if c.is_smoke_test_variant)
 
         print(cyan(sep))
         print(f"  {bold('AgentX Evaluation')}  {dim('—')}  {name}")
@@ -105,9 +117,11 @@ class EvaluationRunContext:
         if display:
             print(f"  {dim('Agent :')} {display}  {dim(f'({framework} / {runtime})')}")
         print()
-        print(
-            f"{bold('Executing')}  {n_q} question{'s' if n_q != 1 else ''} × {n_r} run{'s' if n_r != 1 else ''}"
-        )
+        exec_line = f"{bold('Executing')}  {n_q} question{'s' if n_q != 1 else ''} × {n_r} run{'s' if n_r != 1 else ''}"
+        if n_smoke:
+            variant_word = "variant" if n_smoke == 1 else "variants"
+            exec_line += f"  {dim(f'(+{n_smoke} smoke-test {variant_word})')}"
+        print(exec_line)
 
         # Resume: skip already-submitted keys
         already_done = self._fetch_submitted_keys()
@@ -228,12 +242,61 @@ class EvaluationRunContext:
     # Step 3: analyze + report
     # ------------------------------------------------------------------
 
-    def analyze(self) -> Report:
+    def analyze(
+        self,
+        mode: Optional[str] = None,
+        quality_mode: Optional[str] = None,
+        judges: Optional[List[str]] = None,
+        poll_interval: float = 5.0,
+        timeout: float = 1800.0,
+    ) -> Report:
+        """Generate the qualitative AI analysis report.
+
+        Runs the same durable, multi-stage pipeline as the dashboard's "Analyze" button: each
+        response is scored by 1-3 LLM judges (``judges``), then reduced into the final report.
+        This starts the job and polls until it finishes, which can take noticeably longer than a
+        single LLM call for larger runs.
+
+        Args:
+            mode: "auto" (default), "sync", or "batch" - how item scoring executes server-side.
+            quality_mode: "quality_first" or "balanced" - how many items get a second/third judge.
+            judges: 1-3 model ids, e.g. ``["gpt-5.5", "claude-opus-4-8"]``. Defaults to a single
+                judge, ``["gpt-5.5"]``, rather than the dashboard's 3-judge default - SDK runs are
+                typically lighter-weight, quick-start evaluations.
+            poll_interval: seconds between status checks while waiting.
+            timeout: give up waiting after this many seconds (the job keeps running server-side;
+                call ``get_report()`` later to check on it).
+        """
+        if judges is not None and not (1 <= len(judges) <= 3):
+            raise ValueError("judges must contain 1-3 model ids")
+        resolved_judges = judges if judges is not None else [_DEFAULT_JUDGE_MODEL]
+
         print()
-        with Spinner("Analyzing — AI is reviewing your results"):
+        with Spinner("Analyzing — AI is reviewing your results") as spinner:
             try:
-                self._client.analyze_run(self._run.run_id)
-                print(f"  {green('✓')}  Analysis complete")
+                self._client.analyze_run(
+                    self._run.run_id,
+                    mode=mode,
+                    quality_mode=quality_mode,
+                    judges=resolved_judges,
+                )
+                deadline = time.monotonic() + timeout
+                status = self._client.get_analysis_status(self._run.run_id)
+                while not status.is_terminal and time.monotonic() < deadline:
+                    level = _ANALYSIS_LEVEL_LABELS.get(status.progress.current_level, "")
+                    spinner.update(
+                        f"Analyzing, {level + ' ' if level else ''}{status.progress.overall_percentage}%"
+                    )
+                    time.sleep(poll_interval)
+                    status = self._client.get_analysis_status(self._run.run_id)
+
+                if not status.is_terminal:
+                    print(f"  {yellow('!')}  Still running after {int(timeout)}s, check the dashboard for status")
+                elif status.status == "failed":
+                    reason = status.failure_reason.message if status.failure_reason else "unknown error"
+                    print(f"  {red('✗')}  Analyze failed: {dim(reason)}")
+                else:
+                    print(f"  {green('✓')}  Analysis complete")
             except Exception as exc:
                 print(f"  {red('✗')}  Analyze failed: {dim(str(exc))}")
                 logger.warning("Analyze request failed: %s", exc)
@@ -280,6 +343,12 @@ class EvaluationsRunner:
         against."""
         return self._client.list_models(provider)
 
+    def get_analysis_status(self, run_id: str) -> AnalysisStatus:
+        """Check on an in-progress ``.analyze()`` job by run id, without needing
+        the ``EvaluationRunContext`` that started it (e.g. from a separate
+        script execution)."""
+        return self._client.get_analysis_status(run_id)
+
     def run(
         self,
         dataset_id: str,
@@ -334,7 +403,9 @@ def _wrap_adapter(adapter: AdapterLike) -> Callable[[EvaluationCase], Evaluation
 
 
 def _build_cases(
-    dataset: Dataset, evaluation_settings: Optional[EvaluationSettings] = None
+    dataset: Dataset,
+    run: EvaluationRun,
+    evaluation_settings: Optional[EvaluationSettings] = None,
 ) -> List[EvaluationCase]:
     cases: List[EvaluationCase] = []
     # When an independent evaluation_settings was chosen (evaluation_settings_id
@@ -353,6 +424,14 @@ def _build_cases(
         evaluation_settings.sovereignty_models if evaluation_settings else dataset.sovereignty_models
     )
     models: List[Optional[str]] = list(sovereignty_models) or [None]
+    # Smoke test: variant text is generated and counted entirely server-side (POST /runs, reusing
+    # the same generation the dashboard's native runs use) and handed back on `run`. The SDK never
+    # re-derives eligibility, count, or text itself, it only turns what the server already decided
+    # into extra cases. Not multiplied across sovereignty_models, matching the server's counting in
+    # finalize/missing-results (one case per variant per question, regardless of comparison models).
+    smoke_variants_by_question = {
+        group.question_index: group.variants for group in (run.smoke_test_variants or [])
+    }
     for q_idx, question in enumerate(dataset.questions):
         mq = question.main_question
         for run_num in range(1, n_runs + 1):
@@ -371,6 +450,21 @@ def _build_cases(
                         model=model,
                     )
                 )
+        for variant_idx, variant_text in enumerate(smoke_variants_by_question.get(q_idx, [])):
+            cases.append(
+                EvaluationCase(
+                    case_id=f"case-{q_idx}",
+                    question_index=q_idx,
+                    run_number=n_runs + variant_idx + 1,
+                    query=variant_text,
+                    expected_results=mq.expected_results,
+                    expected_capabilities=mq.expected_capabilities,
+                    expected_knowledge_base=mq.expected_knowledge_base,
+                    expected_delegations=mq.expected_delegations,
+                    is_smoke_test_variant=True,
+                    smoke_test_variant_text=variant_text,
+                )
+            )
     return cases
 
 

{agentx_python-0.6.6 → agentx_python-0.6.8}/agentx/monitor/__init__.py

@@ -1,10 +1,14 @@
 from agentx.monitor.client import MonitorClient
-from agentx.monitor.models import MonitorPattern
+from agentx.monitor.models import MonitorPattern, MonitorSignal, SignalOccurrence
 from agentx.monitor.patterns import MonitorPatternBuilder, MonitorPatternClient
+from agentx.monitor.signals import MonitorSignalClient
 
 __all__ = [
     "MonitorClient",
     "MonitorPattern",
     "MonitorPatternBuilder",
     "MonitorPatternClient",
+    "MonitorSignal",
+    "SignalOccurrence",
+    "MonitorSignalClient",
 ]

{agentx_python-0.6.6 → agentx_python-0.6.8}/agentx/monitor/client.py

@@ -7,7 +7,7 @@ from typing import Any, List, Optional
 
 import requests
 
-from agentx.monitor.models import MonitorPattern
+from agentx.monitor.models import MonitorPattern, MonitorSignal
 
 logger = logging.getLogger(__name__)
 
@@ -69,8 +69,10 @@ class MonitorClient:
         )
 
         from agentx.monitor.patterns import MonitorPatternClient
+        from agentx.monitor.signals import MonitorSignalClient
 
         self.patterns = MonitorPatternClient(self)
+        self.signals = MonitorSignalClient(self)
 
     # ------------------------------------------------------------------
     # Low-level HTTP
@@ -132,3 +134,35 @@ class MonitorClient:
             "GET", f"/patterns/{pattern_id}", params=self._workspace_params()
         )
         return MonitorPattern(**data["pattern"])
+
+    # ------------------------------------------------------------------
+    # Signal endpoints
+    # ------------------------------------------------------------------
+
+    def list_signals(
+        self,
+        polarity: Optional[str] = None,
+        status: Optional[str] = None,
+        severity: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        limit: int = 50,
+    ) -> List[MonitorSignal]:
+        params = {**(self._workspace_params() or {})}
+        if polarity is not None:
+            params["polarity"] = polarity
+        if status is not None:
+            params["status"] = status
+        if severity is not None:
+            params["severity"] = severity
+        if agent_id is not None:
+            params["agentId"] = agent_id
+        if limit is not None:
+            params["limit"] = limit
+        data = self._request("GET", "/signals", params=params)
+        return [MonitorSignal(**s) for s in data.get("signals", [])]
+
+    def get_signal(self, signal_id: str) -> MonitorSignal:
+        data = self._request(
+            "GET", f"/signals/{signal_id}", params=self._workspace_params()
+        )
+        return MonitorSignal(**data["signal"])

agentx_python-0.6.8/agentx/monitor/models.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+from pydantic import BaseModel, Field
+
+
+class MonitorPattern(BaseModel):
+    """A detection rule checked against production traces. Built via
+    ``client.monitor.patterns.builder(...).publish()`` and referenced by id as a
+    ``pattern_ids`` entry in ``tracer.trace(..., monitor=True, pattern_ids=[...])``.
+
+    A "failure" pattern (the default) raises a signal to triage; a "proper" pattern logs a
+    healthy tally instead. Only one of ``include_terms``/``regex``/``semantic_prompt`` is
+    meaningful at a time, selected by ``detector_kind``.
+    """
+
+    id: str = Field(alias="_id")
+    key: str
+    name: str
+    description: Optional[str] = None
+    category: Optional[str] = None
+    detector_kind: str = Field(default="contains", alias="detectorKind")
+    match_target: List[str] = Field(default_factory=lambda: ["response"], alias="matchTarget")
+    match_mode: str = Field(default="any", alias="matchMode")
+    include_terms: List[str] = Field(default_factory=list, alias="includeTerms")
+    exclude_terms: List[str] = Field(default_factory=list, alias="excludeTerms")
+    regex: Optional[str] = None
+    semantic_prompt: Optional[str] = Field(default=None, alias="semanticPrompt")
+    severity: str = "medium"
+    polarity: str = "failure"
+    enabled: bool = True
+    sample_rate: float = Field(default=1.0, alias="sampleRate")
+    scope_mode: str = Field(default="all", alias="scopeMode")
+    agent_ids: List[str] = Field(default_factory=list, alias="agentIds")
+
+    class Config:
+        extra = "ignore"
+
+
+class SignalOccurrence(BaseModel):
+    """One hit behind a signal, capped at the server's most recent N per signal."""
+
+    agent_id: Optional[Dict[str, Any]] = Field(default=None, alias="agentId")
+    conversation_id: Optional[str] = Field(default=None, alias="conversationId")
+    message_id: Optional[str] = Field(default=None, alias="messageId")
+    trace_id: Optional[str] = Field(default=None, alias="traceId")
+    seen_at: Optional[str] = Field(default=None, alias="seenAt")
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class MonitorSignal(BaseModel):
+    """An alert/finding produced when a trace matched a pattern (or, for a "proper" pattern,
+    a healthy tally). Read-only from the SDK, see ``client.monitor.signals.list()/get()``, a
+    signal is the system's output from checking traces against patterns, not something an SDK
+    caller constructs."""
+
+    id: str = Field(alias="_id")
+    workspace_id: Optional[str] = Field(default=None, alias="workspaceId")
+    # Populated to {"_id", "name", "avatar"} when set, since the server populates this field.
+    agent_id: Optional[Dict[str, Any]] = Field(default=None, alias="agentId")
+    conversation_id: Optional[str] = Field(default=None, alias="conversationId")
+    message_id: Optional[str] = Field(default=None, alias="messageId")
+    type: str
+    severity: str = "medium"
+    polarity: str = "failure"
+    status: str = "open"
+    score: Optional[float] = None
+    threshold: Optional[float] = None
+    summary: str
+    pattern_key: str = Field(alias="patternKey")
+    evidence: Optional[Dict[str, Any]] = None
+    root_cause: Optional[str] = Field(default=None, alias="rootCause")
+    recommended_actions: List[str] = Field(default_factory=list, alias="recommendedActions")
+    review_status: Optional[str] = Field(default=None, alias="reviewStatus")
+    first_seen_at: Optional[str] = Field(default=None, alias="firstSeenAt")
+    last_seen_at: Optional[str] = Field(default=None, alias="lastSeenAt")
+    occurrence_count: int = Field(default=1, alias="occurrenceCount")
+    occurrences: List[SignalOccurrence] = Field(default_factory=list)
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"

agentx_python-0.6.8/agentx/monitor/signals.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import logging
+from typing import List, Optional, TYPE_CHECKING
+
+from agentx.monitor.models import MonitorSignal
+
+if TYPE_CHECKING:
+    from agentx.monitor.client import MonitorClient
+
+logger = logging.getLogger(__name__)
+
+
+class MonitorSignalClient:
+    """Thin wrapper surfaced as ``client.monitor.signals``. Read-only: a signal is produced by
+    checking a trace against patterns (see ``tracer.trace(..., monitor=True, pattern_ids=[...])``),
+    not something the SDK creates directly."""
+
+    def __init__(self, client: "MonitorClient"):
+        self._client = client
+
+    def list(
+        self,
+        polarity: Optional[str] = None,
+        status: Optional[str] = None,
+        severity: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        limit: int = 50,
+    ) -> List[MonitorSignal]:
+        """List this workspace's signals, most recently seen first.
+
+        `polarity` defaults server-side to failures only ("proper", the healthy tally, is
+        excluded); pass ``polarity="all"`` to include both, or ``"proper"``/``"failure"`` to
+        narrow to one kind. `limit` is capped at 100 server-side.
+        """
+        return self._client.list_signals(
+            polarity=polarity, status=status, severity=severity, agent_id=agent_id, limit=limit
+        )
+
+    def get(self, signal_id: str) -> MonitorSignal:
+        return self._client.get_signal(signal_id)

agentx_python-0.6.8/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.6.8"

{agentx_python-0.6.6 → agentx_python-0.6.8/agentx_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.6
+Version: 0.6.8
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team
@@ -85,7 +85,7 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
 - **MCP support** — connect any Model Context Protocol server.
 - **Multi-agent orchestration** — workforces of agents with a designated manager, across LLM vendors.
 - **Production tracing** — one decorator or context manager records every agent run (input, output, latency, tool calls, token usage) into your workspace, for any framework.
-- **Agent Evaluations** — score any agent (LangChain, CrewAI, OpenAI, Anthropic, HTTP, …) with LLM-as-a-judge ratings plus optional cosine and Jaccard similarity metrics.
+- **Agent Evaluations** — score any agent (LangChain, CrewAI, OpenAI, Anthropic, HTTP, …) with LLM-as-a-judge ratings plus optional cosine and Jaccard similarity metrics. Configurable judge prompt/model (OpenAI or Anthropic), per-question judge guidelines, and smoke testing for phrasing robustness.
 - **A2A** — Each agent can be published with agent-to-agent protocol compatible.
 
 ---
@@ -266,6 +266,13 @@ with client.tracer.trace("support-agent", monitor=True, pattern_ids=[pattern.id]
 
 This works independently of the dashboard's per-agent monitoring toggle (Governance > Observe > Agents), which still auto-checks every trace from an agent once enabled there, with no code changes needed either way.
 
+Read the resulting alerts back with `client.monitor.signals.list()`/`.get()`, no dashboard required:
+
+```python
+for signal in client.monitor.signals.list(severity="high"):
+    print(signal.summary, signal.occurrence_count)
+```
+
 See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
 
 ---
@@ -286,9 +293,16 @@ report = (
 print(report.average_rating)       # LLM-graded score, 0–10
 print(report.cosine_similarity)    # embedding cosine, 0–1 (None if not enabled)
 print(report.jaccard_similarity)   # token-set overlap, 0–1 (None if not enabled)
+
+print(report.summary)              # AI-generated narrative from .analyze()
+print(report.recommendations)      # list of prioritized, actionable fixes
 ```
 
-See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, and the complete API reference.
+`.analyze()` also generates a full qualitative report (strengths, weaknesses, instruction adherence, reasoning quality, and recommendations), running the same durable, multi-judge pipeline as the dashboard's "Analyze" button. `analyze(mode=..., quality_mode=..., judges=[...])` controls how items are scored and by which models. See [AI analysis report](EVALUATIONS.md#ai-analysis-report) in the full guide for the complete field and parameter reference.
+
+Ask a case's question several extra ways each run, LLM-paraphrased server-side, to catch agents that break on phrasing rather than substance, and override the judge's prompt/model per config, see [Smoke testing](EVALUATIONS.md#smoke-testing-phrasing-robustness) and [Configuring the judge](EVALUATIONS.md#configuring-the-judge) in the full guide.
+
+See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, smoke testing, judge configuration, and the complete API reference.
 
 ---
 

{agentx_python-0.6.6 → agentx_python-0.6.8}/agentx_python.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ agentx/monitor/__init__.py
 agentx/monitor/client.py
 agentx/monitor/models.py
 agentx/monitor/patterns.py
+agentx/monitor/signals.py
 agentx/resources/__init__.py
 agentx/resources/agent.py
 agentx/resources/conversation.py

agentx_python-0.6.6/agentx/monitor/models.py

@@ -1,37 +0,0 @@
-from __future__ import annotations
-
-from typing import List, Optional
-from pydantic import BaseModel, Field
-
-
-class MonitorPattern(BaseModel):
-    """A detection rule checked against production traces. Built via
-    ``client.monitor.patterns.builder(...).publish()`` and referenced by id as a
-    ``pattern_ids`` entry in ``tracer.trace(..., monitor=True, pattern_ids=[...])``.
-
-    A "failure" pattern (the default) raises a signal to triage; a "proper" pattern logs a
-    healthy tally instead. Only one of ``include_terms``/``regex``/``semantic_prompt`` is
-    meaningful at a time, selected by ``detector_kind``.
-    """
-
-    id: str = Field(alias="_id")
-    key: str
-    name: str
-    description: Optional[str] = None
-    category: Optional[str] = None
-    detector_kind: str = Field(default="contains", alias="detectorKind")
-    match_target: List[str] = Field(default_factory=lambda: ["response"], alias="matchTarget")
-    match_mode: str = Field(default="any", alias="matchMode")
-    include_terms: List[str] = Field(default_factory=list, alias="includeTerms")
-    exclude_terms: List[str] = Field(default_factory=list, alias="excludeTerms")
-    regex: Optional[str] = None
-    semantic_prompt: Optional[str] = Field(default=None, alias="semanticPrompt")
-    severity: str = "medium"
-    polarity: str = "failure"
-    enabled: bool = True
-    sample_rate: float = Field(default=1.0, alias="sampleRate")
-    scope_mode: str = Field(default="all", alias="scopeMode")
-    agent_ids: List[str] = Field(default_factory=list, alias="agentIds")
-
-    class Config:
-        extra = "ignore"

agentx_python-0.6.6/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.6.6"