agentx-python 0.6.7__tar.gz → 0.6.9__tar.gz

{agentx_python-0.6.7 → agentx_python-0.6.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.7
+Version: 0.6.9
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team
@@ -85,7 +85,7 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
 - **MCP support** — connect any Model Context Protocol server.
 - **Multi-agent orchestration** — workforces of agents with a designated manager, across LLM vendors.
 - **Production tracing** — one decorator or context manager records every agent run (input, output, latency, tool calls, token usage) into your workspace, for any framework.
-- **Agent Evaluations** — score any agent (LangChain, CrewAI, OpenAI, Anthropic, HTTP, …) with LLM-as-a-judge ratings plus optional cosine and Jaccard similarity metrics.
+- **Agent Evaluations** — score any agent (LangChain, CrewAI, OpenAI, Anthropic, HTTP, …) with LLM-as-a-judge ratings plus optional cosine and Jaccard similarity metrics. Configurable judge prompt/model (OpenAI or Anthropic), per-question judge guidelines, and smoke testing for phrasing robustness.
 - **A2A** — Each agent can be published with agent-to-agent protocol compatible.
 
 ---
@@ -266,6 +266,13 @@ with client.tracer.trace("support-agent", monitor=True, pattern_ids=[pattern.id]
 
 This works independently of the dashboard's per-agent monitoring toggle (Governance > Observe > Agents), which still auto-checks every trace from an agent once enabled there, with no code changes needed either way.
 
+Read the resulting alerts back with `client.monitor.signals.list()`/`.get()`, no dashboard required:
+
+```python
+for signal in client.monitor.signals.list(severity="high"):
+    print(signal.summary, signal.occurrence_count)
+```
+
 See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
 
 ---
@@ -293,7 +300,9 @@ print(report.recommendations)      # list of prioritized, actionable fixes
 
 `.analyze()` also generates a full qualitative report (strengths, weaknesses, instruction adherence, reasoning quality, and recommendations), running the same durable, multi-judge pipeline as the dashboard's "Analyze" button. `analyze(mode=..., quality_mode=..., judges=[...])` controls how items are scored and by which models. See [AI analysis report](EVALUATIONS.md#ai-analysis-report) in the full guide for the complete field and parameter reference.
 
-See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, and the complete API reference.
+Ask a case's question several extra ways each run, LLM-paraphrased server-side, to catch agents that break on phrasing rather than substance, and override the judge's prompt/model per config, see [Smoke testing](EVALUATIONS.md#smoke-testing-phrasing-robustness) and [Configuring the judge](EVALUATIONS.md#configuring-the-judge) in the full guide.
+
+See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, smoke testing, judge configuration, and the complete API reference.
 
 ---
 

{agentx_python-0.6.7 → agentx_python-0.6.9}/README.md

@@ -37,7 +37,7 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
 - **MCP support** — connect any Model Context Protocol server.
 - **Multi-agent orchestration** — workforces of agents with a designated manager, across LLM vendors.
 - **Production tracing** — one decorator or context manager records every agent run (input, output, latency, tool calls, token usage) into your workspace, for any framework.
-- **Agent Evaluations** — score any agent (LangChain, CrewAI, OpenAI, Anthropic, HTTP, …) with LLM-as-a-judge ratings plus optional cosine and Jaccard similarity metrics.
+- **Agent Evaluations** — score any agent (LangChain, CrewAI, OpenAI, Anthropic, HTTP, …) with LLM-as-a-judge ratings plus optional cosine and Jaccard similarity metrics. Configurable judge prompt/model (OpenAI or Anthropic), per-question judge guidelines, and smoke testing for phrasing robustness.
 - **A2A** — Each agent can be published with agent-to-agent protocol compatible.
 
 ---
@@ -218,6 +218,13 @@ with client.tracer.trace("support-agent", monitor=True, pattern_ids=[pattern.id]
 
 This works independently of the dashboard's per-agent monitoring toggle (Governance > Observe > Agents), which still auto-checks every trace from an agent once enabled there, with no code changes needed either way.
 
+Read the resulting alerts back with `client.monitor.signals.list()`/`.get()`, no dashboard required:
+
+```python
+for signal in client.monitor.signals.list(severity="high"):
+    print(signal.summary, signal.occurrence_count)
+```
+
 See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
 
 ---
@@ -245,7 +252,9 @@ print(report.recommendations)      # list of prioritized, actionable fixes
 
 `.analyze()` also generates a full qualitative report (strengths, weaknesses, instruction adherence, reasoning quality, and recommendations), running the same durable, multi-judge pipeline as the dashboard's "Analyze" button. `analyze(mode=..., quality_mode=..., judges=[...])` controls how items are scored and by which models. See [AI analysis report](EVALUATIONS.md#ai-analysis-report) in the full guide for the complete field and parameter reference.
 
-See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, and the complete API reference.
+Ask a case's question several extra ways each run, LLM-paraphrased server-side, to catch agents that break on phrasing rather than substance, and override the judge's prompt/model per config, see [Smoke testing](EVALUATIONS.md#smoke-testing-phrasing-robustness) and [Configuring the judge](EVALUATIONS.md#configuring-the-judge) in the full guide.
+
+See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, smoke testing, judge configuration, and the complete API reference.
 
 ---
 

{agentx_python-0.6.7 → agentx_python-0.6.9}/agentx/agentx.py

@@ -43,7 +43,8 @@ class AgentX:
         self.evaluations = EvaluationsRunner(_eval_client)
 
         # Monitor: create/reuse patterns (client.monitor.patterns) that a trace can be checked
-        # against at send time via tracer.trace(..., monitor=True, pattern_ids=[...]).
+        # against at send time via tracer.trace(..., monitor=True, pattern_ids=[...]), then read
+        # back the resulting alerts/findings with client.monitor.signals.
         self.monitor = MonitorClient(
             api_key=self.api_key,
             sdk_version=VERSION,

{agentx_python-0.6.7 → agentx_python-0.6.9}/agentx/evaluations/datasets.py

@@ -27,6 +27,8 @@ class DatasetBuilder:
         acceptance_criteria: Optional[str] = None,
         rejection_criteria: Optional[str] = None,
         evaluation_criteria: Optional[str] = None,
+        judge_prompt: Optional[str] = None,
+        judge_model: Optional[str] = None,
         vector_similarity: bool = False,
         jaccard_similarity: bool = False,
         bleu_score: bool = False,
@@ -44,6 +46,13 @@ class DatasetBuilder:
             "evaluationCriteria": evaluation_criteria,
             "questions": [],
         }
+        # LLM-as-judge overrides for this dataset's own grading config. Omit either to keep the
+        # server default (raw prompt template / OpenAI gpt-5.5, see EVALUATIONS.md). judge_model
+        # must be one of client.evaluations.list_models() (OpenAI or Anthropic).
+        if judge_prompt is not None:
+            self._payload["judgePrompt"] = judge_prompt
+        if judge_model is not None:
+            self._payload["judgeModel"] = judge_model
         # Opt-in similarity metrics, surfaced on the report as cosine_similarity /
         # jaccard_similarity / bleu_score / rouge_score (computed against each
         # case's expected_results).
@@ -74,8 +83,20 @@ class DatasetBuilder:
         expected_knowledge_base: Optional[List[str]] = None,
         expected_delegations: Optional[List[str]] = None,
         follow_up_questions: Optional[List[Dict[str, Any]]] = None,
+        judge_guideline: Optional[str] = None,
+        smoke_test_count: Optional[int] = None,
+        smoke_test_guidance: Optional[str] = None,
     ) -> "DatasetBuilder":
-        main = {"query": query}
+        """Add a case. `judge_guideline` is optional grading guidance specific to this question.
+
+        `smoke_test_count`, when set (1-10), asks this question that many extra ways each
+        evaluation run, LLM-paraphrased server-side, to catch agents that are brittle to phrasing
+        rather than genuinely wrong. `smoke_test_guidance` optionally steers what kind of variants
+        get generated (e.g. tone, adversarial phrasing, different languages); the SDK never
+        generates or counts variants itself, both fields are only ever consumed server-side.
+        Ignored on `follow_up_questions`, only the opening question of a case can be smoke-tested.
+        """
+        main: Dict[str, Any] = {"query": query}
         if expected_results:
             main["expectedResults"] = expected_results
         if expected_capabilities:
@@ -84,6 +105,12 @@ class DatasetBuilder:
             main["expectedKnowledgeBase"] = expected_knowledge_base
         if expected_delegations:
             main["expectedDelegations"] = expected_delegations
+        if judge_guideline:
+            main["judgeGuideline"] = judge_guideline
+        if smoke_test_count:
+            main["smokeTest"] = {"enabled": True, "count": smoke_test_count}
+            if smoke_test_guidance:
+                main["smokeTest"]["guidance"] = smoke_test_guidance
         self._payload["questions"].append(
             {
                 "main_question": main,
@@ -217,6 +244,8 @@ class DatasetClient:
         acceptance_criteria: Optional[str] = None,
         rejection_criteria: Optional[str] = None,
         evaluation_criteria: Optional[str] = None,
+        judge_prompt: Optional[str] = None,
+        judge_model: Optional[str] = None,
         vector_similarity: bool = False,
         jaccard_similarity: bool = False,
         bleu_score: bool = False,
@@ -232,6 +261,8 @@ class DatasetClient:
             acceptance_criteria=acceptance_criteria,
             rejection_criteria=rejection_criteria,
             evaluation_criteria=evaluation_criteria,
+            judge_prompt=judge_prompt,
+            judge_model=judge_model,
             vector_similarity=vector_similarity,
             jaccard_similarity=jaccard_similarity,
             bleu_score=bleu_score,

{agentx_python-0.6.7 → agentx_python-0.6.9}/agentx/evaluations/evaluation_settings.py

@@ -24,6 +24,8 @@ class EvaluationSettingsBuilder:
         acceptance_criteria: Optional[str] = None,
         rejection_criteria: Optional[str] = None,
         evaluation_criteria: Optional[str] = None,
+        judge_prompt: Optional[str] = None,
+        judge_model: Optional[str] = None,
         vector_similarity: bool = False,
         jaccard_similarity: bool = False,
         bleu_score: bool = False,
@@ -40,6 +42,13 @@ class EvaluationSettingsBuilder:
             "rejectionCriteria": rejection_criteria,
             "evaluationCriteria": evaluation_criteria,
         }
+        # LLM-as-judge overrides. Omit either to keep the server default (raw prompt template /
+        # OpenAI gpt-5.5, see EVALUATIONS.md). judge_model must be one of
+        # client.evaluations.list_models() (OpenAI or Anthropic).
+        if judge_prompt is not None:
+            self._payload["judgePrompt"] = judge_prompt
+        if judge_model is not None:
+            self._payload["judgeModel"] = judge_model
         # Opt-in similarity metrics, mirrors DatasetBuilder's config kwargs.
         if vector_similarity:
             vs: Dict[str, Any] = {"enabled": True}
@@ -79,6 +88,8 @@ class EvaluationSettingsClient:
         acceptance_criteria: Optional[str] = None,
         rejection_criteria: Optional[str] = None,
         evaluation_criteria: Optional[str] = None,
+        judge_prompt: Optional[str] = None,
+        judge_model: Optional[str] = None,
         vector_similarity: bool = False,
         jaccard_similarity: bool = False,
         bleu_score: bool = False,
@@ -94,6 +105,8 @@ class EvaluationSettingsClient:
             acceptance_criteria=acceptance_criteria,
             rejection_criteria=rejection_criteria,
             evaluation_criteria=evaluation_criteria,
+            judge_prompt=judge_prompt,
+            judge_model=judge_model,
             vector_similarity=vector_similarity,
             jaccard_similarity=jaccard_similarity,
             bleu_score=bleu_score,

{agentx_python-0.6.7 → agentx_python-0.6.9}/agentx/evaluations/models.py

@@ -31,14 +31,30 @@ class ObservableTrace(BaseModel):
 # ---------------------------------------------------------------------------
 
 
+class SmokeTestSettings(BaseModel):
+    """Only meaningful on a question's main_question. See DatasetBuilder.add_case's
+    smoke_test_count/smoke_test_guidance for how to set this."""
+
+    enabled: bool = False
+    count: int = 1
+    guidance: Optional[str] = None
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
 class TestCase(BaseModel):
     query: str
-    expected_results: Optional[str] = None
-    expected_capabilities: Optional[List[str]] = None
-    expected_knowledge_base: Optional[List[str]] = None
-    expected_delegations: Optional[List[str]] = None
+    expected_results: Optional[str] = Field(default=None, alias="expectedResults")
+    expected_capabilities: Optional[List[str]] = Field(default=None, alias="expectedCapabilities")
+    expected_knowledge_base: Optional[List[str]] = Field(default=None, alias="expectedKnowledgeBase")
+    expected_delegations: Optional[List[str]] = Field(default=None, alias="expectedDelegations")
+    judge_guideline: Optional[str] = Field(default=None, alias="judgeGuideline")
+    smoke_test: Optional[SmokeTestSettings] = Field(default=None, alias="smokeTest")
 
     class Config:
+        populate_by_name = True
         extra = "ignore"
 
 
@@ -89,6 +105,10 @@ class EvaluationSettings(BaseModel):
     acceptance_criteria: Optional[str] = Field(default=None, alias="acceptanceCriteria")
     rejection_criteria: Optional[str] = Field(default=None, alias="rejectionCriteria")
     evaluation_criteria: Optional[str] = Field(default=None, alias="evaluationCriteria")
+    # LLM-as-judge overrides. None means "use the server default" (raw prompt template / OpenAI
+    # gpt-5.5). See client.evaluations.settings.builder(judge_prompt=..., judge_model=...).
+    judge_prompt: Optional[str] = Field(default=None, alias="judgePrompt")
+    judge_model: Optional[str] = Field(default=None, alias="judgeModel")
     status: str = "published"
     # Sovereignty & Portability — models selected to compare when this config runs.
     # Hoisted from the nested ``sovereigntyIndex`` object when enabled.
@@ -202,12 +222,30 @@ class LiveStatistics(BaseModel):
         extra = "ignore"
 
 
+class SmokeTestVariantGroup(BaseModel):
+    """Paraphrased variants for one question, generated server-side (reusing the same
+    generation the dashboard's native runs use) and frozen for the lifetime of the run.
+    The SDK never generates or counts these itself, it only consumes what's returned here."""
+
+    question_index: int = Field(alias="questionIndex")
+    variants: List[str] = Field(default_factory=list)
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
 class EvaluationRun(BaseModel):
     run_id: str = Field(alias="runId")
     dataset_id: str = Field(alias="datasetId")
     dataset_version_id: Optional[str] = Field(default=None, alias="datasetVersionId")
     status: str = "in_progress"
     limits: ServerLimits = Field(default_factory=ServerLimits)
+    # Present only when at least one question in the dataset has smokeTest.enabled. See
+    # SmokeTestVariantGroup.
+    smoke_test_variants: Optional[List[SmokeTestVariantGroup]] = Field(
+        default=None, alias="smokeTestVariants"
+    )
 
     class Config:
         populate_by_name = True
@@ -232,6 +270,12 @@ class EvaluationCase(BaseModel):
     # dataset selects comparison models; your callable can read it to pick the
     # model. The SDK also tags the submitted result with it.
     model: Optional[str] = None
+    # Smoke test: True when `query` is a server-generated paraphrase variant rather than the
+    # dataset's original question text (see SmokeTestVariantGroup). Your callable doesn't need
+    # to branch on this, `query` is already the text to ask, but it's available if you want to
+    # log or handle variants differently.
+    is_smoke_test_variant: bool = False
+    smoke_test_variant_text: Optional[str] = None
 
     class Config:
         extra = "ignore"
@@ -278,6 +322,10 @@ class EvaluationResult(BaseModel):
     # the dashboard's "Message Trace Details -> Execution Timeline" render the full execution
     # trace for this case, not just the lightweight observable_trace events above.
     trace_id: Optional[str] = Field(default=None, alias="traceId")
+    # Smoke test: set by execute() from the originating EvaluationCase, not something you need to
+    # set yourself when returning a plain str/dict from your callable.
+    is_smoke_test_variant: Optional[bool] = Field(default=None, alias="isSmokeTestVariant")
+    smoke_test_variant_text: Optional[str] = Field(default=None, alias="smokeTestVariantText")
 
     class Config:
         populate_by_name = True

{agentx_python-0.6.7 → agentx_python-0.6.9}/agentx/evaluations/results.py

@@ -30,6 +30,8 @@ def normalize_result(
         raw.case_id = case.case_id
         raw.question_index = case.question_index
         raw.run_number = case.run_number
+        raw.is_smoke_test_variant = case.is_smoke_test_variant
+        raw.smoke_test_variant_text = case.smoke_test_variant_text
         return raw
 
     output: Optional[dict] = None
@@ -106,6 +108,8 @@ def normalize_result(
         ),
         metadata=metadata,
         traceId=trace_id,
+        isSmokeTestVariant=case.is_smoke_test_variant,
+        smokeTestVariantText=case.smoke_test_variant_text,
     )
 
 
@@ -126,4 +130,6 @@ def normalize_error(
             retryable=False,
         ),
         timings=ResultTimings(latencyMs=latency_ms) if latency_ms is not None else None,
+        isSmokeTestVariant=case.is_smoke_test_variant,
+        smokeTestVariantText=case.smoke_test_variant_text,
     )

{agentx_python-0.6.7 → agentx_python-0.6.9}/agentx/evaluations/runner.py

@@ -93,7 +93,7 @@ class EvaluationRunContext:
     def execute(self, adapter: AdapterLike) -> "EvaluationRunContext":
         """Run all cases locally and submit batches to AgentX."""
         normalized = _wrap_adapter(adapter)
-        cases = _build_cases(self._dataset, self._evaluation_settings)
+        cases = _build_cases(self._dataset, self._run, self._evaluation_settings)
         max_batch = self._run.limits.max_batch_size
 
         # Banner
@@ -108,6 +108,7 @@ class EvaluationRunContext:
             if self._evaluation_settings
             else self._dataset.number_of_requests
         )
+        n_smoke = sum(1 for c in cases if c.is_smoke_test_variant)
 
         print(cyan(sep))
         print(f"  {bold('AgentX Evaluation')}  {dim('—')}  {name}")
@@ -116,9 +117,11 @@ class EvaluationRunContext:
         if display:
             print(f"  {dim('Agent :')} {display}  {dim(f'({framework} / {runtime})')}")
         print()
-        print(
-            f"{bold('Executing')}  {n_q} question{'s' if n_q != 1 else ''} × {n_r} run{'s' if n_r != 1 else ''}"
-        )
+        exec_line = f"{bold('Executing')}  {n_q} question{'s' if n_q != 1 else ''} × {n_r} run{'s' if n_r != 1 else ''}"
+        if n_smoke:
+            variant_word = "variant" if n_smoke == 1 else "variants"
+            exec_line += f"  {dim(f'(+{n_smoke} smoke-test {variant_word})')}"
+        print(exec_line)
 
         # Resume: skip already-submitted keys
         already_done = self._fetch_submitted_keys()
@@ -400,7 +403,9 @@ def _wrap_adapter(adapter: AdapterLike) -> Callable[[EvaluationCase], Evaluation
 
 
 def _build_cases(
-    dataset: Dataset, evaluation_settings: Optional[EvaluationSettings] = None
+    dataset: Dataset,
+    run: EvaluationRun,
+    evaluation_settings: Optional[EvaluationSettings] = None,
 ) -> List[EvaluationCase]:
     cases: List[EvaluationCase] = []
     # When an independent evaluation_settings was chosen (evaluation_settings_id
@@ -419,6 +424,14 @@ def _build_cases(
         evaluation_settings.sovereignty_models if evaluation_settings else dataset.sovereignty_models
     )
     models: List[Optional[str]] = list(sovereignty_models) or [None]
+    # Smoke test: variant text is generated and counted entirely server-side (POST /runs, reusing
+    # the same generation the dashboard's native runs use) and handed back on `run`. The SDK never
+    # re-derives eligibility, count, or text itself, it only turns what the server already decided
+    # into extra cases. Not multiplied across sovereignty_models, matching the server's counting in
+    # finalize/missing-results (one case per variant per question, regardless of comparison models).
+    smoke_variants_by_question = {
+        group.question_index: group.variants for group in (run.smoke_test_variants or [])
+    }
     for q_idx, question in enumerate(dataset.questions):
         mq = question.main_question
         for run_num in range(1, n_runs + 1):
@@ -437,6 +450,21 @@ def _build_cases(
                         model=model,
                     )
                 )
+        for variant_idx, variant_text in enumerate(smoke_variants_by_question.get(q_idx, [])):
+            cases.append(
+                EvaluationCase(
+                    case_id=f"case-{q_idx}",
+                    question_index=q_idx,
+                    run_number=n_runs + variant_idx + 1,
+                    query=variant_text,
+                    expected_results=mq.expected_results,
+                    expected_capabilities=mq.expected_capabilities,
+                    expected_knowledge_base=mq.expected_knowledge_base,
+                    expected_delegations=mq.expected_delegations,
+                    is_smoke_test_variant=True,
+                    smoke_test_variant_text=variant_text,
+                )
+            )
     return cases
 
 

{agentx_python-0.6.7 → agentx_python-0.6.9}/agentx/integrations/langchain.py

@@ -20,6 +20,7 @@ Requires: ``pip install "agentx-python[langchain]"``
 """
 from __future__ import annotations
 
+import threading
 import time
 from typing import Any, Dict, List, Optional, Union
 from uuid import UUID
@@ -262,6 +263,13 @@ class AgentXCallbackHandler(BaseCallbackHandler):
         self._pending_retrieval_steps: List[Dict[str, Any]] = []
         # run_id → {"start": float, "query": str}
         self._retrieval_starts: Dict[UUID, Dict[str, Any]] = {}
+        # Guards appends to a top-level run's shared aggregate lists (tool_calls,
+        # perf_tool_calls, execution_steps, retrieval_steps). LangGraph's ToolNode
+        # runs multiple tool calls from one AIMessage concurrently via a thread
+        # pool (see langgraph.prebuilt.tool_node.ToolNode._func), so on_tool_end /
+        # on_tool_error can fire from several threads at once for the same
+        # top-level run.
+        self._state_lock = threading.Lock()
 
     # ------------------------------------------------------------------
     # Chain lifecycle
@@ -311,14 +319,14 @@ class AgentXCallbackHandler(BaseCallbackHandler):
             return
         latency_ms = int((time.time() - state["start"]) * 1000)
         output = _extract_output(outputs)
+        # Each tool_call dict already carries its own start_time/end_time (set in
+        # on_tool_end/on_tool_error), so no re-pairing against perf_tool_calls by
+        # index is needed here. That used to be done via zip(), which silently
+        # mispaired timestamps when LangGraph's ToolNode ran several tool calls
+        # from one AIMessage concurrently (see _state_lock's docstring): two
+        # lists appended to from different threads don't necessarily end up in
+        # the same relative order.
         tool_calls = state["tool_calls"] or _extract_tool_calls_from_messages(outputs)
-        if state["tool_calls"] and len(state["tool_calls"]) == len(state["perf_tool_calls"]):
-            # Enrich with the timestamps perf_tool_calls tracked in lockstep,
-            # so tool calls interleave correctly in a merged span's timeline.
-            tool_calls = [
-                {**tc, "start_time": perf.get("start_time"), "end_time": perf.get("end_time")}
-                for tc, perf in zip(state["tool_calls"], state["perf_tool_calls"])
-            ]
 
         active_span = self._tracer.current_span
         if active_span is not None:
@@ -332,6 +340,7 @@ class AgentXCallbackHandler(BaseCallbackHandler):
                 input=state["input"],
                 output=output,
                 model=state.get("model"),
+                framework="langchain",
                 input_tokens=state["input_tokens"] or None,
                 output_tokens=state["output_tokens"] or None,
             )
@@ -383,6 +392,7 @@ class AgentXCallbackHandler(BaseCallbackHandler):
                 retrieval_steps=state["retrieval_steps"],
                 input=state["input"],
                 model=state.get("model"),
+                framework="langchain",
                 input_tokens=state["input_tokens"] or None,
                 output_tokens=state["output_tokens"] or None,
             )
@@ -485,41 +495,42 @@ class AgentXCallbackHandler(BaseCallbackHandler):
             # Extract token usage from LLMResult for this call
             call_input_tokens: Optional[int] = None
             call_output_tokens: Optional[int] = None
-            if top and top in self._runs:
-                usage = {}
-                if hasattr(response, "llm_output") and isinstance(response.llm_output, dict):
-                    usage = response.llm_output.get("token_usage") or response.llm_output.get("usage") or {}
-                # Also check generations for token counts (some providers put it there)
-                if not usage and hasattr(response, "generations"):
-                    for gen_list in (response.generations or []):
-                        for gen in (gen_list or []):
-                            gen_info = getattr(gen, "generation_info", None) or {}
-                            if gen_info.get("prompt_tokens") or gen_info.get("completion_tokens"):
-                                usage = gen_info
-                                break
-                if usage:
-                    call_input_tokens = int(
-                        usage.get("prompt_tokens") or usage.get("input_tokens") or usage.get("prompt_token_count") or 0
-                    )
-                    call_output_tokens = int(
-                        usage.get("completion_tokens") or usage.get("output_tokens") or usage.get("candidates_token_count") or 0
-                    )
-                    self._runs[top]["input_tokens"] += call_input_tokens
-                    self._runs[top]["output_tokens"] += call_output_tokens
-
-            if start_t is not None and top and top in self._runs:
-                steps = self._runs[top]["execution_steps"]
-                steps.append({
-                    "name": f"LLM Call {len(steps) + 1}",
-                    "duration_ms": (end_t - start_t) * 1000,
-                    "start_time": start_t,
-                    "end_time": end_t,
-                    "model": llm_state.get("model"),
-                    "input": llm_state.get("input"),
-                    "output": _extract_llm_output(response),
-                    "inputTokenSize": call_input_tokens,
-                    "outputTokenSize": call_output_tokens,
-                })
+            with self._state_lock:
+                if top and top in self._runs:
+                    usage = {}
+                    if hasattr(response, "llm_output") and isinstance(response.llm_output, dict):
+                        usage = response.llm_output.get("token_usage") or response.llm_output.get("usage") or {}
+                    # Also check generations for token counts (some providers put it there)
+                    if not usage and hasattr(response, "generations"):
+                        for gen_list in (response.generations or []):
+                            for gen in (gen_list or []):
+                                gen_info = getattr(gen, "generation_info", None) or {}
+                                if gen_info.get("prompt_tokens") or gen_info.get("completion_tokens"):
+                                    usage = gen_info
+                                    break
+                    if usage:
+                        call_input_tokens = int(
+                            usage.get("prompt_tokens") or usage.get("input_tokens") or usage.get("prompt_token_count") or 0
+                        )
+                        call_output_tokens = int(
+                            usage.get("completion_tokens") or usage.get("output_tokens") or usage.get("candidates_token_count") or 0
+                        )
+                        self._runs[top]["input_tokens"] += call_input_tokens
+                        self._runs[top]["output_tokens"] += call_output_tokens
+
+                if start_t is not None and top and top in self._runs:
+                    steps = self._runs[top]["execution_steps"]
+                    steps.append({
+                        "name": f"LLM Call {len(steps) + 1}",
+                        "duration_ms": (end_t - start_t) * 1000,
+                        "start_time": start_t,
+                        "end_time": end_t,
+                        "model": llm_state.get("model"),
+                        "input": llm_state.get("input"),
+                        "output": _extract_llm_output(response),
+                        "inputTokenSize": call_input_tokens,
+                        "outputTokenSize": call_output_tokens,
+                    })
 
     # ------------------------------------------------------------------
     # Tool lifecycle
@@ -561,18 +572,25 @@ class AgentXCallbackHandler(BaseCallbackHandler):
             "input": state["tool_input"],
             "output": str(output),
             "latency_ms": latency_ms,
+            "success": True,
+            # Set directly on the tool_call dict (not just perf_tool_calls below) so
+            # on_chain_end's merged-span path doesn't need to re-pair the two lists by
+            # index later. See _state_lock's docstring for why that used to be unsafe.
+            "start_time": start_t,
+            "end_time": end_t,
         }
         top = self._find_top_ancestor(parent_run_id)
-        if top and top in self._runs:
-            self._runs[top]["tool_calls"].append(tool_call)
-            self._runs[top]["perf_tool_calls"].append({
-                "name": state["tool_name"],
-                "duration_ms": (end_t - start_t) * 1000,
-                "start_time": start_t,
-                "end_time": end_t,
-                "input": state["tool_input"],
-                "output": tool_call["output"],
-            })
+        with self._state_lock:
+            if top and top in self._runs:
+                self._runs[top]["tool_calls"].append(tool_call)
+                self._runs[top]["perf_tool_calls"].append({
+                    "name": state["tool_name"],
+                    "duration_ms": (end_t - start_t) * 1000,
+                    "start_time": start_t,
+                    "end_time": end_t,
+                    "input": state["tool_input"],
+                    "output": tool_call["output"],
+                })
 
     def on_tool_error(
         self,
@@ -593,18 +611,22 @@ class AgentXCallbackHandler(BaseCallbackHandler):
             "input": state.get("tool_input"),
             "output": f"ERROR: {error}",
             "latency_ms": int((end_t - start_t) * 1000),
+            "success": False,
+            "start_time": start_t,
+            "end_time": end_t,
         }
         top = self._find_top_ancestor(parent_run_id)
-        if top and top in self._runs:
-            self._runs[top]["tool_calls"].append(tool_call)
-            self._runs[top]["perf_tool_calls"].append({
-                "name": state.get("tool_name", "unknown"),
-                "duration_ms": (end_t - start_t) * 1000,
-                "start_time": start_t,
-                "end_time": end_t,
-                "input": tool_call["input"],
-                "output": tool_call["output"],
-            })
+        with self._state_lock:
+            if top and top in self._runs:
+                self._runs[top]["tool_calls"].append(tool_call)
+                self._runs[top]["perf_tool_calls"].append({
+                    "name": state.get("tool_name", "unknown"),
+                    "duration_ms": (end_t - start_t) * 1000,
+                    "start_time": start_t,
+                    "end_time": end_t,
+                    "input": tool_call["input"],
+                    "output": tool_call["output"],
+                })
 
     # ------------------------------------------------------------------
     # Retriever lifecycle

{agentx_python-0.6.7 → agentx_python-0.6.9}/agentx/monitor/__init__.py

@@ -1,10 +1,14 @@
 from agentx.monitor.client import MonitorClient
-from agentx.monitor.models import MonitorPattern
+from agentx.monitor.models import MonitorPattern, MonitorSignal, SignalOccurrence
 from agentx.monitor.patterns import MonitorPatternBuilder, MonitorPatternClient
+from agentx.monitor.signals import MonitorSignalClient
 
 __all__ = [
     "MonitorClient",
     "MonitorPattern",
     "MonitorPatternBuilder",
     "MonitorPatternClient",
+    "MonitorSignal",
+    "SignalOccurrence",
+    "MonitorSignalClient",
 ]

{agentx_python-0.6.7 → agentx_python-0.6.9}/agentx/monitor/client.py

@@ -7,7 +7,7 @@ from typing import Any, List, Optional
 
 import requests
 
-from agentx.monitor.models import MonitorPattern
+from agentx.monitor.models import MonitorPattern, MonitorSignal
 
 logger = logging.getLogger(__name__)
 
@@ -69,8 +69,10 @@ class MonitorClient:
         )
 
         from agentx.monitor.patterns import MonitorPatternClient
+        from agentx.monitor.signals import MonitorSignalClient
 
         self.patterns = MonitorPatternClient(self)
+        self.signals = MonitorSignalClient(self)
 
     # ------------------------------------------------------------------
     # Low-level HTTP
@@ -132,3 +134,35 @@ class MonitorClient:
             "GET", f"/patterns/{pattern_id}", params=self._workspace_params()
         )
         return MonitorPattern(**data["pattern"])
+
+    # ------------------------------------------------------------------
+    # Signal endpoints
+    # ------------------------------------------------------------------
+
+    def list_signals(
+        self,
+        polarity: Optional[str] = None,
+        status: Optional[str] = None,
+        severity: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        limit: int = 50,
+    ) -> List[MonitorSignal]:
+        params = {**(self._workspace_params() or {})}
+        if polarity is not None:
+            params["polarity"] = polarity
+        if status is not None:
+            params["status"] = status
+        if severity is not None:
+            params["severity"] = severity
+        if agent_id is not None:
+            params["agentId"] = agent_id
+        if limit is not None:
+            params["limit"] = limit
+        data = self._request("GET", "/signals", params=params)
+        return [MonitorSignal(**s) for s in data.get("signals", [])]
+
+    def get_signal(self, signal_id: str) -> MonitorSignal:
+        data = self._request(
+            "GET", f"/signals/{signal_id}", params=self._workspace_params()
+        )
+        return MonitorSignal(**data["signal"])

agentx_python-0.6.9/agentx/monitor/models.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+from pydantic import BaseModel, Field
+
+
+class MonitorPattern(BaseModel):
+    """A detection rule checked against production traces. Built via
+    ``client.monitor.patterns.builder(...).publish()`` and referenced by id as a
+    ``pattern_ids`` entry in ``tracer.trace(..., monitor=True, pattern_ids=[...])``.
+
+    A "failure" pattern (the default) raises a signal to triage; a "proper" pattern logs a
+    healthy tally instead. Only one of ``include_terms``/``regex``/``semantic_prompt`` is
+    meaningful at a time, selected by ``detector_kind``.
+    """
+
+    id: str = Field(alias="_id")
+    key: str
+    name: str
+    description: Optional[str] = None
+    category: Optional[str] = None
+    detector_kind: str = Field(default="contains", alias="detectorKind")
+    match_target: List[str] = Field(default_factory=lambda: ["response"], alias="matchTarget")
+    match_mode: str = Field(default="any", alias="matchMode")
+    include_terms: List[str] = Field(default_factory=list, alias="includeTerms")
+    exclude_terms: List[str] = Field(default_factory=list, alias="excludeTerms")
+    regex: Optional[str] = None
+    semantic_prompt: Optional[str] = Field(default=None, alias="semanticPrompt")
+    severity: str = "medium"
+    polarity: str = "failure"
+    enabled: bool = True
+    sample_rate: float = Field(default=1.0, alias="sampleRate")
+    scope_mode: str = Field(default="all", alias="scopeMode")
+    agent_ids: List[str] = Field(default_factory=list, alias="agentIds")
+
+    class Config:
+        extra = "ignore"
+
+
+class SignalOccurrence(BaseModel):
+    """One hit behind a signal, capped at the server's most recent N per signal."""
+
+    agent_id: Optional[Dict[str, Any]] = Field(default=None, alias="agentId")
+    conversation_id: Optional[str] = Field(default=None, alias="conversationId")
+    message_id: Optional[str] = Field(default=None, alias="messageId")
+    trace_id: Optional[str] = Field(default=None, alias="traceId")
+    seen_at: Optional[str] = Field(default=None, alias="seenAt")
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class MonitorSignal(BaseModel):
+    """An alert/finding produced when a trace matched a pattern (or, for a "proper" pattern,
+    a healthy tally). Read-only from the SDK, see ``client.monitor.signals.list()/get()``, a
+    signal is the system's output from checking traces against patterns, not something an SDK
+    caller constructs."""
+
+    id: str = Field(alias="_id")
+    workspace_id: Optional[str] = Field(default=None, alias="workspaceId")
+    # Populated to {"_id", "name", "avatar"} when set, since the server populates this field.
+    agent_id: Optional[Dict[str, Any]] = Field(default=None, alias="agentId")
+    conversation_id: Optional[str] = Field(default=None, alias="conversationId")
+    message_id: Optional[str] = Field(default=None, alias="messageId")
+    type: str
+    severity: str = "medium"
+    polarity: str = "failure"
+    status: str = "open"
+    score: Optional[float] = None
+    threshold: Optional[float] = None
+    summary: str
+    pattern_key: str = Field(alias="patternKey")
+    evidence: Optional[Dict[str, Any]] = None
+    root_cause: Optional[str] = Field(default=None, alias="rootCause")
+    recommended_actions: List[str] = Field(default_factory=list, alias="recommendedActions")
+    review_status: Optional[str] = Field(default=None, alias="reviewStatus")
+    first_seen_at: Optional[str] = Field(default=None, alias="firstSeenAt")
+    last_seen_at: Optional[str] = Field(default=None, alias="lastSeenAt")
+    occurrence_count: int = Field(default=1, alias="occurrenceCount")
+    occurrences: List[SignalOccurrence] = Field(default_factory=list)
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"

agentx_python-0.6.9/agentx/monitor/signals.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import logging
+from typing import List, Optional, TYPE_CHECKING
+
+from agentx.monitor.models import MonitorSignal
+
+if TYPE_CHECKING:
+    from agentx.monitor.client import MonitorClient
+
+logger = logging.getLogger(__name__)
+
+
+class MonitorSignalClient:
+    """Thin wrapper surfaced as ``client.monitor.signals``. Read-only: a signal is produced by
+    checking a trace against patterns (see ``tracer.trace(..., monitor=True, pattern_ids=[...])``),
+    not something the SDK creates directly."""
+
+    def __init__(self, client: "MonitorClient"):
+        self._client = client
+
+    def list(
+        self,
+        polarity: Optional[str] = None,
+        status: Optional[str] = None,
+        severity: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        limit: int = 50,
+    ) -> List[MonitorSignal]:
+        """List this workspace's signals, most recently seen first.
+
+        `polarity` defaults server-side to failures only ("proper", the healthy tally, is
+        excluded); pass ``polarity="all"`` to include both, or ``"proper"``/``"failure"`` to
+        narrow to one kind. `limit` is capped at 100 server-side.
+        """
+        return self._client.list_signals(
+            polarity=polarity, status=status, severity=severity, agent_id=agent_id, limit=limit
+        )
+
+    def get(self, signal_id: str) -> MonitorSignal:
+        return self._client.get_signal(signal_id)

{agentx_python-0.6.7 → agentx_python-0.6.9}/agentx/tracing/tracer.py

@@ -114,6 +114,9 @@ class _TraceSpan:
         self._execution_steps: list = []
         self._retrieval_steps: list = []
         self._captured_model: Optional[str] = None
+        # Adopted from a merged child run (e.g. AgentXCallbackHandler) when this span itself
+        # wasn't opened with an explicit framework= — see _merge_child_run below.
+        self._captured_framework: Optional[str] = None
         self._input_tokens: int = 0
         self._output_tokens: int = 0
         # Guards _merge_child_run — with Tracer.use_span(), multiple threads
@@ -186,7 +189,7 @@ class _TraceSpan:
             latency_ms=latency_ms,
             error=self._error,
             metadata=self._metadata,
-            framework=self._framework,
+            framework=self._framework or self._captured_framework,
             model=self._model or self._captured_model,
             tool_calls=self.tool_calls or None,
             session_id=self._session_id,
@@ -250,6 +253,7 @@ class _TraceSpan:
         input: Any = None,
         output: Any = None,
         model: Optional[str] = None,
+        framework: Optional[str] = None,
         input_tokens: Optional[int] = None,
         output_tokens: Optional[int] = None,
     ) -> None:
@@ -281,6 +285,8 @@ class _TraceSpan:
                 self.output = output
             if model and not self._captured_model:
                 self._captured_model = model
+            if framework and not self._captured_framework:
+                self._captured_framework = framework
             if input_tokens:
                 self._input_tokens += input_tokens
             if output_tokens:

agentx_python-0.6.9/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.6.9"

{agentx_python-0.6.7 → agentx_python-0.6.9}/agentx_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.7
+Version: 0.6.9
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team
@@ -85,7 +85,7 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
 - **MCP support** — connect any Model Context Protocol server.
 - **Multi-agent orchestration** — workforces of agents with a designated manager, across LLM vendors.
 - **Production tracing** — one decorator or context manager records every agent run (input, output, latency, tool calls, token usage) into your workspace, for any framework.
-- **Agent Evaluations** — score any agent (LangChain, CrewAI, OpenAI, Anthropic, HTTP, …) with LLM-as-a-judge ratings plus optional cosine and Jaccard similarity metrics.
+- **Agent Evaluations** — score any agent (LangChain, CrewAI, OpenAI, Anthropic, HTTP, …) with LLM-as-a-judge ratings plus optional cosine and Jaccard similarity metrics. Configurable judge prompt/model (OpenAI or Anthropic), per-question judge guidelines, and smoke testing for phrasing robustness.
 - **A2A** — Each agent can be published with agent-to-agent protocol compatible.
 
 ---
@@ -266,6 +266,13 @@ with client.tracer.trace("support-agent", monitor=True, pattern_ids=[pattern.id]
 
 This works independently of the dashboard's per-agent monitoring toggle (Governance > Observe > Agents), which still auto-checks every trace from an agent once enabled there, with no code changes needed either way.
 
+Read the resulting alerts back with `client.monitor.signals.list()`/`.get()`, no dashboard required:
+
+```python
+for signal in client.monitor.signals.list(severity="high"):
+    print(signal.summary, signal.occurrence_count)
+```
+
 See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
 
 ---
@@ -293,7 +300,9 @@ print(report.recommendations)      # list of prioritized, actionable fixes
 
 `.analyze()` also generates a full qualitative report (strengths, weaknesses, instruction adherence, reasoning quality, and recommendations), running the same durable, multi-judge pipeline as the dashboard's "Analyze" button. `analyze(mode=..., quality_mode=..., judges=[...])` controls how items are scored and by which models. See [AI analysis report](EVALUATIONS.md#ai-analysis-report) in the full guide for the complete field and parameter reference.
 
-See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, and the complete API reference.
+Ask a case's question several extra ways each run, LLM-paraphrased server-side, to catch agents that break on phrasing rather than substance, and override the judge's prompt/model per config, see [Smoke testing](EVALUATIONS.md#smoke-testing-phrasing-robustness) and [Configuring the judge](EVALUATIONS.md#configuring-the-judge) in the full guide.
+
+See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, smoke testing, judge configuration, and the complete API reference.
 
 ---
 

{agentx_python-0.6.7 → agentx_python-0.6.9}/agentx_python.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ agentx/monitor/__init__.py
 agentx/monitor/client.py
 agentx/monitor/models.py
 agentx/monitor/patterns.py
+agentx/monitor/signals.py
 agentx/resources/__init__.py
 agentx/resources/agent.py
 agentx/resources/conversation.py

agentx_python-0.6.7/agentx/monitor/models.py

@@ -1,37 +0,0 @@
-from __future__ import annotations
-
-from typing import List, Optional
-from pydantic import BaseModel, Field
-
-
-class MonitorPattern(BaseModel):
-    """A detection rule checked against production traces. Built via
-    ``client.monitor.patterns.builder(...).publish()`` and referenced by id as a
-    ``pattern_ids`` entry in ``tracer.trace(..., monitor=True, pattern_ids=[...])``.
-
-    A "failure" pattern (the default) raises a signal to triage; a "proper" pattern logs a
-    healthy tally instead. Only one of ``include_terms``/``regex``/``semantic_prompt`` is
-    meaningful at a time, selected by ``detector_kind``.
-    """
-
-    id: str = Field(alias="_id")
-    key: str
-    name: str
-    description: Optional[str] = None
-    category: Optional[str] = None
-    detector_kind: str = Field(default="contains", alias="detectorKind")
-    match_target: List[str] = Field(default_factory=lambda: ["response"], alias="matchTarget")
-    match_mode: str = Field(default="any", alias="matchMode")
-    include_terms: List[str] = Field(default_factory=list, alias="includeTerms")
-    exclude_terms: List[str] = Field(default_factory=list, alias="excludeTerms")
-    regex: Optional[str] = None
-    semantic_prompt: Optional[str] = Field(default=None, alias="semanticPrompt")
-    severity: str = "medium"
-    polarity: str = "failure"
-    enabled: bool = True
-    sample_rate: float = Field(default=1.0, alias="sampleRate")
-    scope_mode: str = Field(default="all", alias="scopeMode")
-    agent_ids: List[str] = Field(default_factory=list, alias="agentIds")
-
-    class Config:
-        extra = "ignore"

agentx_python-0.6.7/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.6.7"