agentx-python 0.6.31__tar.gz → 0.6.33__tar.gz

{agentx_python-0.6.31 → agentx_python-0.6.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.31
+Version: 0.6.33
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.31 → agentx_python-0.6.33}/agentx/agentx.py

@@ -60,6 +60,16 @@ class AgentX:
         # ground truth behind the dashboard's Judge Calibration card. Self-host only.
         self.outcomes = OutcomesClient(api_key=self.api_key)
 
+        from agentx.projects import ProjectsClient
+
+        # Project CRUD (self-host): isolated tenants with their own API keys (P1.1).
+        self.projects = ProjectsClient(api_key=self.api_key)
+
+        from agentx.traces import TracesClient
+
+        # The read side of tracing: trace-by-id detail and paginated listing (P1.2).
+        self.traces = TracesClient(api_key=self.api_key)
+
         from agentx.feedback import FeedbackClient
 
         # Forward end-user votes ("up"/"down") on traced responses - a "down" raises a signal

{agentx_python-0.6.31 → agentx_python-0.6.33}/agentx/evaluations/_term.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import itertools
 import sys
+import os
 import threading
 import time
 
@@ -61,8 +62,12 @@ class Spinner:
         self._message = message
         self._stop = threading.Event()
         self._thread: threading.Thread | None = None
+        # AGENTX_EVAL_QUIET=1: no spinner thread at all - CI logs stay clean.
+        self._quiet = os.getenv("AGENTX_EVAL_QUIET", "").lower() in ("1", "true", "yes")
 
     def __enter__(self) -> "Spinner":
+        if self._quiet:
+            return self
         if not _IS_TTY:
             print(f"  {self._message}...", flush=True)
             return self
@@ -78,7 +83,7 @@ class Spinner:
             print(f"  {message}...", flush=True)
 
     def __exit__(self, *_) -> None:
-        if not _IS_TTY:
+        if self._quiet or not _IS_TTY:
             return
         self._stop.set()
         if self._thread:

{agentx_python-0.6.31 → agentx_python-0.6.33}/agentx/evaluations/datasets.py

@@ -35,6 +35,7 @@ class DatasetBuilder:
         rouge_score: bool = False,
         similarity_model: Optional[str] = None,
         sovereignty_models: Optional[List[str]] = None,
+        code_scorers: Optional[List[Dict[str, Any]]] = None,
     ):
         self._client = client
         self._payload: Dict[str, Any] = {
@@ -63,6 +64,21 @@ class DatasetBuilder:
             self._payload["vectorSimilarity"] = vs
         if jaccard_similarity:
             self._payload["jaccardSimilarity"] = {"enabled": True}
+        # Offline code scorers, versioned in the repo next to the dataset they guard (P1.4):
+        # each entry is {"name", "code"} (a JS function body invoked as
+        # score({input, output, expected, toolCalls})), optional "enabled" (default True).
+        if code_scorers:
+            import uuid as _uuid
+
+            self._payload["codeScorers"] = [
+                {
+                    "id": scorer.get("id") or _uuid.uuid4().hex[:12],
+                    "name": scorer["name"],
+                    "code": scorer["code"],
+                    "enabled": scorer.get("enabled", True),
+                }
+                for scorer in code_scorers
+            ]
         if bleu_score:
             self._payload["bleuScore"] = {"enabled": True}
         if rouge_score:

{agentx_python-0.6.31 → agentx_python-0.6.33}/agentx/evaluations/models.py

@@ -399,6 +399,65 @@ class BatchAppendResponse(BaseModel):
         extra = "ignore"
 
 
+# ---------------------------------------------------------------------------
+# Typed run-result rows (P1.5)
+# ---------------------------------------------------------------------------
+
+
+class RunResultRow(BaseModel):
+    """One row of a finished run's results - typed and snake_case, replacing the raw wire dicts
+    ``run.results()`` used to return. ``.raw`` keeps the full wire dict for anything not yet
+    modeled. Dict-style access (``row["rating"]``, ``row.get(...)``) still works for one
+    deprecation cycle and warns; prefer the attributes."""
+
+    rating: Optional[float] = None
+    justification: Optional[str] = None
+    question_text: Optional[str] = Field(default=None, alias="questionText")
+    response: Optional[str] = None
+    trace_id: Optional[str] = Field(default=None, alias="traceId")
+    latency_ms: Optional[float] = Field(default=None, alias="latencyMs")
+    input_tokens: Optional[int] = Field(default=None, alias="inputTokens")
+    output_tokens: Optional[int] = Field(default=None, alias="outputTokens")
+    cosine_similarity: Optional[float] = Field(default=None, alias="cosineSimilarity")
+    jaccard_similarity: Optional[float] = Field(default=None, alias="jaccardSimilarity")
+    bleu_score: Optional[float] = Field(default=None, alias="bleuScore")
+    rouge_score: Optional[float] = Field(default=None, alias="rougeScore")
+    code_scorer_results: Optional[List[Dict[str, Any]]] = Field(default=None, alias="codeScorerResults")
+    raw: Dict[str, Any] = Field(default_factory=dict)
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+    @classmethod
+    def from_wire(cls, wire: Dict[str, Any]) -> "RunResultRow":
+        row = cls.model_validate(wire)
+        row.raw = wire
+        return row
+
+    def __getitem__(self, key: str) -> Any:
+        import warnings
+
+        warnings.warn(
+            "Dict-style access on run results is deprecated - use typed attributes "
+            '(row.rating, row.jaccard_similarity) or row.raw["..."] for unmodeled fields.',
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self.raw[key]
+
+    def get(self, key: str, default: Any = None) -> Any:
+        import warnings
+
+        warnings.warn(
+            "Dict-style access on run results is deprecated - use typed attributes "
+            'or row.raw.get("...") for unmodeled fields.',
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self.raw.get(key, default)
+
+
 # ---------------------------------------------------------------------------
 # Analysis / report
 # ---------------------------------------------------------------------------

{agentx_python-0.6.31 → agentx_python-0.6.33}/agentx/evaluations/results.py

@@ -10,7 +10,6 @@ from agentx.evaluations.models import (
     ResultTimings,
 )
 from agentx.evaluations.tracing import build_trace
-from agentx.evaluations.redaction import redact_dict
 
 
 def _to_int(value: Any) -> Optional[int]:
@@ -59,7 +58,7 @@ def normalize_result(
         retrieval_context = raw.get("retrieval_context") or raw.get("retrievalContext")
         meta_raw = raw.get("metadata")
         if isinstance(meta_raw, dict):
-            metadata = redact_dict(meta_raw)
+            metadata = meta_raw
 
         # Extract token counts - top-level keys take priority, fall back to metadata
         input_tokens = _to_int(raw.get("input_tokens"))

{agentx_python-0.6.31 → agentx_python-0.6.33}/agentx/evaluations/runner.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import os
 import time
 import uuid
 from typing import Any, Callable, Dict, List, Optional, Set, Union
@@ -21,7 +22,6 @@ from agentx.evaluations.models import (
     ModelInfo,
     Report,
 )
-from agentx.evaluations.redaction import redact_dict
 from agentx.evaluations.reporting import print_report
 from agentx.evaluations.results import normalize_result, normalize_error
 from agentx.evaluations._term import (
@@ -38,6 +38,17 @@ from agentx.evaluations._term import (
 
 logger = logging.getLogger(__name__)
 
+
+def _eval_quiet() -> bool:
+    """AGENTX_EVAL_QUIET=1 silences the interactive progress UI (spinners, per-case lines) for
+    CI logs - results, gate verdicts, and errors still print. Read per call so tests can toggle."""
+    return os.getenv("AGENTX_EVAL_QUIET", "").lower() in ("1", "true", "yes")
+
+
+def _say(*args, **kwargs) -> None:
+    if not _eval_quiet():
+        print(*args, **kwargs)
+
 AdapterLike = Union[
     Callable[[EvaluationCase], Any],
     RawCallableAdapter,
@@ -127,18 +138,18 @@ class EvaluationRunContext:
         )
         n_smoke = sum(1 for c in cases if c.is_smoke_test_variant)
 
-        print(cyan(sep))
-        print(f"  {bold('AgentX Evaluation')}  {dim(' - ')}  {name}")
-        print(cyan(sep))
-        print(f"  {dim('Run   :')} {dim(self._run.run_id)}")
+        _say(cyan(sep))
+        _say(f"  {bold('AgentX Evaluation')}  {dim(' - ')}  {name}")
+        _say(cyan(sep))
+        _say(f"  {dim('Run   :')} {dim(self._run.run_id)}")
         if display:
-            print(f"  {dim('Agent :')} {display}  {dim(f'({framework} / {runtime})')}")
-        print()
+            _say(f"  {dim('Agent :')} {display}  {dim(f'({framework} / {runtime})')}")
+        _say()
         exec_line = f"{bold('Executing')}  {n_q} question{'s' if n_q != 1 else ''} × {n_r} run{'s' if n_r != 1 else ''}"
         if n_smoke:
             variant_word = "variant" if n_smoke == 1 else "variants"
             exec_line += f"  {dim(f'(+{n_smoke} smoke-test {variant_word})')}"
-        print(exec_line)
+        _say(exec_line)
 
         # Resume: skip already-submitted keys
         already_done = self._fetch_submitted_keys()
@@ -186,7 +197,7 @@ class EvaluationRunContext:
                 resp = self._client.append_results(self._run.run_id, batch_id, batch)
                 if resp.live_statistics is not None:
                     self._live_stats = resp.live_statistics
-                print(
+                _say(
                     f"  {green('✓')}  Scored {resp.accepted} result{'s' if resp.accepted != 1 else ''}"
                 )
                 logger.info(
@@ -197,7 +208,7 @@ class EvaluationRunContext:
                     resp.failed_validation,
                 )
             except Exception as exc:
-                print(f"  {red('✗')}  Scoring failed: {dim(str(exc))}")
+                _say(f"  {red('✗')}  Scoring failed: {dim(str(exc))}")
                 logger.error("Failed to submit batch %s: %s", batch_id[:8], exc)
 
     def _fetch_submitted_keys(self) -> Set[str]:
@@ -214,16 +225,16 @@ class EvaluationRunContext:
     # ------------------------------------------------------------------
 
     def finalize(self) -> "EvaluationRunContext":
-        print()
+        _say()
         with Spinner("Finalizing - submitting results"):
             try:
                 data = self._client.finalize_run(self._run.run_id)
                 if isinstance(data, dict) and data.get("liveStatistics") is not None:
                     self._live_stats = LiveStatistics(**data["liveStatistics"])
-                print(f"  {green('✓')}  Finalized")
+                _say(f"  {green('✓')}  Finalized")
                 logger.info("Run %s finalized", self._run.run_id)
             except Exception as exc:
-                print(f"  {red('✗')}  Finalize failed: {dim(str(exc))}")
+                _say(f"  {red('✗')}  Finalize failed: {dim(str(exc))}")
                 logger.error("Finalize failed: %s", exc)
         return self
 
@@ -256,11 +267,11 @@ class EvaluationRunContext:
             caller=caller,
         )
         result = GateResult(data)
-        print()
+        _say()
         for check in result.checks:
             mark = green("✓") if check.get("passed") else red("✗")
-            print(f"  {mark}  [{check.get('check')}] {check.get('detail')}")
-        print(f"  {green('✓  GATE PASSED') if result.passed else red('✗  GATE FAILED')}")
+            _say(f"  {mark}  [{check.get('check')}] {check.get('detail')}")
+        _say(f"  {green('✓  GATE PASSED') if result.passed else red('✗  GATE FAILED')}")
         return result
 
     # ------------------------------------------------------------------
@@ -274,9 +285,13 @@ class EvaluationRunContext:
     def results(self) -> list:
         """Per-result rows for this run (rating, justification, code scorer rows, trace ids,
         latency/tokens, similarity metrics) - what the dashboard's run detail table shows,
-        fetched fresh from the engine."""
+        fetched fresh from the engine. Returns typed ``RunResultRow`` objects (snake_case
+        attributes; ``.raw`` is the wire dict; dict-style access warns for one cycle - P1.5)."""
+        from agentx.evaluations.models import RunResultRow
+
         detail = self._client.get_run(self.run_id)
-        return detail.get("results", []) if isinstance(detail, dict) else []
+        rows = detail.get("results", []) if isinstance(detail, dict) else []
+        return [RunResultRow.from_wire(r) for r in rows]
 
     @property
     def run_id(self) -> str:
@@ -336,7 +351,7 @@ class EvaluationRunContext:
             raise ValueError("judges must contain 1-3 model ids")
         resolved_judges = judges if judges is not None else [_DEFAULT_JUDGE_MODEL]
 
-        print()
+        _say()
         with Spinner("Analyzing - AI is reviewing your results") as spinner:
             try:
                 self._client.analyze_run(
@@ -356,14 +371,14 @@ class EvaluationRunContext:
                     status = self._client.get_analysis_status(self._run.run_id)
 
                 if not status.is_terminal:
-                    print(f"  {yellow('!')}  Still running after {int(timeout)}s, check the dashboard for status")
+                    _say(f"  {yellow('!')}  Still running after {int(timeout)}s, check the dashboard for status")
                 elif status.status == "failed":
                     reason = status.failure_reason.message if status.failure_reason else "unknown error"
-                    print(f"  {red('✗')}  Analyze failed: {dim(reason)}")
+                    _say(f"  {red('✗')}  Analyze failed: {dim(reason)}")
                 else:
-                    print(f"  {green('✓')}  Analysis complete")
+                    _say(f"  {green('✓')}  Analysis complete")
             except Exception as exc:
-                print(f"  {red('✗')}  Analyze failed: {dim(str(exc))}")
+                _say(f"  {red('✗')}  Analyze failed: {dim(str(exc))}")
                 logger.warning("Analyze request failed: %s", exc)
 
         try:
@@ -375,7 +390,7 @@ class EvaluationRunContext:
             # the one signal that something went wrong used to be a logger.warning that is
             # invisible unless the caller configured logging. Say it on stdout, and let the
             # status carry the truth for anything reading the object.
-            print(f"  {red('✗')}  Could not fetch the report: {dim(str(exc))}")
+            _say(f"  {red('✗')}  Could not fetch the report: {dim(str(exc))}")
             logger.warning("Could not fetch report: %s", exc)
             report = Report(
                 runId=self._run.run_id,
@@ -384,7 +399,7 @@ class EvaluationRunContext:
             )
 
         self._report = report
-        print()
+        _say()
         print_report(report)
         return report
 
@@ -616,4 +631,4 @@ def _print_progress(
     line = f"  {tag}  {counter} {label}  {query_preview}"
     if suffix:
         line += f"  {suffix}"
-    print(line)
+    _say(line)

{agentx_python-0.6.31 → agentx_python-0.6.33}/agentx/monitor/client.py

@@ -82,6 +82,10 @@ class MonitorClient:
 
         self.patterns = MonitorPatternClient(self)
         self.signals = MonitorSignalClient(self)
+        from agentx.monitor.scorers import ScorersClient
+        # Scorers-catalog administration as code: template enable/disable, code/external scorer
+        # CRUD and dry runs - full parity with the dashboard's Scorers page (P1.3).
+        self.scorers = ScorersClient(api_key=api_key)
         self.profile = MonitorProfileClient(self)
         self.online_evaluators = MonitorOnlineEvaluatorClient(self)
         from agentx.monitor.sessions import MonitorSessionClient

{agentx_python-0.6.31 → agentx_python-0.6.33}/agentx/monitor/models.py

@@ -122,7 +122,7 @@ class MonitorProfile(BaseModel):
     ``None`` from ``get()`` means this agent has never been configured and is running on
     platform defaults (e.g. the built-in latency threshold defaults to 20000ms).
 
-    Self-host only: ``coverage_mode``/``sample_rate``/``retention_days``/``redaction_mode``, and
+    Self-host only: ``coverage_mode``/``sample_rate``/``retention_days``, and
     ``threshold_overrides["latencyMs"]`` are project-level defaults now (set once for every agent
     via the dashboard's Platform Settings screen), not real per-agent settings - this model and
     ``update()`` still accept/return them for wire compatibility, but a self-host engine no longer
@@ -141,7 +141,6 @@ class MonitorProfile(BaseModel):
     channels: List[str] = Field(default_factory=list)
     threshold_overrides: Optional[Dict[str, Any]] = Field(default=None, alias="thresholdOverrides")
     retention_days: int = Field(default=30, alias="retentionDays")
-    redaction_mode: str = Field(default="standard", alias="redactionMode")
     approval_policy: Optional[Dict[str, str]] = Field(default=None, alias="approvalPolicy")
     created_at: Optional[str] = Field(default=None, alias="createdAt")
     updated_at: Optional[str] = Field(default=None, alias="updatedAt")

{agentx_python-0.6.31 → agentx_python-0.6.33}/agentx/monitor/profile.py

@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
 
 class MonitorProfileClient:
     """Thin wrapper surfaced as ``client.monitor.profile``: get/update a single agent's Monitor
-    coverage and detection settings (coverage mode, sample rate, retention, redaction, approval
+    coverage and detection settings (coverage mode, sample rate, retention, approval
     policy, and threshold_overrides, e.g. the built-in "Latency regression" pattern's threshold).
 
     Unlike patterns/signals, a profile is scoped to one agent per call, since that mirrors how
@@ -41,13 +41,12 @@ class MonitorProfileClient:
         dataset_id: Optional[str] = None,
         threshold_overrides: Optional[Dict[str, Any]] = None,
         retention_days: Optional[int] = None,
-        redaction_mode: Optional[str] = None,
         approval_policy: Optional[Dict[str, str]] = None,
     ) -> MonitorProfile:
         """Update (and enable, if not already) this agent's Monitor profile. Only fields passed
         here are changed; everything else on the existing profile is left as is.
 
-        Self-host only: ``coverage_mode``/``sample_rate``/``retention_days``/``redaction_mode``,
+        Self-host only: ``coverage_mode``/``sample_rate``/``retention_days``,
         and ``threshold_overrides["latencyMs"]`` are project-level defaults now (see
         ``MonitorProfile``'s docstring) - set them via the dashboard's Platform Settings screen
         instead, passing them here is accepted but has no effect. ``enabled``/
@@ -67,7 +66,6 @@ class MonitorProfileClient:
             "datasetId": dataset_id,
             "thresholdOverrides": threshold_overrides,
             "retentionDays": retention_days,
-            "redactionMode": redaction_mode,
             "approvalPolicy": approval_policy,
         }
         payload = {k: v for k, v in payload.items() if v is not None}

agentx_python-0.6.33/agentx/monitor/scorers.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional, Sequence
+
+import requests
+
+from agentx.util import api_base, get_headers
+
+logger = logging.getLogger(__name__)
+
+
+class AgentXScorersError(Exception):
+    pass
+
+
+class ScorersClient:
+    """Surfaced as ``client.monitor.scorers``: administer the Scorers catalog as code.
+
+    Covers what the dashboard's Scorers page does:
+
+    - **Template scorers** (the shipped zero-LLM detectors): ``templates()`` lists them with
+      enablement, ``enable()``/``disable()`` flip them. Everything is opt-in - a fresh project
+      runs nothing until a scorer is enabled.
+    - **Code scorers**: ``create_code()`` deploys your own Python/JavaScript
+      ``handler(input, output, expected, metadata, trace)`` run in-engine per sampled trace.
+    - **External scorers**: ``create_external()`` registers your HTTP endpoint (contract v2:
+      the full trace record plus its span subtree).
+    - Shared CRUD: ``list()``, ``update()``, ``delete()``, and ``dry_run()`` (executes a code
+      scorer, or POSTs the sample payload to an external URL, without persisting anything).
+
+    The engine resource name for code/external scorers remains ``custom-evaluators`` on the
+    wire.
+    """
+
+    def __init__(self, api_key: Optional[str] = None):
+        self._api_key = api_key
+
+    def _request(self, method: str, path: str, json: Any = None, params: Any = None) -> Any:
+        resp = requests.request(
+            method,
+            f"{api_base()}/agent-monitoring{path}",
+            headers={**get_headers(self._api_key), "Content-Type": "application/json"},
+            json=json,
+            params=params,
+            timeout=20,
+        )
+        if resp.status_code >= 400:
+            try:
+                detail = resp.json().get("error", resp.reason)
+            except ValueError:
+                detail = resp.reason
+            raise AgentXScorersError(f"Scorer request failed ({resp.status_code}): {detail}")
+        return resp.json() if resp.text else {}
+
+    # ------------------------------------------------------------------
+    # Template scorers (built-in, opt-in)
+    # ------------------------------------------------------------------
+
+    def templates(self) -> List[Dict[str, Any]]:
+        """The shipped template scorers with their keys, rules, and ``enabled`` state."""
+        patterns = self._request("GET", "/patterns").get("patterns", [])
+        return [p for p in patterns if p.get("source") == "builtIn"]
+
+    def _enabled_template_keys(self) -> List[str]:
+        return [p["key"] for p in self.templates() if p.get("enabled")]
+
+    def enable(self, keys: Sequence[str]) -> List[str]:
+        """Enable template scorers by key (e.g. ``["pii-in-response"]``), preserving what is
+        already on. Returns the resulting enabled-key list."""
+        merged = sorted(set(self._enabled_template_keys()) | set(keys))
+        self._request("PUT", "/settings/monitoring-defaults", json={"enabledBuiltinPatterns": merged})
+        return merged
+
+    def disable(self, keys: Sequence[str]) -> List[str]:
+        """Disable template scorers by key, preserving the rest. Returns the resulting list."""
+        merged = sorted(set(self._enabled_template_keys()) - set(keys))
+        self._request("PUT", "/settings/monitoring-defaults", json={"enabledBuiltinPatterns": merged})
+        return merged
+
+    # ------------------------------------------------------------------
+    # Code / external scorers
+    # ------------------------------------------------------------------
+
+    def list(self) -> List[Dict[str, Any]]:
+        """All code and external scorers (wire kind: ``"code"`` / ``"external"``)."""
+        return self._request("GET", "/custom-evaluators").get("evaluators", [])
+
+    def create_code(
+        self,
+        name: str,
+        script: str,
+        *,
+        language: str = "python",
+        alert_below: float = 0.5,
+        sample_rate: float = 0.1,
+        severity: str = "medium",
+        enabled: bool = True,
+        scope_mode: str = "all",
+        agent_ids: Optional[Sequence[str]] = None,
+    ) -> Dict[str, Any]:
+        """Deploy a code scorer. ``script`` defines ``handler(input, output, expected,
+        metadata, trace)`` returning a 0..1 score, ``{"score", "name"?, "metadata"?}``, or
+        ``None`` to skip; a score below ``alert_below`` raises a signal."""
+        if language not in ("python", "javascript"):
+            raise AgentXScorersError('language must be "python" or "javascript"')
+        return self._request("POST", "/custom-evaluators", json={
+            "name": name,
+            "kind": "code",
+            "language": language,
+            "script": script,
+            "alertBelow": alert_below,
+            "sampleRate": sample_rate,
+            "severity": severity,
+            "enabled": enabled,
+            "scopeMode": scope_mode,
+            "agentIds": list(agent_ids) if agent_ids else [],
+        })["evaluator"]
+
+    def create_external(
+        self,
+        name: str,
+        url: str,
+        *,
+        sample_rate: float = 0.1,
+        severity: str = "medium",
+        enabled: bool = True,
+        invert_match: bool = False,
+        scope_mode: str = "all",
+        agent_ids: Optional[Sequence[str]] = None,
+    ) -> Dict[str, Any]:
+        """Register an external scorer endpoint (POSTed the v2 payload per sampled trace)."""
+        return self._request("POST", "/custom-evaluators", json={
+            "name": name,
+            "url": url,
+            "sampleRate": sample_rate,
+            "severity": severity,
+            "enabled": enabled,
+            "invertMatch": invert_match,
+            "scopeMode": scope_mode,
+            "agentIds": list(agent_ids) if agent_ids else [],
+        })["evaluator"]
+
+    def update(self, scorer_id: str, **fields: Any) -> Dict[str, Any]:
+        """Update a code/external scorer. snake_case kwargs are converted (``alert_below`` ->
+        ``alertBelow`` etc.); kind is immutable."""
+        wire = {_SNAKE_TO_WIRE.get(k, k): v for k, v in fields.items()}
+        return self._request("PUT", f"/custom-evaluators/{scorer_id}", json=wire)["evaluator"]
+
+    def delete(self, scorer_id: str) -> None:
+        self._request("DELETE", f"/custom-evaluators/{scorer_id}")
+
+    def events(self, scorer_id: str, window: str = "24h") -> List[Dict[str, Any]]:
+        """The scorer's per-check history (score, matched, justification, trace ids)."""
+        return self._request("GET", f"/custom-evaluators/{scorer_id}/events", params={"window": window}).get("events", [])
+
+    def dry_run(self, **payload: Any) -> Dict[str, Any]:
+        """Execute a scorer against the built-in sample without persisting: pass either
+        ``url=...`` (external) or ``kind="code", language=..., script=...`` (code)."""
+        wire = {_SNAKE_TO_WIRE.get(k, k): v for k, v in payload.items()}
+        return self._request("POST", "/custom-evaluators/dry-run", json=wire)
+
+
+_SNAKE_TO_WIRE = {
+    "alert_below": "alertBelow",
+    "sample_rate": "sampleRate",
+    "scope_mode": "scopeMode",
+    "agent_ids": "agentIds",
+    "invert_match": "invertMatch",
+}

agentx_python-0.6.33/agentx/projects.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from agentx.util import api_base, get_headers
+
+logger = logging.getLogger(__name__)
+
+
+class AgentXProjectsError(Exception):
+    pass
+
+
+class ProjectsClient:
+    """Surfaced as ``client.projects``: create, list, and delete the engine's projects
+    (self-host). Each project is a fully isolated tenant - own API key, own traces, scorers,
+    datasets, and settings. ``create()`` returns the new project's ``apiKey``; construct a new
+    ``AgentX(api_key=...)`` with it to work inside that project (the pattern integration tests
+    use for per-run isolation).
+
+    In ``AGENTX_AUTH=enabled`` mode project management is session-scoped to signed-in dashboard
+    users; this client covers the default self-host (auth-disabled) mode.
+    """
+
+    def __init__(self, api_key: Optional[str] = None):
+        self._api_key = api_key
+
+    def _request(self, method: str, path: str, json: Any = None) -> Any:
+        resp = requests.request(
+            method,
+            f"{api_base()}{path}",
+            headers={**get_headers(self._api_key), "Content-Type": "application/json"},
+            json=json,
+            timeout=15,
+        )
+        if resp.status_code >= 400:
+            try:
+                detail = resp.json().get("error", resp.reason)
+            except ValueError:
+                detail = resp.reason
+            raise AgentXProjectsError(f"Projects request failed ({resp.status_code}): {detail}")
+        return resp.json() if resp.text else {}
+
+    def create(self, name: str) -> Dict[str, Any]:
+        """Create a project; the returned dict includes ``_id``, ``name``, and ``apiKey``."""
+        return self._request("POST", "/projects", json={"name": name})["project"]
+
+    def list(self) -> List[Dict[str, Any]]:
+        """All projects on the instance, each with its ``apiKey`` and ``isDefault`` flag."""
+        return self._request("GET", "/projects").get("projects", [])
+
+    def delete(self, project_id: str) -> None:
+        """Delete a project and every row it owns (traces, scorers, datasets, runs). The
+        default project cannot be deleted. Irreversible."""
+        self._request("DELETE", f"/projects/{project_id}")

agentx_python-0.6.33/agentx/traces.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from agentx.util import api_base, get_headers
+
+logger = logging.getLogger(__name__)
+
+
+class AgentXTracesError(Exception):
+    pass
+
+
+class TracesClient:
+    """Surfaced as ``client.traces``: the READ side of tracing (``client.tracer`` writes).
+
+    ``get(trace_id)`` returns one trace's full detail (input/output/error, model, latency,
+    token counts incl. cache, session/span linkage, metadata, estimated cost) - the same wire
+    the dashboard's trace dialog reads. ``list()`` pages through the project's traces newest
+    first. For a whole conversation, ``client.monitor.sessions.spans(session_id)`` remains the
+    span-tree read.
+    """
+
+    def __init__(self, api_key: Optional[str] = None):
+        self._api_key = api_key
+
+    def _request(self, path: str, params: Optional[Dict[str, Any]] = None) -> Any:
+        resp = requests.get(
+            f"{api_base()}{path}",
+            headers=get_headers(self._api_key),
+            params=params or {},
+            timeout=15,
+        )
+        if resp.status_code >= 400:
+            try:
+                detail = resp.json().get("error", resp.reason)
+            except ValueError:
+                detail = resp.reason
+            raise AgentXTracesError(f"Trace request failed ({resp.status_code}): {detail}")
+        return resp.json()
+
+    def get(self, trace_id: str) -> Dict[str, Any]:
+        """One trace's detail row. Raises on 404."""
+        return self._request(f"/ingest/traces/{trace_id}")
+
+    def list(
+        self,
+        limit: int = 50,
+        cursor: Optional[str] = None,
+        framework: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """A page of traces, newest first: ``{"traces": [...], "nextCursor": str | None}``.
+        Pass ``cursor`` from the previous page to continue."""
+        params: Dict[str, Any] = {"limit": limit}
+        if cursor:
+            params["cursor"] = cursor
+        if framework:
+            params["framework"] = framework
+        return self._request("/ingest/traces", params)

{agentx_python-0.6.31 → agentx_python-0.6.33}/agentx/tracing/tracer.py

@@ -153,6 +153,15 @@ class _TraceSpan:
         if exc_val is not None and self._error is None:
             self._error = str(exc_val)
 
+        if self._sync and self._parent_span_id is None:
+            # sync=True means the WHOLE tree is delivered before this block returns: child
+            # spans (tool calls, LLM calls) were enqueued asynchronously during the block, so
+            # drain them before the root's own synchronous send. Without this, read-after-trace
+            # intermittently misses children (root lands, children still in flight) - the exact
+            # race the enterprise assessment reproduced (P0.1). Bounded by the same 5s budget
+            # flush() uses; child-only spans keep their async fire-and-forget behavior.
+            self._tracer.flush(timeout=5.0)
+
         self._trace_id = self._tracer._send(
             sync=self._sync,
             monitor=self._monitor,
@@ -833,7 +842,10 @@ class Tracer:
         By default the trace is queued and sent on a background thread - fire-and-forget, never
         blocks the caller, but there's no way to learn the resulting trace_id. Pass ``sync=True``
         to send it synchronously instead (blocks until ingested) so ``span.trace_id`` is populated
-        once the ``with`` block exits - e.g. to attach the trace to an evaluation result::
+        once the ``with`` block exits. On a root span, ``sync=True`` covers the WHOLE tree: any
+        child spans recorded inside the block (tool calls, LLM calls) are drained before the
+        root is sent, so a read immediately after the block sees every span, not just the root.
+        Use it e.g. to attach the trace to an evaluation result::
 
             with client.tracer.trace("support_agent_call", framework="openai", sync=True) as span:
                 resp = call_llm(...)

agentx_python-0.6.33/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.6.33"

{agentx_python-0.6.31 → agentx_python-0.6.33}/agentx_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.31
+Version: 0.6.33
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.31 → agentx_python-0.6.33}/agentx_python.egg-info/SOURCES.txt

@@ -7,7 +7,9 @@ agentx/cli.py
 agentx/exceptions.py
 agentx/feedback.py
 agentx/outcomes.py
+agentx/projects.py
 agentx/py.typed
+agentx/traces.py
 agentx/util.py
 agentx/version.py
 agentx/evaluations/__init__.py
@@ -17,7 +19,6 @@ agentx/evaluations/datasets.py
 agentx/evaluations/evaluation_settings.py
 agentx/evaluations/models.py
 agentx/evaluations/prompts.py
-agentx/evaluations/redaction.py
 agentx/evaluations/reporting.py
 agentx/evaluations/results.py
 agentx/evaluations/runner.py
@@ -48,6 +49,7 @@ agentx/monitor/models.py
 agentx/monitor/online_evaluators.py
 agentx/monitor/patterns.py
 agentx/monitor/profile.py
+agentx/monitor/scorers.py
 agentx/monitor/sessions.py
 agentx/monitor/signals.py
 agentx/resources/__init__.py

agentx_python-0.6.31/agentx/evaluations/redaction.py

@@ -1,57 +0,0 @@
-from __future__ import annotations
-
-import re
-from typing import Any, Dict
-
-# Patterns that look like secrets
-_SECRET_PATTERNS = [
-    re.compile(r"sk-[A-Za-z0-9]{20,}"),  # OpenAI / Anthropic style keys
-    re.compile(r"Bearer\s+[A-Za-z0-9\-._~+/]+=*", re.IGNORECASE),
-    re.compile(r"(?i)(password|passwd|secret|token|api[_-]?key)\s*[=:]\s*\S+"),
-    re.compile(r"[A-Za-z0-9+/]{40,}={0,2}"),  # long base64-like strings
-]
-
-_REDACTED = "[REDACTED]"
-
-
-def redact_string(value: str) -> str:
-    for pattern in _SECRET_PATTERNS:
-        value = pattern.sub(_REDACTED, value)
-    return value
-
-
-def redact_dict(obj: Any, _depth: int = 0) -> Any:
-    if _depth > 10:
-        return obj
-    if isinstance(obj, dict):
-        return {k: _redact_value(k, v, _depth) for k, v in obj.items()}
-    if isinstance(obj, list):
-        return [redact_dict(item, _depth + 1) for item in obj]
-    if isinstance(obj, str):
-        return redact_string(obj)
-    return obj
-
-
-_SENSITIVE_KEYS = {
-    "password",
-    "passwd",
-    "secret",
-    "token",
-    "api_key",
-    "apikey",
-    "authorization",
-    "auth",
-    "cookie",
-    "session",
-    "credential",
-    "private_key",
-    "privatekey",
-    "access_key",
-    "accesskey",
-}
-
-
-def _redact_value(key: str, value: Any, depth: int) -> Any:
-    if isinstance(key, str) and key.lower() in _SENSITIVE_KEYS:
-        return _REDACTED
-    return redact_dict(value, depth + 1)

agentx_python-0.6.31/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.6.31"