agentx-python 0.6.32__tar.gz → 0.6.34__tar.gz

{agentx_python-0.6.32 → agentx_python-0.6.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.32
+Version: 0.6.34
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.32 → agentx_python-0.6.34}/agentx/agentx.py

@@ -60,6 +60,22 @@ class AgentX:
         # ground truth behind the dashboard's Judge Calibration card. Self-host only.
         self.outcomes = OutcomesClient(api_key=self.api_key)
 
+        from agentx.projects import ProjectsClient
+
+        # Project CRUD (self-host): isolated tenants with their own API keys (P1.1).
+        self.projects = ProjectsClient(api_key=self.api_key)
+
+        from agentx.traces import TracesClient
+
+        # The read side of tracing: trace-by-id detail and paginated listing (P1.2).
+        self.traces = TracesClient(api_key=self.api_key)
+
+        from agentx.export import ExportClient
+
+        # Bulk NDJSON egress for backup/migration (P2.1): manifest, per-entity streaming, and
+        # directory dumps. Self-host only.
+        self.export = ExportClient(api_key=self.api_key)
+
         from agentx.feedback import FeedbackClient
 
         # Forward end-user votes ("up"/"down") on traced responses - a "down" raises a signal

{agentx_python-0.6.32 → agentx_python-0.6.34}/agentx/evaluations/_term.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import itertools
 import sys
+import os
 import threading
 import time
 
@@ -61,8 +62,12 @@ class Spinner:
         self._message = message
         self._stop = threading.Event()
         self._thread: threading.Thread | None = None
+        # AGENTX_EVAL_QUIET=1: no spinner thread at all - CI logs stay clean.
+        self._quiet = os.getenv("AGENTX_EVAL_QUIET", "").lower() in ("1", "true", "yes")
 
     def __enter__(self) -> "Spinner":
+        if self._quiet:
+            return self
         if not _IS_TTY:
             print(f"  {self._message}...", flush=True)
             return self
@@ -78,7 +83,7 @@ class Spinner:
             print(f"  {message}...", flush=True)
 
     def __exit__(self, *_) -> None:
-        if not _IS_TTY:
+        if self._quiet or not _IS_TTY:
             return
         self._stop.set()
         if self._thread:

{agentx_python-0.6.32 → agentx_python-0.6.34}/agentx/evaluations/datasets.py

@@ -35,6 +35,7 @@ class DatasetBuilder:
         rouge_score: bool = False,
         similarity_model: Optional[str] = None,
         sovereignty_models: Optional[List[str]] = None,
+        code_scorers: Optional[List[Dict[str, Any]]] = None,
     ):
         self._client = client
         self._payload: Dict[str, Any] = {
@@ -63,6 +64,21 @@ class DatasetBuilder:
             self._payload["vectorSimilarity"] = vs
         if jaccard_similarity:
             self._payload["jaccardSimilarity"] = {"enabled": True}
+        # Offline code scorers, versioned in the repo next to the dataset they guard (P1.4):
+        # each entry is {"name", "code"} (a JS function body invoked as
+        # score({input, output, expected, toolCalls})), optional "enabled" (default True).
+        if code_scorers:
+            import uuid as _uuid
+
+            self._payload["codeScorers"] = [
+                {
+                    "id": scorer.get("id") or _uuid.uuid4().hex[:12],
+                    "name": scorer["name"],
+                    "code": scorer["code"],
+                    "enabled": scorer.get("enabled", True),
+                }
+                for scorer in code_scorers
+            ]
         if bleu_score:
             self._payload["bleuScore"] = {"enabled": True}
         if rouge_score:

{agentx_python-0.6.32 → agentx_python-0.6.34}/agentx/evaluations/models.py

@@ -399,6 +399,65 @@ class BatchAppendResponse(BaseModel):
         extra = "ignore"
 
 
+# ---------------------------------------------------------------------------
+# Typed run-result rows (P1.5)
+# ---------------------------------------------------------------------------
+
+
+class RunResultRow(BaseModel):
+    """One row of a finished run's results - typed and snake_case, replacing the raw wire dicts
+    ``run.results()`` used to return. ``.raw`` keeps the full wire dict for anything not yet
+    modeled. Dict-style access (``row["rating"]``, ``row.get(...)``) still works for one
+    deprecation cycle and warns; prefer the attributes."""
+
+    rating: Optional[float] = None
+    justification: Optional[str] = None
+    question_text: Optional[str] = Field(default=None, alias="questionText")
+    response: Optional[str] = None
+    trace_id: Optional[str] = Field(default=None, alias="traceId")
+    latency_ms: Optional[float] = Field(default=None, alias="latencyMs")
+    input_tokens: Optional[int] = Field(default=None, alias="inputTokens")
+    output_tokens: Optional[int] = Field(default=None, alias="outputTokens")
+    cosine_similarity: Optional[float] = Field(default=None, alias="cosineSimilarity")
+    jaccard_similarity: Optional[float] = Field(default=None, alias="jaccardSimilarity")
+    bleu_score: Optional[float] = Field(default=None, alias="bleuScore")
+    rouge_score: Optional[float] = Field(default=None, alias="rougeScore")
+    code_scorer_results: Optional[List[Dict[str, Any]]] = Field(default=None, alias="codeScorerResults")
+    raw: Dict[str, Any] = Field(default_factory=dict)
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+    @classmethod
+    def from_wire(cls, wire: Dict[str, Any]) -> "RunResultRow":
+        row = cls.model_validate(wire)
+        row.raw = wire
+        return row
+
+    def __getitem__(self, key: str) -> Any:
+        import warnings
+
+        warnings.warn(
+            "Dict-style access on run results is deprecated - use typed attributes "
+            '(row.rating, row.jaccard_similarity) or row.raw["..."] for unmodeled fields.',
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self.raw[key]
+
+    def get(self, key: str, default: Any = None) -> Any:
+        import warnings
+
+        warnings.warn(
+            "Dict-style access on run results is deprecated - use typed attributes "
+            'or row.raw.get("...") for unmodeled fields.',
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self.raw.get(key, default)
+
+
 # ---------------------------------------------------------------------------
 # Analysis / report
 # ---------------------------------------------------------------------------

{agentx_python-0.6.32 → agentx_python-0.6.34}/agentx/evaluations/runner.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import os
 import time
 import uuid
 from typing import Any, Callable, Dict, List, Optional, Set, Union
@@ -37,6 +38,17 @@ from agentx.evaluations._term import (
 
 logger = logging.getLogger(__name__)
 
+
+def _eval_quiet() -> bool:
+    """AGENTX_EVAL_QUIET=1 silences the interactive progress UI (spinners, per-case lines) for
+    CI logs - results, gate verdicts, and errors still print. Read per call so tests can toggle."""
+    return os.getenv("AGENTX_EVAL_QUIET", "").lower() in ("1", "true", "yes")
+
+
+def _say(*args, **kwargs) -> None:
+    if not _eval_quiet():
+        print(*args, **kwargs)
+
 AdapterLike = Union[
     Callable[[EvaluationCase], Any],
     RawCallableAdapter,
@@ -126,18 +138,18 @@ class EvaluationRunContext:
         )
         n_smoke = sum(1 for c in cases if c.is_smoke_test_variant)
 
-        print(cyan(sep))
-        print(f"  {bold('AgentX Evaluation')}  {dim(' - ')}  {name}")
-        print(cyan(sep))
-        print(f"  {dim('Run   :')} {dim(self._run.run_id)}")
+        _say(cyan(sep))
+        _say(f"  {bold('AgentX Evaluation')}  {dim(' - ')}  {name}")
+        _say(cyan(sep))
+        _say(f"  {dim('Run   :')} {dim(self._run.run_id)}")
         if display:
-            print(f"  {dim('Agent :')} {display}  {dim(f'({framework} / {runtime})')}")
-        print()
+            _say(f"  {dim('Agent :')} {display}  {dim(f'({framework} / {runtime})')}")
+        _say()
         exec_line = f"{bold('Executing')}  {n_q} question{'s' if n_q != 1 else ''} × {n_r} run{'s' if n_r != 1 else ''}"
         if n_smoke:
             variant_word = "variant" if n_smoke == 1 else "variants"
             exec_line += f"  {dim(f'(+{n_smoke} smoke-test {variant_word})')}"
-        print(exec_line)
+        _say(exec_line)
 
         # Resume: skip already-submitted keys
         already_done = self._fetch_submitted_keys()
@@ -185,7 +197,7 @@ class EvaluationRunContext:
                 resp = self._client.append_results(self._run.run_id, batch_id, batch)
                 if resp.live_statistics is not None:
                     self._live_stats = resp.live_statistics
-                print(
+                _say(
                     f"  {green('✓')}  Scored {resp.accepted} result{'s' if resp.accepted != 1 else ''}"
                 )
                 logger.info(
@@ -196,7 +208,7 @@ class EvaluationRunContext:
                     resp.failed_validation,
                 )
             except Exception as exc:
-                print(f"  {red('✗')}  Scoring failed: {dim(str(exc))}")
+                _say(f"  {red('✗')}  Scoring failed: {dim(str(exc))}")
                 logger.error("Failed to submit batch %s: %s", batch_id[:8], exc)
 
     def _fetch_submitted_keys(self) -> Set[str]:
@@ -213,16 +225,16 @@ class EvaluationRunContext:
     # ------------------------------------------------------------------
 
     def finalize(self) -> "EvaluationRunContext":
-        print()
+        _say()
         with Spinner("Finalizing - submitting results"):
             try:
                 data = self._client.finalize_run(self._run.run_id)
                 if isinstance(data, dict) and data.get("liveStatistics") is not None:
                     self._live_stats = LiveStatistics(**data["liveStatistics"])
-                print(f"  {green('✓')}  Finalized")
+                _say(f"  {green('✓')}  Finalized")
                 logger.info("Run %s finalized", self._run.run_id)
             except Exception as exc:
-                print(f"  {red('✗')}  Finalize failed: {dim(str(exc))}")
+                _say(f"  {red('✗')}  Finalize failed: {dim(str(exc))}")
                 logger.error("Finalize failed: %s", exc)
         return self
 
@@ -255,11 +267,11 @@ class EvaluationRunContext:
             caller=caller,
         )
         result = GateResult(data)
-        print()
+        _say()
         for check in result.checks:
             mark = green("✓") if check.get("passed") else red("✗")
-            print(f"  {mark}  [{check.get('check')}] {check.get('detail')}")
-        print(f"  {green('✓  GATE PASSED') if result.passed else red('✗  GATE FAILED')}")
+            _say(f"  {mark}  [{check.get('check')}] {check.get('detail')}")
+        _say(f"  {green('✓  GATE PASSED') if result.passed else red('✗  GATE FAILED')}")
         return result
 
     # ------------------------------------------------------------------
@@ -273,9 +285,13 @@ class EvaluationRunContext:
     def results(self) -> list:
         """Per-result rows for this run (rating, justification, code scorer rows, trace ids,
         latency/tokens, similarity metrics) - what the dashboard's run detail table shows,
-        fetched fresh from the engine."""
+        fetched fresh from the engine. Returns typed ``RunResultRow`` objects (snake_case
+        attributes; ``.raw`` is the wire dict; dict-style access warns for one cycle - P1.5)."""
+        from agentx.evaluations.models import RunResultRow
+
         detail = self._client.get_run(self.run_id)
-        return detail.get("results", []) if isinstance(detail, dict) else []
+        rows = detail.get("results", []) if isinstance(detail, dict) else []
+        return [RunResultRow.from_wire(r) for r in rows]
 
     @property
     def run_id(self) -> str:
@@ -335,7 +351,7 @@ class EvaluationRunContext:
             raise ValueError("judges must contain 1-3 model ids")
         resolved_judges = judges if judges is not None else [_DEFAULT_JUDGE_MODEL]
 
-        print()
+        _say()
         with Spinner("Analyzing - AI is reviewing your results") as spinner:
             try:
                 self._client.analyze_run(
@@ -355,14 +371,14 @@ class EvaluationRunContext:
                     status = self._client.get_analysis_status(self._run.run_id)
 
                 if not status.is_terminal:
-                    print(f"  {yellow('!')}  Still running after {int(timeout)}s, check the dashboard for status")
+                    _say(f"  {yellow('!')}  Still running after {int(timeout)}s, check the dashboard for status")
                 elif status.status == "failed":
                     reason = status.failure_reason.message if status.failure_reason else "unknown error"
-                    print(f"  {red('✗')}  Analyze failed: {dim(reason)}")
+                    _say(f"  {red('✗')}  Analyze failed: {dim(reason)}")
                 else:
-                    print(f"  {green('✓')}  Analysis complete")
+                    _say(f"  {green('✓')}  Analysis complete")
             except Exception as exc:
-                print(f"  {red('✗')}  Analyze failed: {dim(str(exc))}")
+                _say(f"  {red('✗')}  Analyze failed: {dim(str(exc))}")
                 logger.warning("Analyze request failed: %s", exc)
 
         try:
@@ -374,7 +390,7 @@ class EvaluationRunContext:
             # the one signal that something went wrong used to be a logger.warning that is
             # invisible unless the caller configured logging. Say it on stdout, and let the
             # status carry the truth for anything reading the object.
-            print(f"  {red('✗')}  Could not fetch the report: {dim(str(exc))}")
+            _say(f"  {red('✗')}  Could not fetch the report: {dim(str(exc))}")
             logger.warning("Could not fetch report: %s", exc)
             report = Report(
                 runId=self._run.run_id,
@@ -383,7 +399,7 @@ class EvaluationRunContext:
             )
 
         self._report = report
-        print()
+        _say()
         print_report(report)
         return report
 
@@ -615,4 +631,4 @@ def _print_progress(
     line = f"  {tag}  {counter} {label}  {query_preview}"
     if suffix:
         line += f"  {suffix}"
-    print(line)
+    _say(line)

agentx_python-0.6.34/agentx/export.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+from typing import Any, Dict, Iterator, List, Optional
+
+import requests
+
+from agentx.util import api_base, get_headers
+
+logger = logging.getLogger(__name__)
+
+
+class AgentXExportError(Exception):
+    pass
+
+
+class ExportClient:
+    """Surfaced as ``client.export``: bulk NDJSON egress for backup and migration (self-host).
+
+    The engine's ``GET /export`` manifest lists every exportable entity (traces, signals,
+    events, runs, feedback, outcomes, scorer config, ...) with live row counts;
+    ``GET /export/<entity>`` streams the rows as NDJSON. Everything is scoped to the API key's
+    project, so an export can never cross a tenant boundary.
+
+    Typical uses::
+
+        client.export.dump("./backup")                     # full backup, one .ndjson per entity
+        client.export.dump("./nightly", since=yesterday)   # incremental
+        for row in client.export.iter("traces"):           # stream without touching disk
+            ...
+
+    Restore paths are documented in the self-host backup runbook: replay traces through
+    ``client.tracer`` / ``POST /ingest/traces``, or restore at the database level
+    (``pg_dump`` / SQLite file copy). There is deliberately no blind row-import endpoint.
+    """
+
+    def __init__(self, api_key: Optional[str] = None):
+        self._api_key = api_key
+
+    def manifest(self) -> List[Dict[str, Any]]:
+        """The exportable entities with live row counts: ``[{entity, rows, path}, ...]``."""
+        resp = requests.get(
+            f"{api_base()}/export", headers=get_headers(self._api_key), timeout=30
+        )
+        if resp.status_code >= 400:
+            raise AgentXExportError(f"Export manifest failed ({resp.status_code}): {resp.text[:200]}")
+        return resp.json().get("entities", [])
+
+    def iter(self, entity: str, since: Optional[str] = None) -> Iterator[Dict[str, Any]]:
+        """Stream one entity's rows as dicts without buffering the whole table in memory.
+        ``since`` is an ISO-8601 date for incremental pulls (filters on the entity's own
+        timestamp column, e.g. ``createdAt`` for traces, ``lastSeenAt`` for signals)."""
+        params = {"since": since} if since else None
+        resp = requests.get(
+            f"{api_base()}/export/{entity}",
+            headers=get_headers(self._api_key),
+            params=params,
+            stream=True,
+            timeout=120,
+        )
+        if resp.status_code >= 400:
+            raise AgentXExportError(f"Export of {entity!r} failed ({resp.status_code}): {resp.text[:200]}")
+        for line in resp.iter_lines(decode_unicode=True):
+            if line and line.strip():
+                yield json.loads(line)
+
+    def dump(
+        self,
+        directory: str,
+        entities: Optional[List[str]] = None,
+        since: Optional[str] = None,
+    ) -> Dict[str, int]:
+        """Write ``<entity>.ndjson`` files (plus a ``manifest.json``) into ``directory`` and
+        return ``{entity: rows_written}``. Defaults to every entity the engine advertises;
+        pass ``entities`` to restrict, ``since`` for an incremental snapshot."""
+        os.makedirs(directory, exist_ok=True)
+        manifest = self.manifest()
+        wanted = [e["entity"] for e in manifest] if entities is None else entities
+        written: Dict[str, int] = {}
+        for entity in wanted:
+            path = os.path.join(directory, f"{entity}.ndjson")
+            count = 0
+            with open(path, "w", encoding="utf-8") as fh:
+                for row in self.iter(entity, since=since):
+                    fh.write(json.dumps(row, ensure_ascii=False) + "\n")
+                    count += 1
+            written[entity] = count
+            logger.debug("Exported %d %s rows to %s", count, entity, path)
+        with open(os.path.join(directory, "manifest.json"), "w", encoding="utf-8") as fh:
+            json.dump(
+                {"entities": manifest, "written": written, **({"since": since} if since else {})},
+                fh,
+                indent=2,
+            )
+        return written

{agentx_python-0.6.32 → agentx_python-0.6.34}/agentx/monitor/client.py

@@ -82,6 +82,10 @@ class MonitorClient:
 
         self.patterns = MonitorPatternClient(self)
         self.signals = MonitorSignalClient(self)
+        from agentx.monitor.scorers import ScorersClient
+        # Scorers-catalog administration as code: template enable/disable, code/external scorer
+        # CRUD and dry runs - full parity with the dashboard's Scorers page (P1.3).
+        self.scorers = ScorersClient(api_key=api_key)
         self.profile = MonitorProfileClient(self)
         self.online_evaluators = MonitorOnlineEvaluatorClient(self)
         from agentx.monitor.sessions import MonitorSessionClient
@@ -209,6 +213,17 @@ class MonitorClient:
         plus deltas vs the prior window and the run-outcome breakdown."""
         return self._request("GET", "/kpis", params={"window": window})
 
+    def calibration(self, window: str = "7d") -> dict:
+        """Project-level judge calibration over a window ("24h", "7d", or "30d"): how often
+        AgentX's own verdicts agreed with real-world ground truth reported later (ops outcomes
+        via ``client.outcomes`` and end-user downvotes). Returns the dashboard's Judge
+        Calibration numbers: compared count, agreement, falsePositiveRate, falseNegativeRate.
+        Per-evaluator calibration lives on ``client.monitor.online_evaluators.calibration``."""
+        return self._request(
+            "GET", "/agent-monitoring/calibration",
+            base=self._api_root(), params={"window": window},
+        )
+
     # ------------------------------------------------------------------
     # Signal endpoints
     # ------------------------------------------------------------------

agentx_python-0.6.34/agentx/monitor/scorers.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional, Sequence
+
+import requests
+
+from agentx.util import api_base, get_headers
+
+logger = logging.getLogger(__name__)
+
+
+class AgentXScorersError(Exception):
+    pass
+
+
+class ScorersClient:
+    """Surfaced as ``client.monitor.scorers``: administer the Scorers catalog as code.
+
+    Covers what the dashboard's Scorers page does:
+
+    - **Template scorers** (the shipped zero-LLM detectors): ``templates()`` lists them with
+      enablement, ``enable()``/``disable()`` flip them. Everything is opt-in - a fresh project
+      runs nothing until a scorer is enabled.
+    - **Code scorers**: ``create_code()`` deploys your own Python/JavaScript
+      ``handler(input, output, expected, metadata, trace)`` run in-engine per sampled trace.
+    - **External scorers**: ``create_external()`` registers your HTTP endpoint (contract v2:
+      the full trace record plus its span subtree).
+    - Shared CRUD: ``list()``, ``update()``, ``delete()``, and ``dry_run()`` (executes a code
+      scorer, or POSTs the sample payload to an external URL, without persisting anything).
+
+    The engine resource name for code/external scorers remains ``custom-evaluators`` on the
+    wire.
+    """
+
+    def __init__(self, api_key: Optional[str] = None):
+        self._api_key = api_key
+
+    def _request(self, method: str, path: str, json: Any = None, params: Any = None) -> Any:
+        resp = requests.request(
+            method,
+            f"{api_base()}/agent-monitoring{path}",
+            headers={**get_headers(self._api_key), "Content-Type": "application/json"},
+            json=json,
+            params=params,
+            timeout=20,
+        )
+        if resp.status_code >= 400:
+            try:
+                detail = resp.json().get("error", resp.reason)
+            except ValueError:
+                detail = resp.reason
+            raise AgentXScorersError(f"Scorer request failed ({resp.status_code}): {detail}")
+        return resp.json() if resp.text else {}
+
+    # ------------------------------------------------------------------
+    # Template scorers (built-in, opt-in)
+    # ------------------------------------------------------------------
+
+    def templates(self) -> List[Dict[str, Any]]:
+        """The shipped template scorers with their keys, rules, and ``enabled`` state."""
+        patterns = self._request("GET", "/patterns").get("patterns", [])
+        return [p for p in patterns if p.get("source") == "builtIn"]
+
+    def _enabled_template_keys(self) -> List[str]:
+        return [p["key"] for p in self.templates() if p.get("enabled")]
+
+    def enable(self, keys: Sequence[str]) -> List[str]:
+        """Enable template scorers by key (e.g. ``["pii-in-response"]``), preserving what is
+        already on. Returns the resulting enabled-key list."""
+        merged = sorted(set(self._enabled_template_keys()) | set(keys))
+        self._request("PUT", "/settings/monitoring-defaults", json={"enabledBuiltinPatterns": merged})
+        return merged
+
+    def disable(self, keys: Sequence[str]) -> List[str]:
+        """Disable template scorers by key, preserving the rest. Returns the resulting list."""
+        merged = sorted(set(self._enabled_template_keys()) - set(keys))
+        self._request("PUT", "/settings/monitoring-defaults", json={"enabledBuiltinPatterns": merged})
+        return merged
+
+    # ------------------------------------------------------------------
+    # Code / external scorers
+    # ------------------------------------------------------------------
+
+    def list(self) -> List[Dict[str, Any]]:
+        """All code and external scorers (wire kind: ``"code"`` / ``"external"``)."""
+        return self._request("GET", "/custom-evaluators").get("evaluators", [])
+
+    def create_code(
+        self,
+        name: str,
+        script: str,
+        *,
+        language: str = "python",
+        alert_below: float = 0.5,
+        sample_rate: float = 0.1,
+        severity: str = "medium",
+        enabled: bool = True,
+        scope_mode: str = "all",
+        agent_ids: Optional[Sequence[str]] = None,
+    ) -> Dict[str, Any]:
+        """Deploy a code scorer. ``script`` defines ``handler(input, output, expected,
+        metadata, trace)`` returning a 0..1 score, ``{"score", "name"?, "metadata"?}``, or
+        ``None`` to skip; a score below ``alert_below`` raises a signal."""
+        if language not in ("python", "javascript"):
+            raise AgentXScorersError('language must be "python" or "javascript"')
+        return self._request("POST", "/custom-evaluators", json={
+            "name": name,
+            "kind": "code",
+            "language": language,
+            "script": script,
+            "alertBelow": alert_below,
+            "sampleRate": sample_rate,
+            "severity": severity,
+            "enabled": enabled,
+            "scopeMode": scope_mode,
+            "agentIds": list(agent_ids) if agent_ids else [],
+        })["evaluator"]
+
+    def create_external(
+        self,
+        name: str,
+        url: str,
+        *,
+        sample_rate: float = 0.1,
+        severity: str = "medium",
+        enabled: bool = True,
+        invert_match: bool = False,
+        scope_mode: str = "all",
+        agent_ids: Optional[Sequence[str]] = None,
+    ) -> Dict[str, Any]:
+        """Register an external scorer endpoint (POSTed the v2 payload per sampled trace)."""
+        return self._request("POST", "/custom-evaluators", json={
+            "name": name,
+            "url": url,
+            "sampleRate": sample_rate,
+            "severity": severity,
+            "enabled": enabled,
+            "invertMatch": invert_match,
+            "scopeMode": scope_mode,
+            "agentIds": list(agent_ids) if agent_ids else [],
+        })["evaluator"]
+
+    def update(self, scorer_id: str, **fields: Any) -> Dict[str, Any]:
+        """Update a code/external scorer. snake_case kwargs are converted (``alert_below`` ->
+        ``alertBelow`` etc.); kind is immutable."""
+        wire = {_SNAKE_TO_WIRE.get(k, k): v for k, v in fields.items()}
+        return self._request("PUT", f"/custom-evaluators/{scorer_id}", json=wire)["evaluator"]
+
+    def delete(self, scorer_id: str) -> None:
+        self._request("DELETE", f"/custom-evaluators/{scorer_id}")
+
+    def events(self, scorer_id: str, window: str = "24h") -> List[Dict[str, Any]]:
+        """The scorer's per-check history (score, matched, justification, trace ids)."""
+        return self._request("GET", f"/custom-evaluators/{scorer_id}/events", params={"window": window}).get("events", [])
+
+    def dry_run(self, **payload: Any) -> Dict[str, Any]:
+        """Execute a scorer against the built-in sample without persisting: pass either
+        ``url=...`` (external) or ``kind="code", language=..., script=...`` (code)."""
+        wire = {_SNAKE_TO_WIRE.get(k, k): v for k, v in payload.items()}
+        return self._request("POST", "/custom-evaluators/dry-run", json=wire)
+
+
+_SNAKE_TO_WIRE = {
+    "alert_below": "alertBelow",
+    "sample_rate": "sampleRate",
+    "scope_mode": "scopeMode",
+    "agent_ids": "agentIds",
+    "invert_match": "invertMatch",
+}

agentx_python-0.6.34/agentx/projects.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from agentx.util import api_base, get_headers
+
+logger = logging.getLogger(__name__)
+
+
+class AgentXProjectsError(Exception):
+    pass
+
+
+class ProjectsClient:
+    """Surfaced as ``client.projects``: create, list, and delete the engine's projects
+    (self-host). Each project is a fully isolated tenant - own API key, own traces, scorers,
+    datasets, and settings. ``create()`` returns the new project's ``apiKey``; construct a new
+    ``AgentX(api_key=...)`` with it to work inside that project (the pattern integration tests
+    use for per-run isolation).
+
+    In ``AGENTX_AUTH=enabled`` mode project management is session-scoped to signed-in dashboard
+    users; this client covers the default self-host (auth-disabled) mode.
+    """
+
+    def __init__(self, api_key: Optional[str] = None):
+        self._api_key = api_key
+
+    def _request(self, method: str, path: str, json: Any = None) -> Any:
+        resp = requests.request(
+            method,
+            f"{api_base()}{path}",
+            headers={**get_headers(self._api_key), "Content-Type": "application/json"},
+            json=json,
+            timeout=15,
+        )
+        if resp.status_code >= 400:
+            try:
+                detail = resp.json().get("error", resp.reason)
+            except ValueError:
+                detail = resp.reason
+            raise AgentXProjectsError(f"Projects request failed ({resp.status_code}): {detail}")
+        return resp.json() if resp.text else {}
+
+    def create(self, name: str) -> Dict[str, Any]:
+        """Create a project; the returned dict includes ``_id``, ``name``, and ``apiKey``."""
+        return self._request("POST", "/projects", json={"name": name})["project"]
+
+    def list(self) -> List[Dict[str, Any]]:
+        """All projects on the instance, each with its ``apiKey`` and ``isDefault`` flag."""
+        return self._request("GET", "/projects").get("projects", [])
+
+    def delete(self, project_id: str) -> None:
+        """Delete a project and every row it owns (traces, scorers, datasets, runs). The
+        default project cannot be deleted. Irreversible."""
+        self._request("DELETE", f"/projects/{project_id}")

agentx_python-0.6.34/agentx/traces.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from agentx.util import api_base, get_headers
+
+logger = logging.getLogger(__name__)
+
+
+class AgentXTracesError(Exception):
+    pass
+
+
+class TracesClient:
+    """Surfaced as ``client.traces``: the READ side of tracing (``client.tracer`` writes).
+
+    ``get(trace_id)`` returns one trace's full detail (input/output/error, model, latency,
+    token counts incl. cache, session/span linkage, metadata, estimated cost) - the same wire
+    the dashboard's trace dialog reads. ``list()`` pages through the project's traces newest
+    first. For a whole conversation, ``client.monitor.sessions.spans(session_id)`` remains the
+    span-tree read.
+    """
+
+    def __init__(self, api_key: Optional[str] = None):
+        self._api_key = api_key
+
+    def _request(self, path: str, params: Optional[Dict[str, Any]] = None) -> Any:
+        resp = requests.get(
+            f"{api_base()}{path}",
+            headers=get_headers(self._api_key),
+            params=params or {},
+            timeout=15,
+        )
+        if resp.status_code >= 400:
+            try:
+                detail = resp.json().get("error", resp.reason)
+            except ValueError:
+                detail = resp.reason
+            raise AgentXTracesError(f"Trace request failed ({resp.status_code}): {detail}")
+        return resp.json()
+
+    def get(self, trace_id: str) -> Dict[str, Any]:
+        """One trace's detail row. Raises on 404."""
+        return self._request(f"/ingest/traces/{trace_id}")
+
+    def list(
+        self,
+        limit: int = 50,
+        cursor: Optional[str] = None,
+        framework: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """A page of traces, newest first: ``{"traces": [...], "nextCursor": str | None}``.
+        Pass ``cursor`` from the previous page to continue."""
+        params: Dict[str, Any] = {"limit": limit}
+        if cursor:
+            params["cursor"] = cursor
+        if framework:
+            params["framework"] = framework
+        return self._request("/ingest/traces", params)

{agentx_python-0.6.32 → agentx_python-0.6.34}/agentx/tracing/tracer.py

@@ -153,6 +153,15 @@ class _TraceSpan:
         if exc_val is not None and self._error is None:
             self._error = str(exc_val)
 
+        if self._sync and self._parent_span_id is None:
+            # sync=True means the WHOLE tree is delivered before this block returns: child
+            # spans (tool calls, LLM calls) were enqueued asynchronously during the block, so
+            # drain them before the root's own synchronous send. Without this, read-after-trace
+            # intermittently misses children (root lands, children still in flight) - the exact
+            # race the enterprise assessment reproduced (P0.1). Bounded by the same 5s budget
+            # flush() uses; child-only spans keep their async fire-and-forget behavior.
+            self._tracer.flush(timeout=5.0)
+
         self._trace_id = self._tracer._send(
             sync=self._sync,
             monitor=self._monitor,
@@ -833,7 +842,10 @@ class Tracer:
         By default the trace is queued and sent on a background thread - fire-and-forget, never
         blocks the caller, but there's no way to learn the resulting trace_id. Pass ``sync=True``
         to send it synchronously instead (blocks until ingested) so ``span.trace_id`` is populated
-        once the ``with`` block exits - e.g. to attach the trace to an evaluation result::
+        once the ``with`` block exits. On a root span, ``sync=True`` covers the WHOLE tree: any
+        child spans recorded inside the block (tool calls, LLM calls) are drained before the
+        root is sent, so a read immediately after the block sees every span, not just the root.
+        Use it e.g. to attach the trace to an evaluation result::
 
             with client.tracer.trace("support_agent_call", framework="openai", sync=True) as span:
                 resp = call_llm(...)

agentx_python-0.6.34/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.6.34"

{agentx_python-0.6.32 → agentx_python-0.6.34}/agentx_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.32
+Version: 0.6.34
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.32 → agentx_python-0.6.34}/agentx_python.egg-info/SOURCES.txt

@@ -5,9 +5,12 @@ agentx/__init__.py
 agentx/agentx.py
 agentx/cli.py
 agentx/exceptions.py
+agentx/export.py
 agentx/feedback.py
 agentx/outcomes.py
+agentx/projects.py
 agentx/py.typed
+agentx/traces.py
 agentx/util.py
 agentx/version.py
 agentx/evaluations/__init__.py
@@ -47,6 +50,7 @@ agentx/monitor/models.py
 agentx/monitor/online_evaluators.py
 agentx/monitor/patterns.py
 agentx/monitor/profile.py
+agentx/monitor/scorers.py
 agentx/monitor/sessions.py
 agentx/monitor/signals.py
 agentx/resources/__init__.py

agentx_python-0.6.32/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.6.32"