agentx-python 0.8.13__py3-none-any.whl → 0.8.15__py3-none-any.whl

agentx/integrations/anthropic.py

@@ -13,7 +13,7 @@ Usage::
 
 Works with both ``anthropic.Anthropic`` and ``anthropic.AsyncAnthropic`` clients.
 
-Requires: ``pip install agentx[anthropic]``
+Requires: ``pip install "agentx-python[anthropic]"``
 """
 from __future__ import annotations
 

agentx/integrations/crewai.py

@@ -14,7 +14,7 @@ Or as a context manager around your own kickoff::
         result = crew.kickoff(inputs={"topic": "AI"})
         span.output = result.raw
 
-Requires: ``pip install agentx[crewai]``
+Requires: ``pip install "agentx-python[crewai]"``
 """
 from __future__ import annotations
 

agentx/monitor/client.py

@@ -67,6 +67,22 @@ class CalibrationSummary(dict):
     def review_label_count(self) -> int:
         return int(self.get("reviewLabelCount") or 0)
 
+    @property
+    def alpha(self):
+        """Chance-corrected agreement (Krippendorff's alpha over the binary verdict pair) -
+        the raw ``agreement_rate`` corrected for what a weighted coin would score on this
+        label mix. ``None`` below the server's sample floor (``alphaMinItems`` labeled pairs)
+        or when every label is identical: withheld, never fabricated. 1 = perfect, 0 = no
+        better than chance, negative = systematically opposed."""
+        return self.get("alpha")
+
+    @property
+    def alpha_band(self):
+        """Human-readable band for ``alpha`` (poor/slight/fair/moderate/substantial/
+        near-perfect), computed server-side so every surface reads the same alpha the
+        same way."""
+        return self.get("alphaBand")
+
 
 class MonitorClient:
     """Low-level HTTP client for the Monitor API (``/monitor``). Accessed via
@@ -131,6 +147,11 @@ class MonitorClient:
         # surface that matches the product; evaluations.settings and online_evaluators below
         # remain as its profile-level views.
         self.judge_scorers = JudgeScorersClient(api_key=api_key, base_url=self._api_root())
+        from agentx.monitor.improvement_groups import ImprovementGroupsClient
+
+        # Auto-improve: confirmed production failures -> improvement report -> code fix (via
+        # the AgentX-Eval-Skill auto-improve skill). Self-host only.
+        self.improvement_groups = ImprovementGroupsClient(api_key=api_key, base_url=self._api_root())
         self.profile = MonitorProfileClient(self)
         # Legacy view of an LLM Judge Scorer's online profile - constructed lazily so its
         # DeprecationWarning fires on first USE, not for every client that never touches it.
@@ -317,8 +338,12 @@ class MonitorClient:
         AgentX's own verdicts agreed with real-world ground truth reported later (ops outcomes
         via ``client.outcomes``, end-user downvotes, and human review labels). Returns the
         dashboard's Judge Calibration numbers with these exact keys: ``comparedCount``,
-        ``agreementRate``, ``falsePositiveRate``, ``falseNegativeRate`` (plus
-        ``reportedCount``/``reviewLabelCount``/``noVerdictCount``). Per-scorer calibration
+        ``agreementRate``, ``falsePositiveRate``, ``falseNegativeRate``, ``alpha``,
+        ``alphaBand``, ``alphaMinItems`` (plus
+        ``reportedCount``/``reviewLabelCount``/``noVerdictCount``). ``agreementRate`` is raw
+        agreement and inflates under class imbalance; ``alpha`` is the chance-corrected
+        version (Krippendorff's alpha - null until ``alphaMinItems`` labeled pairs exist).
+        Per-scorer calibration
         lives on ``client.monitor.judge_scorers.calibration(scorer_id)``."""
         return CalibrationSummary(
             self._request(

agentx/monitor/improvement_groups.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from agentx.util import api_base, get_headers
+
+
+class AgentXImprovementGroupsError(Exception):
+    pass
+
+
+class ImprovementGroupsClient:
+    """Surfaced as ``client.monitor.improvement_groups``: the auto-improve loop's accumulator.
+
+    Batch lifecycle: one COLLECTING group at a time. Every Confirm verdict in signal review
+    automatically lands the confirmed failure there - accumulation is free, declining is
+    choosing Ignore. ``generate_report`` SPENDS the batch: one LLM pass clusters the confirmed
+    failures into issues with recommendations, the group is sealed onto that report (keeping
+    exactly its source cases), and the pending accumulator is thereby cleared - the next
+    Confirm starts a fresh batch, and the next generate makes a new report from it. The report's id is the
+    hand-off: paste it into the AgentX-Eval-Skill ``auto-improve`` skill, which fetches the
+    report (``get_report``) and triages the fixes against your agent's actual source code.
+
+    Evidence here is exclusively ONLINE - production verdicts a human confirmed - never
+    offline dataset runs. Self-host only.
+    """
+
+    def __init__(self, api_key: Optional[str] = None, base_url: Optional[str] = None):
+        self._api_key = api_key
+        self._base_url = (base_url or api_base()).rstrip("/")
+
+    def _request(self, method: str, path: str, json: Any = None, timeout: int = 120) -> Any:
+        resp = requests.request(
+            method,
+            f"{self._base_url}/agent-monitoring{path}",
+            headers={**get_headers(self._api_key), "Content-Type": "application/json"},
+            json=json,
+            timeout=timeout,
+        )
+        if resp.status_code >= 400:
+            try:
+                detail = resp.json().get("error", resp.reason)
+            except ValueError:
+                detail = resp.reason
+            raise AgentXImprovementGroupsError(f"Improvement group request failed ({resp.status_code}): {detail}")
+        return resp.json() if resp.text else {}
+
+    def list(self) -> List[Dict[str, Any]]:
+        return self._request("GET", "/improvement-groups").get("improvementGroups", [])
+
+    def get(self, group_id: str) -> Dict[str, Any]:
+        """The group with its members - each a confirmed failure's evidence snapshot."""
+        return self._request("GET", f"/improvement-groups/{group_id}")["improvementGroup"]
+
+    def remove_member(self, group_id: str, member_id: str) -> None:
+        """Prune a member before spending the group (a confirm that turned out uninteresting)."""
+        self._request("DELETE", f"/improvement-groups/{group_id}/members/{member_id}")
+
+    def generate_report(self, group_id: str, model: Optional[str] = None) -> Dict[str, Any]:
+        """Spend the group: one real LLM call clustering the confirmed failures into issues
+        with recommendations. Returns the report; its ``_id`` is what the auto-improve skill
+        takes. Explicit and billed - never called implicitly."""
+        payload: Dict[str, Any] = {}
+        if model is not None:
+            payload["model"] = model
+        return self._request("POST", f"/improvement-groups/{group_id}/report", json=payload, timeout=300)["report"]
+
+    def list_reports(self) -> List[Dict[str, Any]]:
+        return self._request("GET", "/improvement-reports").get("improvementReports", [])
+
+    def get_report(self, report_id: str) -> Dict[str, Any]:
+        """Fetch a report by the id the dashboard (or generate_report) handed out - the exact
+        call the auto-improve skill makes."""
+        return self._request("GET", f"/improvement-reports/{report_id}")["report"]

agentx/monitor/judge_scorers.py

@@ -258,11 +258,19 @@ class JudgeScorersClient:
 
     def calibration(self, scorer_id: str, window: str = "7d") -> dict:
         """How this scorer's verdicts compare against recorded ground truth (triage
-        corrections, outcomes, end-user votes) over the window."""
+        corrections, outcomes, end-user votes) over the window. Beyond the raw
+        ``agreementRate``, the response carries ``alpha``/``alphaBand`` (chance-corrected
+        agreement - Krippendorff's alpha, null until ``alphaMinItems`` labeled pairs exist)
+        and ``ratingMae`` (mean absolute error against human re-scores, over the
+        ``withCorrectedScore`` pairs that carry a number). ``window`` accepts "24h", "7d",
+        "30d", or "rubric" - only verdicts produced by the CURRENT rubric (since its criteria
+        were last edited, clamped to 30 days), which is what the dashboard's Tune Judge flow
+        uses by default; the response's ``window``/``since`` echo the boundary applied."""
         return self._request("GET", f"/online-evaluators/{self._profile_id(scorer_id)}/calibration?window={window}")
 
     def tune(self, scorer_id: str, window: str = "7d") -> dict:
-        """Propose a rewrite of the rubric from calibration disagreements (LLM call, slow)."""
+        """Propose a rewrite of the rubric from calibration disagreements (LLM call, slow).
+        ``window`` accepts the same values as :meth:`calibration`, including "rubric"."""
         data = self._request(
             "POST", f"/online-evaluators/{self._profile_id(scorer_id)}/tune", json={"window": window}, timeout=300
         )

agentx/version.py

@@ -1,7 +1,7 @@
-VERSION = "0.8.13"
+VERSION = "0.8.15"
 
 # The AgentX-trace-eval release this SDK version is tested against - what `agentx-trace-eval`
 # installs and converges to (see agentx/cli.py). Bump together with VERSION when releasing, so
 # every published SDK names a known-good engine+dashboard pair. Users can override with
 # AGENTX_TRACE_EVAL_VERSION=<tag|latest>.
-ENGINE_VERSION = "v0.3.7"
+ENGINE_VERSION = "v0.3.11"

{agentx_python-0.8.13.dist-info → agentx_python-0.8.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.8.13
+Version: 0.8.15
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team
@@ -8,7 +8,7 @@ Author-email: contact@agentx.so
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.6
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: urllib3>=1.26.11
@@ -50,6 +50,7 @@ Requires-Dist: litellm>=1.0.0; extra == "all"
 Requires-Dist: llama-index-core>=0.10.0; extra == "all"
 Requires-Dist: autogen-agentchat>=0.4.0; extra == "all"
 Requires-Dist: autogen-core>=0.4.0; extra == "all"
+Requires-Dist: mlflow>=3.6.0; extra == "all"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
@@ -109,12 +110,14 @@ pip install --upgrade agentx-python
 
 Requires Python 3.9 or newer.
 
-#### Run self host eval framework locally
+#### Run the self-host governance suite locally
 
-```
-agentx-trace-eval --dev --update
+```bash
+agentx-trace-eval --dev
 ```
 
+(See [Self-host](#self-host) below for what this downloads and how to point the SDK at it.)
+
 ---
 
 ## Quick start
@@ -137,7 +140,7 @@ report = (
     .analyze()
 )
 
-print(report.average_rating)   # LLM-graded score, 0–10
+print(report.average_rating)   # LLM-graded score, 0-10
 print(report.summary)          # AI-generated narrative from .analyze()
 ```
 
@@ -158,9 +161,9 @@ report = (
     .analyze()
 )
 
-print(report.average_rating)       # LLM-graded score, 0–10
-print(report.cosine_similarity)    # embedding cosine, 0–1 (None if not enabled)
-print(report.jaccard_similarity)   # token-set overlap, 0–1 (None if not enabled)
+print(report.average_rating)       # LLM-graded score, 0-10
+print(report.cosine_similarity)    # embedding cosine, 0-1 (None if not enabled)
+print(report.jaccard_similarity)   # token-set overlap, 0-1 (None if not enabled)
 
 print(report.summary)              # AI-generated narrative from .analyze()
 print(report.recommendations)      # list of prioritized, actionable fixes
@@ -184,7 +187,7 @@ client.evaluations.run(
 
 See [Prompt registry](EVALUATIONS.md#prompt-registry) in the full guide, or [self-host's docs](https://docs.agentx.so/improve/prompt-management) for the "Suggest improvement" dashboard flow (self-host only - no hosted-SaaS equivalent yet).
 
-On self-host, a finalized run can also **gate a CI job**: `run.gate(fail_under=7, no_regression=True)` (on the run context `.execute()` returns) checks the run's average rating against an absolute floor and/or the dataset's previous run, prints per-check verdicts into the CI log, and returns an exit code - `sys.exit(gate.exit_code)` blocks the merge on regression. Recorded gates appear in the dashboard's CI Gates tab. See [self-host's CI docs](https://docs.agentx.so/integrations/self-host-ci) for the GitHub Actions recipe.
+On self-host, a finalized run can also **gate a CI job**: `run.gate(fail_under=7, no_regression=True)` (on the run context `.execute()` returns) checks the run's average rating against an absolute floor and/or the dataset's previous run, prints per-check verdicts into the CI log, and returns a `GateResult` - `sys.exit(gate.exit_code)` blocks the merge on regression. Recorded gates appear in the dashboard's CI Gates tab. See [self-host's CI docs](https://docs.agentx.so/integrations/self-host-ci) for the GitHub Actions recipe.
 
 See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide - dataset builder, framework adapters, similarity metrics, smoke testing, judge configuration, prompt registry, and the complete API reference.
 
@@ -252,6 +255,10 @@ extra:
 | LlamaIndex            | `pip install "agentx-python[llamaindex]"`    | `AgentXLlamaIndexHandler` |
 | AutoGen               | `pip install "agentx-python[autogen]"`       | `AgentXAutoGenObserver`   |
 
+Two more platforms are covered by **pull importers** rather than in-process hooks, each with its
+own CLI: `agentx-moveworks` (Moveworks Data API sync, no extra needed) and `agentx-databricks`
+(`pip install "agentx-python[databricks]"`, MLflow/Databricks trace sync).
+
 Or plain Python - wrap any function with `@tracer.trace(...)` and it just works, no framework
 required. Tracing is **platform agnostic**: each integration stamps its platform label
 automatically, a plain trace auto-detects the one orchestration framework imported in the
@@ -292,12 +299,14 @@ for signal in client.monitor.signals.list(severity="high"):
     print(signal.summary, signal.occurrence_count)
 ```
 
-Per-agent coverage/threshold settings (sample rate, retention, and threshold overrides like the built-in "Latency regression" pattern's threshold) are `client.monitor.profile.get()`/`.update()`:
+Per-agent monitoring settings (enable/disable, detection categories, notification channels) are `client.monitor.profile.get()`/`.update()`:
 
 ```python
-client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000})
+client.monitor.profile.update("agent_123", info_detection_enabled=False)
 ```
 
+On self-host, coverage mode, sample rate, retention, and the built-in latency threshold are project-level defaults set in the dashboard's Platform Settings; `update()` still accepts them for wire compatibility, but only the per-agent fields above take effect there.
+
 Self-host also has **online evaluators**: a real LLM judge scoring a sample of live traffic continuously, distinct from a pattern's rule-matching. A score below `alert_threshold` raises a signal the same way a failing pattern does, deduped and triage-ready in `client.monitor.signals`.
 
 ```python
@@ -371,7 +380,7 @@ export AGENTX_API_BASE_URL=http://localhost:4700/api/v1
 export AGENTX_API_KEY=<printed by agentx-trace-eval on first run>
 ```
 
-`agentx-trace-eval` isn't this SDK's own code - the engine itself is a separate, compiled binary, downloaded on demand rather than bundled into this package, so installing `agentx-python` doesn't get any heavier for the (much more common) case of just talking to the hosted AgentX API. See that repo's README for what's included, and `AGENTX_INSTALL_DIR`/`AGENTX_TRACE_EVAL_VERSION`/`AGENTX_TRACE_EVAL_SKIP_WEB` env vars to control where/what it installs.
+`agentx-trace-eval` isn't this SDK's own code - the engine itself is a separate, compiled binary, downloaded on demand rather than bundled into this package, so installing `agentx-python` doesn't get any heavier for the (much more common) case of just talking to the hosted AgentX API. Each SDK release pins the engine release it was tested against and converges the install to that pin, so upgrading the SDK upgrades the engine too. Two flags to know: `--update` (consumed by this launcher) force-reinstalls the resolved engine release, while `--upgrade` passes through to `agentx-server` and re-downloads the dashboard bundle before serving. See that repo's README for what's included, and the `AGENTX_INSTALL_DIR`/`AGENTX_TRACE_EVAL_VERSION`/`AGENTX_TRACE_EVAL_SKIP_WEB` env vars to control where/what it installs.
 
 ---
 

{agentx_python-0.8.13.dist-info → agentx_python-0.8.15.dist-info}/RECORD

@@ -10,7 +10,7 @@ agentx/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 agentx/testing.py,sha256=0shZEid_vhJgBegpO_loUq6APYDxQrUgD3jvcRdIDV8,7372
 agentx/traces.py,sha256=sz9gxutlDKNgf0fdCkyzKsGwK5eMAJuIEl3NIwGrsuo,2287
 agentx/util.py,sha256=lt2Kpg4Fj8whrSuAhLEahcPEIYRVpK9RCC5y9ShdTJs,703
-agentx/version.py,sha256=h93nyZEahmg-F_qXioTJhKMFmXgzcYA3exV9z2on3LQ,365
+agentx/version.py,sha256=Z07Xy3YcMuznG9WwxLmXoO9vPta0vt4xMz047BZIZqY,366
 agentx/evaluations/__init__.py,sha256=Erv7RGFlRxGTG4rVb2uHhCqLLEX_iAWKqkZKzK6CumE,262
 agentx/evaluations/_term.py,sha256=WFpiNzdgDBeJJ-Gg-6X7TwwxllobuE4OqFUTuDQvS3Y,2529
 agentx/evaluations/client.py,sha256=UlYl8OFmIjk8oqmL_LEpjnTn6CmUPBM1kgVgzkUKfT8,31999
@@ -29,9 +29,9 @@ agentx/evaluations/adapters/precomputed.py,sha256=vxnQELmpiU2NkQATfiVjG7v1ZmmkrE
 agentx/evaluations/adapters/raw.py,sha256=FkDq_mdf21-xt5EHnGyIGcS6VZxkEkFcmM9VTd-T5sA,1130
 agentx/integrations/__init__.py,sha256=W-EcQhHE01-SGPeKTB2AgfvUyRzNP4BqUMMtP2qQ4Q0,647
 agentx/integrations/_traced_call.py,sha256=O6f8HKRHDfFwoUIZDtvENVsv_OsHwzuyPMbzBakKVdo,6662
-agentx/integrations/anthropic.py,sha256=HNFiwtljfWsAJOmc2XXseKSVb8ThUr6BZ6CWT7uGv18,10383
+agentx/integrations/anthropic.py,sha256=Fdt1bVBKYuhUa8S4nsZLNqg4REUzaZsh-7RXEWuJ-0g,10392
 agentx/integrations/autogen.py,sha256=cZryUl45Sdr9t1ba4tkz45XfkvjCud1hPp6pV0auKy8,8863
-agentx/integrations/crewai.py,sha256=AMxAP2Rpdojf2iaMVNcs8_LhnUGJnsS1NaX-w-1Gicc,10703
+agentx/integrations/crewai.py,sha256=Ofid_YvIFYFoN12_gA3PjKAhlJgPNUh9sNyIbBcRkZs,10712
 agentx/integrations/databricks.py,sha256=vKXnur-LWMTzctFuIKte2N826sydXVlpi1qFXo2dfio,18044
 agentx/integrations/google_adk.py,sha256=swLpAfkuL2m1VZUSf_3ImwsrcTSxVdS5EIKRu2Tf4R4,12476
 agentx/integrations/google_genai.py,sha256=2patIdgyjEEW70FLP7ng4hcnFY-p0a1jPi1Pof28KBo,13272
@@ -43,8 +43,9 @@ agentx/integrations/openai.py,sha256=1KLs-hJaeW2emHPwNkmC3zcGksL3zd87uWbcLGRW704
 agentx/integrations/openai_agents.py,sha256=tCa3kPxqux5LnNNDQ6zA05OybQrjkc_LeE08YuNPJEs,12742
 agentx/monitor/__init__.py,sha256=MLiASSvQnCy2jqVNRTgPWFhUSf8lGYRQeEgEVVhE5Tg,574
 agentx/monitor/agents.py,sha256=8Xk4jWmNTvdtjFixHiLmw4jj96uqIXWIT6KW8zXoWrc,1006
-agentx/monitor/client.py,sha256=nY1cvjbCqL6BZMY6ubM-Q6JUKGgUM5CdBektV99EArQ,20812
-agentx/monitor/judge_scorers.py,sha256=Mu6OOnfjp1SRkZMcOyriO7zPawv-O2H2l2X-0io5WLM,15025
+agentx/monitor/client.py,sha256=FGrP8CrzwX7i_F4qRNypdERgmCcCIwsOy1WUpsfFBn4,22191
+agentx/monitor/improvement_groups.py,sha256=-6-FcJnyiTLqeh9z8D9wyFD1QFqEsdDdDZdDJhC6QXw,3712
+agentx/monitor/judge_scorers.py,sha256=Zmi6knBAEN344Vz6jK9c6KcacTvWz_GsmabNl08hz5E,15756
 agentx/monitor/models.py,sha256=yTC3WTdTziHMkdeAXlTro-4YiSbBUHMgf9kNZHGfPDA,8470
 agentx/monitor/online_evaluators.py,sha256=YWnsBmlIbYvj4LCaX4guwx4HS5y3oEzCQfo6ae9VH_w,8693
 agentx/monitor/patterns.py,sha256=ruBqLoA2T2KQuqRbbn1kWkIEkiOal3SvyPl9opGyhzI,4185
@@ -64,9 +65,9 @@ agentx/tracing/eval_scope.py,sha256=ElMbPxpqpQBVuaUnHNlw9RIQu71oyOpd0yR55bDH8eI,
 agentx/tracing/framework_detect.py,sha256=uV4O7Th-4_2UkdooAyaWCOIA0jwLJbblFQ_ucFDjeJM,2638
 agentx/tracing/ingest_client.py,sha256=spnm6bV3NXGuj2w7HwdKiJdKDJWRUFmSP3jl3rcGqkQ,18202
 agentx/tracing/tracer.py,sha256=TxY6zG6hbM81jpfrqP4imaEvoyy_hnJVUSTnoAVHMG0,56240
-agentx_python-0.8.13.dist-info/licenses/LICENSE,sha256=gZVsM-nLsE8vlaY6NXXsVoo6IlCClxkToAWmhgT3y_s,10762
-agentx_python-0.8.13.dist-info/METADATA,sha256=cit9iBc_TXXOlcp8AObpzlXdbx3FLuaVTbBsBdggcJ8,21154
-agentx_python-0.8.13.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-agentx_python-0.8.13.dist-info/entry_points.txt,sha256=rQqF1JTY3T1yfviBU1r8PU-5mBdfOk6P4bZ1L4BskIM,172
-agentx_python-0.8.13.dist-info/top_level.txt,sha256=s-q-HB9Gb_QdrZNacSeQyF_c25gQooMy7DlxzgLOHPk,7
-agentx_python-0.8.13.dist-info/RECORD,,
+agentx_python-0.8.15.dist-info/licenses/LICENSE,sha256=gZVsM-nLsE8vlaY6NXXsVoo6IlCClxkToAWmhgT3y_s,10762
+agentx_python-0.8.15.dist-info/METADATA,sha256=G4jRUL5ff535TmaGfJYm7cs8pFxmDlHhwBBcuSpycZc,22117
+agentx_python-0.8.15.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+agentx_python-0.8.15.dist-info/entry_points.txt,sha256=rQqF1JTY3T1yfviBU1r8PU-5mBdfOk6P4bZ1L4BskIM,172
+agentx_python-0.8.15.dist-info/top_level.txt,sha256=s-q-HB9Gb_QdrZNacSeQyF_c25gQooMy7DlxzgLOHPk,7
+agentx_python-0.8.15.dist-info/RECORD,,