margin 0.10.0__tar.gz → 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of margin might be problematic. Click here for more details.
- {margin-0.10.0/margin.egg-info → margin-0.11.0}/PKG-INFO +26 -1
- {margin-0.10.0 → margin-0.11.0}/README.md +23 -0
- margin-0.11.0/adapters/__init__.py +0 -0
- margin-0.11.0/adapters/agent/__init__.py +34 -0
- margin-0.11.0/adapters/agent/flow.py +72 -0
- margin-0.11.0/adapters/agent/metrics.py +78 -0
- margin-0.11.0/adapters/agent/wrap.py +114 -0
- margin-0.11.0/adapters/dataframe/__init__.py +7 -0
- margin-0.11.0/adapters/dataframe/pandas_hook.py +99 -0
- margin-0.11.0/adapters/dataframe/quality.py +110 -0
- margin-0.11.0/adapters/fastapi/__init__.py +6 -0
- margin-0.11.0/adapters/fastapi/endpoints.py +99 -0
- margin-0.11.0/adapters/fastapi/middleware.py +116 -0
- margin-0.11.0/adapters/mcp/__init__.py +50 -0
- margin-0.11.0/adapters/mcp/__main__.py +5 -0
- margin-0.11.0/adapters/mcp/server.py +177 -0
- margin-0.11.0/adapters/mcp/state.py +96 -0
- margin-0.11.0/adapters/numpy/__init__.py +9 -0
- margin-0.11.0/adapters/numpy/array_health.py +220 -0
- margin-0.11.0/adapters/pytest/__init__.py +7 -0
- margin-0.11.0/adapters/pytest/plugin.py +255 -0
- margin-0.11.0/adapters/pytest/suite.py +104 -0
- {margin-0.10.0 → margin-0.11.0/margin.egg-info}/PKG-INFO +26 -1
- {margin-0.10.0 → margin-0.11.0}/margin.egg-info/SOURCES.txt +22 -0
- margin-0.11.0/margin.egg-info/entry_points.txt +2 -0
- margin-0.11.0/margin.egg-info/requires.txt +3 -0
- margin-0.11.0/margin.egg-info/top_level.txt +2 -0
- {margin-0.10.0 → margin-0.11.0}/pyproject.toml +9 -2
- margin-0.10.0/margin.egg-info/top_level.txt +0 -1
- {margin-0.10.0 → margin-0.11.0}/LICENSE +0 -0
- {margin-0.10.0 → margin-0.11.0}/NOTICE +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/__init__.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/__main__.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/algebra.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/anomaly.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/bridge.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/calibrate.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/causal.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/composite.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/confidence.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/config.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/contract.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/controller.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/correlate.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/diff.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/drift.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/events.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/fingerprint.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/flow.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/forecast.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/health.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/intent.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/issue.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/ledger.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/loop.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/observation.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/persist.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/policy/__init__.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/policy/compose.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/policy/core.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/policy/temporal.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/policy/trace.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/policy/tuning.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/policy/validate.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/predicates.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/provenance.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/streaming.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/transitions.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/uncertain.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin/validity.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/margin.egg-info/dependency_links.txt +0 -0
- {margin-0.10.0 → margin-0.11.0}/setup.cfg +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_absence.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_algebra.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_anomaly.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_bridge.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_calibrate.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_causal.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_composite.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_confidence.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_config.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_contract.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_controller.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_correlate.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_diff.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_drift.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_events.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_fingerprint_ext.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_float_tolerance.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_flow.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_forecast.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_health.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_intent.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_issue.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_ledger.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_loop.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_observation.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_persist.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_pipeline_features.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_policy.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_policy_compose.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_policy_temporal.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_policy_trace.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_policy_tuning.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_policy_validate.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_predicates.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_provenance.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_reverse_bridge.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_staleness.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_streaming.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_transitions.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_uncertain.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_validity.py +0 -0
- {margin-0.10.0 → margin-0.11.0}/tests/test_windowing.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: margin
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: Typed health classification, uncertainty algebra, and correction auditing for any system that measures things and needs to explain what happened.
|
|
5
5
|
Author: Cope Labs LLC
|
|
6
6
|
License: MIT
|
|
@@ -25,6 +25,8 @@ Requires-Python: >=3.10
|
|
|
25
25
|
Description-Content-Type: text/markdown
|
|
26
26
|
License-File: LICENSE
|
|
27
27
|
License-File: NOTICE
|
|
28
|
+
Provides-Extra: mcp
|
|
29
|
+
Requires-Dist: mcp>=1.0; extra == "mcp"
|
|
28
30
|
Dynamic: license-file
|
|
29
31
|
|
|
30
32
|
# margin
|
|
@@ -62,6 +64,28 @@ print(guidance.to_atom())
|
|
|
62
64
|
|
|
63
65
|
Every metric is polarity-aware (more tokens = worse, higher confidence = better), sigma-normalised so you can compare them, and emits typed diagnostics when something goes wrong. Nothing stringly-typed; your decision loop reads typed enums.
|
|
64
66
|
|
|
67
|
+
### Works with any MCP-capable client
|
|
68
|
+
|
|
69
|
+
Install the MCP extra and you get an stdio server any model's client (Claude Desktop, Claude Code, Cursor, Cline, Windsurf, …) can spawn:
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
pip install margin[mcp]
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
```json
|
|
76
|
+
{
|
|
77
|
+
"mcpServers": {
|
|
78
|
+
"margin": {
|
|
79
|
+
"command": "margin-mcp"
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
The model now has a `margin_observe(values)` tool and can read `margin://state` / `margin://issues` between turns to reason about its own typed self-state. Full details: [adapters/mcp/README.md](adapters/mcp/README.md).
|
|
86
|
+
|
|
87
|
+
---
|
|
88
|
+
|
|
65
89
|
Not building an agent? Margin works on any scalar with a threshold:
|
|
66
90
|
|
|
67
91
|
```python
|
|
@@ -235,6 +259,7 @@ Core adapters ship in the package. Each is a Parser factory + optional Flow / In
|
|
|
235
259
|
| Adapter | What it monitors |
|
|
236
260
|
|---|---|
|
|
237
261
|
| [**agent**](https://github.com/Cope-Labs/margin/tree/main/adapters/agent/) | LLM / agent step health — latency, tokens, tool success rate, retries, confidence. Includes a three-stage `plan → act → reflect` Flow and a default feasibility Intent. |
|
|
262
|
+
| [**mcp**](https://github.com/Cope-Labs/margin/tree/main/adapters/mcp/) | stdio Model Context Protocol server. Exposes `margin_observe` / `margin_reset` tools and `margin://state` / `margin://issues` resources to any MCP client. `pip install margin[mcp]`. |
|
|
238
263
|
| [**dataframe**](https://github.com/Cope-Labs/margin/tree/main/adapters/dataframe/) | Data quality (completeness, null rate, drift, freshness, schema) |
|
|
239
264
|
| [**numpy**](https://github.com/Cope-Labs/margin/tree/main/adapters/numpy/) | Array health (NaN rate, drift, distribution shape) |
|
|
240
265
|
| [**fastapi**](https://github.com/Cope-Labs/margin/tree/main/adapters/fastapi/) | Endpoint health (latency, error rate, throughput, queue depth) — ASGI middleware |
|
|
@@ -33,6 +33,28 @@ print(guidance.to_atom())
|
|
|
33
33
|
|
|
34
34
|
Every metric is polarity-aware (more tokens = worse, higher confidence = better), sigma-normalised so you can compare them, and emits typed diagnostics when something goes wrong. Nothing stringly-typed; your decision loop reads typed enums.
|
|
35
35
|
|
|
36
|
+
### Works with any MCP-capable client
|
|
37
|
+
|
|
38
|
+
Install the MCP extra and you get an stdio server any model's client (Claude Desktop, Claude Code, Cursor, Cline, Windsurf, …) can spawn:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install margin[mcp]
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
```json
|
|
45
|
+
{
|
|
46
|
+
"mcpServers": {
|
|
47
|
+
"margin": {
|
|
48
|
+
"command": "margin-mcp"
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
The model now has a `margin_observe(values)` tool and can read `margin://state` / `margin://issues` between turns to reason about its own typed self-state. Full details: [adapters/mcp/README.md](adapters/mcp/README.md).
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
36
58
|
Not building an agent? Margin works on any scalar with a threshold:
|
|
37
59
|
|
|
38
60
|
```python
|
|
@@ -206,6 +228,7 @@ Core adapters ship in the package. Each is a Parser factory + optional Flow / In
|
|
|
206
228
|
| Adapter | What it monitors |
|
|
207
229
|
|---|---|
|
|
208
230
|
| [**agent**](https://github.com/Cope-Labs/margin/tree/main/adapters/agent/) | LLM / agent step health — latency, tokens, tool success rate, retries, confidence. Includes a three-stage `plan → act → reflect` Flow and a default feasibility Intent. |
|
|
231
|
+
| [**mcp**](https://github.com/Cope-Labs/margin/tree/main/adapters/mcp/) | stdio Model Context Protocol server. Exposes `margin_observe` / `margin_reset` tools and `margin://state` / `margin://issues` resources to any MCP client. `pip install margin[mcp]`. |
|
|
209
232
|
| [**dataframe**](https://github.com/Cope-Labs/margin/tree/main/adapters/dataframe/) | Data quality (completeness, null rate, drift, freshness, schema) |
|
|
210
233
|
| [**numpy**](https://github.com/Cope-Labs/margin/tree/main/adapters/numpy/) | Array health (NaN rate, drift, distribution shape) |
|
|
211
234
|
| [**fastapi**](https://github.com/Cope-Labs/margin/tree/main/adapters/fastapi/) | Endpoint health (latency, error rate, throughput, queue depth) — ASGI middleware |
|
|
File without changes
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""
|
|
2
|
+
margin adapter for agent / LLM-pipeline self-awareness.
|
|
3
|
+
|
|
4
|
+
The primitives (Confidence, Absence, Issue, Flow, Intent, Contract) that margin
|
|
5
|
+
exposes are *exactly* the vocabulary an LLM agent needs when it has to answer
|
|
6
|
+
"how am I doing, and should I continue?" at each step of its run.
|
|
7
|
+
|
|
8
|
+
This adapter packages sensible defaults so you can go from zero to a typed
|
|
9
|
+
health read-out of an agent step in three lines:
|
|
10
|
+
|
|
11
|
+
from adapters.agent import agent_parser, AgentStep
|
|
12
|
+
|
|
13
|
+
parser = agent_parser()
|
|
14
|
+
step = AgentStep(latency_ms=2400, tokens_used=1800,
|
|
15
|
+
tool_success_rate=0.9, retry_count=1, confidence=0.72)
|
|
16
|
+
print(parser.parse(step.to_values()).to_string())
|
|
17
|
+
# [latency_ms:INTACT(+0.40\u03c3)] [tokens_used:INTACT(+0.75\u03c3)]
|
|
18
|
+
# [tool_success_rate:INTACT(-0.05\u03c3)] [retry_count:DEGRADED(...)]
|
|
19
|
+
# [confidence:INTACT(-0.10\u03c3)]
|
|
20
|
+
|
|
21
|
+
Use :func:`agent_flow` when your agent runs in stages (plan \u2192 act \u2192 reflect) and
|
|
22
|
+
you want typed live guidance (ADVANCE/HOLD/RETRY/ROLLBACK) between them.
|
|
23
|
+
|
|
24
|
+
Zero SDK dependencies. See module docstrings for wiring to Anthropic / OpenAI.
|
|
25
|
+
"""
|
|
26
|
+
from .metrics import AGENT_BASELINES, AGENT_THRESHOLDS, agent_parser
|
|
27
|
+
from .flow import AGENT_STAGES, agent_flow
|
|
28
|
+
from .wrap import AgentStep, record_step, agent_intent
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"AGENT_BASELINES", "AGENT_THRESHOLDS", "agent_parser",
|
|
32
|
+
"AGENT_STAGES", "agent_flow",
|
|
33
|
+
"AgentStep", "record_step", "agent_intent",
|
|
34
|
+
]
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""
|
|
2
|
+
A Flow template for a typical three-stage agent loop:
|
|
3
|
+
|
|
4
|
+
PLAN \u2192 ACT \u2192 REFLECT \u2192 (COMPLETE)
|
|
5
|
+
|
|
6
|
+
Each stage uses the same agent_parser, but advances on different predicates:
|
|
7
|
+
|
|
8
|
+
PLAN \u2192 a concrete plan is produced (confidence above threshold)
|
|
9
|
+
ACT \u2192 tool calls are succeeding (tool_success_rate holds)
|
|
10
|
+
REFLECT \u2192 the run is converging (retries haven't exploded)
|
|
11
|
+
|
|
12
|
+
Flow.observe() returns typed live guidance (ADVANCE / HOLD / RETRY / ROLLBACK /
|
|
13
|
+
ESCALATE) at every step, and emits typed Issues for predicate failures and
|
|
14
|
+
degraded stage entries. This means your agent gets a typed answer to
|
|
15
|
+
"continue, retry, or escalate?" after every observation \u2014 not a string you have
|
|
16
|
+
to parse.
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from margin import Flow, Stage, Expression
|
|
21
|
+
|
|
22
|
+
from .metrics import agent_parser
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _has_plan(expr: Expression) -> bool:
|
|
26
|
+
"""PLAN advances once the model's stated confidence crosses intact."""
|
|
27
|
+
conf = next((o for o in expr.observations if o.name == "confidence"), None)
|
|
28
|
+
return conf is not None and conf.health.value == "INTACT"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _tools_holding(expr: Expression) -> bool:
|
|
32
|
+
"""ACT advances once tool success rate is intact and retries aren't ablated."""
|
|
33
|
+
success = next((o for o in expr.observations if o.name == "tool_success_rate"), None)
|
|
34
|
+
retries = next((o for o in expr.observations if o.name == "retry_count"), None)
|
|
35
|
+
if success is None or success.health.value != "INTACT":
|
|
36
|
+
return False
|
|
37
|
+
if retries is not None and retries.health.value == "ABLATED":
|
|
38
|
+
return False
|
|
39
|
+
return True
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _converging(expr: Expression) -> bool:
|
|
43
|
+
"""REFLECT advances once retry_count is intact and confidence is intact."""
|
|
44
|
+
retries = next((o for o in expr.observations if o.name == "retry_count"), None)
|
|
45
|
+
conf = next((o for o in expr.observations if o.name == "confidence"), None)
|
|
46
|
+
return (
|
|
47
|
+
retries is not None and retries.health.value == "INTACT"
|
|
48
|
+
and conf is not None and conf.health.value == "INTACT"
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
AGENT_STAGES = ("plan", "act", "reflect")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def agent_flow(label: str = "agent") -> Flow:
|
|
56
|
+
"""
|
|
57
|
+
Build a typical three-stage agent Flow with shared thresholds.
|
|
58
|
+
|
|
59
|
+
Each Flow.observe() call returns a FlowResult whose ``guidance`` is one of
|
|
60
|
+
ADVANCE / HOLD / RETRY / ROLLBACK / ESCALATE / COMPLETE. Use
|
|
61
|
+
``result.to_atom()`` to get a single-line summary suitable for logging,
|
|
62
|
+
e.g. ``flow:agent:act[2/3] \u2192 ADVANCE``.
|
|
63
|
+
"""
|
|
64
|
+
parser = agent_parser()
|
|
65
|
+
return Flow(
|
|
66
|
+
stages=[
|
|
67
|
+
Stage(name="plan", parser=parser, advance_when=_has_plan),
|
|
68
|
+
Stage(name="act", parser=parser, advance_when=_tools_holding),
|
|
69
|
+
Stage(name="reflect", parser=parser, advance_when=_converging),
|
|
70
|
+
],
|
|
71
|
+
label=label,
|
|
72
|
+
)
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Default baselines and thresholds for common agent / LLM-pipeline metrics.
|
|
3
|
+
|
|
4
|
+
Polarity is the critical part: for cost-like metrics (latency, tokens, retries)
|
|
5
|
+
more is worse, so ``higher_is_better=False``. For quality metrics (confidence,
|
|
6
|
+
tool success rate) more is better. Margin's sigma normalisation threads that
|
|
7
|
+
polarity correctly through every comparison downstream.
|
|
8
|
+
|
|
9
|
+
The baselines below are deliberately conservative defaults for a mid-size
|
|
10
|
+
Claude / GPT call with occasional tool use. Override per-component for your
|
|
11
|
+
specific agent.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from typing import Optional
|
|
16
|
+
|
|
17
|
+
from margin import Parser, Thresholds
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
AGENT_BASELINES: dict[str, float] = {
|
|
21
|
+
"latency_ms": 1500.0, # typical end-to-end call
|
|
22
|
+
"tokens_used": 800.0, # input + output
|
|
23
|
+
"tool_success_rate": 0.95, # fraction of tool calls that succeeded
|
|
24
|
+
"retry_count": 0.0, # zero retries is the happy path
|
|
25
|
+
"confidence": 0.80, # model-stated or rubric-derived
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
AGENT_THRESHOLDS: dict[str, Thresholds] = {
|
|
30
|
+
"latency_ms": Thresholds(
|
|
31
|
+
intact=3000.0, ablated=10000.0, higher_is_better=False,
|
|
32
|
+
),
|
|
33
|
+
"tokens_used": Thresholds(
|
|
34
|
+
intact=3000.0, ablated=10000.0, higher_is_better=False,
|
|
35
|
+
),
|
|
36
|
+
"tool_success_rate": Thresholds(
|
|
37
|
+
intact=0.80, ablated=0.50, higher_is_better=True,
|
|
38
|
+
),
|
|
39
|
+
"retry_count": Thresholds(
|
|
40
|
+
intact=1.0, ablated=3.0, higher_is_better=False,
|
|
41
|
+
),
|
|
42
|
+
"confidence": Thresholds(
|
|
43
|
+
intact=0.60, ablated=0.30, higher_is_better=True,
|
|
44
|
+
),
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def agent_parser(
|
|
49
|
+
baselines: Optional[dict[str, float]] = None,
|
|
50
|
+
thresholds: Optional[dict[str, Thresholds]] = None,
|
|
51
|
+
) -> Parser:
|
|
52
|
+
"""
|
|
53
|
+
Build a Parser with sensible agent-metric defaults.
|
|
54
|
+
|
|
55
|
+
``baselines`` and ``thresholds`` merge over the defaults, so you can pass
|
|
56
|
+
just the overrides you care about::
|
|
57
|
+
|
|
58
|
+
parser = agent_parser(
|
|
59
|
+
baselines={"latency_ms": 5000.0}, # slower model
|
|
60
|
+
thresholds={"confidence": Thresholds(
|
|
61
|
+
intact=0.75, ablated=0.50, higher_is_better=True,
|
|
62
|
+
)},
|
|
63
|
+
)
|
|
64
|
+
"""
|
|
65
|
+
merged_baselines = dict(AGENT_BASELINES)
|
|
66
|
+
if baselines:
|
|
67
|
+
merged_baselines.update(baselines)
|
|
68
|
+
merged_thresholds = dict(AGENT_THRESHOLDS)
|
|
69
|
+
if thresholds:
|
|
70
|
+
merged_thresholds.update(thresholds)
|
|
71
|
+
# Default per-component thresholds cover every baseline, so the Parser-level
|
|
72
|
+
# fallback is only hit if the caller adds an unrelated component.
|
|
73
|
+
fallback = Thresholds(intact=0.0, ablated=0.0)
|
|
74
|
+
return Parser(
|
|
75
|
+
baselines=merged_baselines,
|
|
76
|
+
thresholds=fallback,
|
|
77
|
+
component_thresholds=merged_thresholds,
|
|
78
|
+
)
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Lightweight helpers for recording an agent step and deriving a feasibility
|
|
3
|
+
Intent from it.
|
|
4
|
+
|
|
5
|
+
The ``AgentStep`` dataclass carries the five canonical agent metrics so you can
|
|
6
|
+
collect them once at the call site and feed them to margin in typed form.
|
|
7
|
+
There is no hard dependency on any LLM SDK \u2014 wire your own adapter in a couple
|
|
8
|
+
of lines, e.g. for Anthropic::
|
|
9
|
+
|
|
10
|
+
import time, anthropic
|
|
11
|
+
from adapters.agent import AgentStep, agent_parser
|
|
12
|
+
|
|
13
|
+
client = anthropic.Anthropic()
|
|
14
|
+
parser = agent_parser()
|
|
15
|
+
|
|
16
|
+
def call(messages, tools=None):
|
|
17
|
+
t0 = time.perf_counter()
|
|
18
|
+
resp = client.messages.create(
|
|
19
|
+
model="claude-opus-4-7",
|
|
20
|
+
max_tokens=1024,
|
|
21
|
+
messages=messages,
|
|
22
|
+
tools=tools or [],
|
|
23
|
+
)
|
|
24
|
+
step = AgentStep(
|
|
25
|
+
latency_ms=(time.perf_counter() - t0) * 1000,
|
|
26
|
+
tokens_used=resp.usage.input_tokens + resp.usage.output_tokens,
|
|
27
|
+
tool_success_rate=1.0, # fill in from your tool-runner's accounting
|
|
28
|
+
retry_count=0,
|
|
29
|
+
confidence=0.8, # rubric-derived or model-stated
|
|
30
|
+
)
|
|
31
|
+
expr = parser.parse(step.to_values())
|
|
32
|
+
return resp, expr
|
|
33
|
+
"""
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
from dataclasses import dataclass
|
|
37
|
+
from typing import Optional
|
|
38
|
+
|
|
39
|
+
from margin import (
|
|
40
|
+
Expression, Intent, Requirement, Health, Ledger, Record, Op,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass
|
|
45
|
+
class AgentStep:
|
|
46
|
+
"""
|
|
47
|
+
One agent step's observable metrics.
|
|
48
|
+
|
|
49
|
+
latency_ms: end-to-end wall time for the call in milliseconds
|
|
50
|
+
tokens_used: input + output tokens
|
|
51
|
+
tool_success_rate: fraction of tool calls that succeeded (0.0\u20131.0)
|
|
52
|
+
retry_count: how many retries this step consumed
|
|
53
|
+
confidence: model-stated or rubric-derived confidence (0.0\u20131.0)
|
|
54
|
+
"""
|
|
55
|
+
latency_ms: float
|
|
56
|
+
tokens_used: float
|
|
57
|
+
tool_success_rate: float
|
|
58
|
+
retry_count: float
|
|
59
|
+
confidence: float
|
|
60
|
+
|
|
61
|
+
def to_values(self) -> dict[str, float]:
|
|
62
|
+
"""Dict shape expected by ``agent_parser().parse(...)``."""
|
|
63
|
+
return {
|
|
64
|
+
"latency_ms": self.latency_ms,
|
|
65
|
+
"tokens_used": self.tokens_used,
|
|
66
|
+
"tool_success_rate": self.tool_success_rate,
|
|
67
|
+
"retry_count": self.retry_count,
|
|
68
|
+
"confidence": self.confidence,
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def record_step(
|
|
73
|
+
ledger: Ledger,
|
|
74
|
+
step_index: int,
|
|
75
|
+
expression: Expression,
|
|
76
|
+
tag: str = "",
|
|
77
|
+
) -> Record:
|
|
78
|
+
"""
|
|
79
|
+
Append one agent step's first observation to the ledger so margin's
|
|
80
|
+
drift/forecast/contract layers can reason about trajectory across steps.
|
|
81
|
+
"""
|
|
82
|
+
obs = expression.observations[0] if expression.observations else None
|
|
83
|
+
if obs is None:
|
|
84
|
+
return None
|
|
85
|
+
record = Record(
|
|
86
|
+
step=step_index,
|
|
87
|
+
tag=tag or f"step-{step_index}",
|
|
88
|
+
before=obs,
|
|
89
|
+
fired=False,
|
|
90
|
+
op=Op.NOOP,
|
|
91
|
+
alpha=0.0,
|
|
92
|
+
magnitude=0.0,
|
|
93
|
+
)
|
|
94
|
+
ledger.append(record)
|
|
95
|
+
return record
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def agent_intent(deadline_seconds: Optional[float] = None) -> Intent:
|
|
99
|
+
"""
|
|
100
|
+
Build a default Intent covering 'the agent run is still feasible' \u2014 a
|
|
101
|
+
compact goal-feasibility check you can drop into ``full_step(... intent=...)``.
|
|
102
|
+
|
|
103
|
+
Requirements say latency must stay at DEGRADED-or-better and confidence must
|
|
104
|
+
stay at DEGRADED-or-better. ``deadline_seconds`` sets an ETA-to-violation
|
|
105
|
+
window from the moment ``Intent.evaluate()`` is called.
|
|
106
|
+
"""
|
|
107
|
+
return Intent(
|
|
108
|
+
goal="agent run is still feasible",
|
|
109
|
+
requirements=[
|
|
110
|
+
Requirement(component="latency_ms", min_health=Health.DEGRADED),
|
|
111
|
+
Requirement(component="confidence", min_health=Health.DEGRADED),
|
|
112
|
+
],
|
|
113
|
+
deadline_seconds=deadline_seconds,
|
|
114
|
+
)
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""
|
|
2
|
+
DataFrame / data pipeline adapter for margin.
|
|
3
|
+
|
|
4
|
+
Data quality metrics: completeness, null rate, drift, freshness, row count.
|
|
5
|
+
Works with any tabular data — no pandas dependency required.
|
|
6
|
+
"""
|
|
7
|
+
from .quality import DQ_METRICS, parse_quality, pipeline_expression
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Pandas DataFrame quality hook for margin.
|
|
3
|
+
|
|
4
|
+
Pass a DataFrame, get typed health back. No configuration required —
|
|
5
|
+
computes completeness, null rate, duplicates, and row count automatically.
|
|
6
|
+
|
|
7
|
+
from adapters.dataframe.pandas_hook import dataframe_health
|
|
8
|
+
expr = dataframe_health(df, expected_rows=10000)
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from datetime import datetime
|
|
14
|
+
from typing import Optional
|
|
15
|
+
|
|
16
|
+
from margin.observation import Expression
|
|
17
|
+
from margin.confidence import Confidence
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def dataframe_health(
|
|
21
|
+
df,
|
|
22
|
+
expected_rows: Optional[int] = None,
|
|
23
|
+
expected_columns: Optional[list[str]] = None,
|
|
24
|
+
reference_df=None,
|
|
25
|
+
label: str = "",
|
|
26
|
+
confidence: Confidence = Confidence.MODERATE,
|
|
27
|
+
) -> Expression:
|
|
28
|
+
"""
|
|
29
|
+
Compute data quality metrics from a pandas DataFrame and return
|
|
30
|
+
a typed margin Expression.
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
df: the DataFrame to evaluate
|
|
34
|
+
expected_rows: expected row count (for row_count_ratio)
|
|
35
|
+
expected_columns: expected column names (for schema_match)
|
|
36
|
+
reference_df: reference DataFrame for drift detection
|
|
37
|
+
label: expression label (pipeline name, table name)
|
|
38
|
+
confidence: measurement confidence
|
|
39
|
+
"""
|
|
40
|
+
from .quality import parse_quality, pipeline_expression
|
|
41
|
+
|
|
42
|
+
metrics: dict[str, float] = {}
|
|
43
|
+
|
|
44
|
+
n_rows, n_cols = df.shape
|
|
45
|
+
|
|
46
|
+
# Completeness / null rate
|
|
47
|
+
total_cells = n_rows * n_cols
|
|
48
|
+
if total_cells > 0:
|
|
49
|
+
null_count = int(df.isnull().sum().sum())
|
|
50
|
+
metrics["completeness"] = 1.0 - (null_count / total_cells)
|
|
51
|
+
metrics["null_rate"] = null_count / total_cells
|
|
52
|
+
|
|
53
|
+
# Duplicate rate
|
|
54
|
+
if n_rows > 0:
|
|
55
|
+
n_dupes = int(df.duplicated().sum())
|
|
56
|
+
metrics["duplicate_rate"] = n_dupes / n_rows
|
|
57
|
+
|
|
58
|
+
# Row count ratio
|
|
59
|
+
if expected_rows is not None and expected_rows > 0:
|
|
60
|
+
metrics["row_count_ratio"] = n_rows / expected_rows
|
|
61
|
+
|
|
62
|
+
# Schema match
|
|
63
|
+
if expected_columns is not None and len(expected_columns) > 0:
|
|
64
|
+
actual = set(df.columns)
|
|
65
|
+
expected = set(expected_columns)
|
|
66
|
+
if expected:
|
|
67
|
+
metrics["schema_match"] = len(actual & expected) / len(expected)
|
|
68
|
+
|
|
69
|
+
# Value drift (simple: compare column means if reference provided)
|
|
70
|
+
if reference_df is not None:
|
|
71
|
+
try:
|
|
72
|
+
numeric_cols = df.select_dtypes(include="number").columns
|
|
73
|
+
ref_numeric = reference_df.select_dtypes(include="number").columns
|
|
74
|
+
shared = list(set(numeric_cols) & set(ref_numeric))
|
|
75
|
+
if shared:
|
|
76
|
+
drifts = []
|
|
77
|
+
for col in shared:
|
|
78
|
+
ref_mean = float(reference_df[col].mean())
|
|
79
|
+
cur_mean = float(df[col].mean())
|
|
80
|
+
if abs(ref_mean) > 0.001:
|
|
81
|
+
drifts.append(abs(cur_mean - ref_mean) / abs(ref_mean))
|
|
82
|
+
if drifts:
|
|
83
|
+
metrics["value_drift"] = sum(drifts) / len(drifts)
|
|
84
|
+
except Exception:
|
|
85
|
+
pass # drift is best-effort
|
|
86
|
+
|
|
87
|
+
# Outlier rate (simple: beyond 3 std from mean)
|
|
88
|
+
try:
|
|
89
|
+
numeric = df.select_dtypes(include="number")
|
|
90
|
+
if not numeric.empty and n_rows > 0:
|
|
91
|
+
means = numeric.mean()
|
|
92
|
+
stds = numeric.std()
|
|
93
|
+
outlier_mask = ((numeric - means).abs() > 3 * stds).any(axis=1)
|
|
94
|
+
metrics["outlier_rate"] = float(outlier_mask.sum()) / n_rows
|
|
95
|
+
except Exception:
|
|
96
|
+
pass
|
|
97
|
+
|
|
98
|
+
return pipeline_expression(metrics, pipeline_id=label, confidence=confidence,
|
|
99
|
+
measured_at=datetime.now())
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Data quality metrics as margin observations.
|
|
3
|
+
|
|
4
|
+
Column-level and dataset-level quality checks for any tabular pipeline.
|
|
5
|
+
No pandas dependency — pass in the metrics you've already computed.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
from typing import Optional
|
|
12
|
+
|
|
13
|
+
from margin.health import Thresholds
|
|
14
|
+
from margin.observation import Observation, Expression
|
|
15
|
+
from margin.confidence import Confidence
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class DQMetric:
|
|
20
|
+
name: str
|
|
21
|
+
display_name: str
|
|
22
|
+
thresholds: Thresholds
|
|
23
|
+
baseline: float
|
|
24
|
+
unit: str = ""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
DQ_METRICS: dict[str, DQMetric] = {
|
|
28
|
+
"completeness": DQMetric(
|
|
29
|
+
name="completeness", display_name="Completeness (non-null ratio)",
|
|
30
|
+
thresholds=Thresholds(intact=0.99, ablated=0.80, higher_is_better=True),
|
|
31
|
+
baseline=1.0, unit="ratio",
|
|
32
|
+
),
|
|
33
|
+
"null_rate": DQMetric(
|
|
34
|
+
name="null_rate", display_name="Null Rate",
|
|
35
|
+
thresholds=Thresholds(intact=0.01, ablated=0.20, higher_is_better=False),
|
|
36
|
+
baseline=0.0, unit="ratio",
|
|
37
|
+
),
|
|
38
|
+
"duplicate_rate": DQMetric(
|
|
39
|
+
name="duplicate_rate", display_name="Duplicate Rate",
|
|
40
|
+
thresholds=Thresholds(intact=0.001, ablated=0.05, higher_is_better=False),
|
|
41
|
+
baseline=0.0, unit="ratio",
|
|
42
|
+
),
|
|
43
|
+
"schema_match": DQMetric(
|
|
44
|
+
name="schema_match", display_name="Schema Match",
|
|
45
|
+
thresholds=Thresholds(intact=1.0, ablated=0.90, higher_is_better=True),
|
|
46
|
+
baseline=1.0, unit="ratio",
|
|
47
|
+
),
|
|
48
|
+
"row_count_ratio": DQMetric(
|
|
49
|
+
name="row_count_ratio", display_name="Row Count vs Expected",
|
|
50
|
+
thresholds=Thresholds(intact=0.90, ablated=0.50, higher_is_better=True),
|
|
51
|
+
baseline=1.0, unit="ratio",
|
|
52
|
+
),
|
|
53
|
+
"freshness_hours": DQMetric(
|
|
54
|
+
name="freshness_hours", display_name="Data Freshness",
|
|
55
|
+
thresholds=Thresholds(intact=1.0, ablated=24.0, higher_is_better=False),
|
|
56
|
+
baseline=0.25, unit="hours",
|
|
57
|
+
),
|
|
58
|
+
"value_drift": DQMetric(
|
|
59
|
+
name="value_drift", display_name="Value Distribution Drift",
|
|
60
|
+
thresholds=Thresholds(intact=0.05, ablated=0.30, higher_is_better=False),
|
|
61
|
+
baseline=0.01, unit="KL divergence",
|
|
62
|
+
),
|
|
63
|
+
"outlier_rate": DQMetric(
|
|
64
|
+
name="outlier_rate", display_name="Outlier Rate",
|
|
65
|
+
thresholds=Thresholds(intact=0.02, ablated=0.10, higher_is_better=False),
|
|
66
|
+
baseline=0.005, unit="ratio",
|
|
67
|
+
),
|
|
68
|
+
"type_error_rate": DQMetric(
|
|
69
|
+
name="type_error_rate", display_name="Type Error Rate",
|
|
70
|
+
thresholds=Thresholds(intact=0.0, ablated=0.01, higher_is_better=False),
|
|
71
|
+
baseline=0.0, unit="ratio",
|
|
72
|
+
),
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def parse_quality(
|
|
77
|
+
metrics: dict[str, float],
|
|
78
|
+
confidence: Confidence = Confidence.MODERATE,
|
|
79
|
+
defs: Optional[dict[str, DQMetric]] = None,
|
|
80
|
+
measured_at: Optional[datetime] = None,
|
|
81
|
+
) -> dict[str, Observation]:
|
|
82
|
+
from margin.health import classify
|
|
83
|
+
d = defs or DQ_METRICS
|
|
84
|
+
observations = {}
|
|
85
|
+
for name, value in metrics.items():
|
|
86
|
+
m = d.get(name)
|
|
87
|
+
if m is None:
|
|
88
|
+
continue
|
|
89
|
+
health = classify(value, confidence, thresholds=m.thresholds)
|
|
90
|
+
observations[name] = Observation(
|
|
91
|
+
name=name, health=health, value=value, baseline=m.baseline,
|
|
92
|
+
confidence=confidence, higher_is_better=m.thresholds.higher_is_better,
|
|
93
|
+
measured_at=measured_at,
|
|
94
|
+
)
|
|
95
|
+
return observations
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def pipeline_expression(
|
|
99
|
+
metrics: dict[str, float],
|
|
100
|
+
pipeline_id: str = "",
|
|
101
|
+
confidence: Confidence = Confidence.MODERATE,
|
|
102
|
+
defs: Optional[dict[str, DQMetric]] = None,
|
|
103
|
+
measured_at: Optional[datetime] = None,
|
|
104
|
+
) -> Expression:
|
|
105
|
+
obs = parse_quality(metrics, confidence, defs, measured_at)
|
|
106
|
+
return Expression(
|
|
107
|
+
observations=list(obs.values()),
|
|
108
|
+
confidence=min((o.confidence for o in obs.values()), default=Confidence.INDETERMINATE),
|
|
109
|
+
label=pipeline_id,
|
|
110
|
+
)
|