margin-meter 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- margin_meter-0.1.0/PKG-INFO +117 -0
- margin_meter-0.1.0/README.md +105 -0
- margin_meter-0.1.0/margin_meter/__init__.py +28 -0
- margin_meter-0.1.0/margin_meter/client.py +373 -0
- margin_meter-0.1.0/margin_meter.egg-info/PKG-INFO +117 -0
- margin_meter-0.1.0/margin_meter.egg-info/SOURCES.txt +8 -0
- margin_meter-0.1.0/margin_meter.egg-info/dependency_links.txt +1 -0
- margin_meter-0.1.0/margin_meter.egg-info/top_level.txt +1 -0
- margin_meter-0.1.0/pyproject.toml +22 -0
- margin_meter-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: margin-meter
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Tiny stdlib-only client that emits LLM call + outcome economics to a Margin ingest API.
|
|
5
|
+
Author: Margin
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/subhsubh24/Margin.ai
|
|
8
|
+
Project-URL: Source, https://github.com/subhsubh24/Margin.ai/tree/main/sdk/python
|
|
9
|
+
Keywords: llm,cost,observability,finops,margin
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
|
|
13
|
+
# margin-meter (Python SDK)
|
|
14
|
+
|
|
15
|
+
The tiny client a **Python** project imports to connect to Margin. It wraps your
|
|
16
|
+
LLM calls and records their outcomes, emitting each one **over HTTP** to a
|
|
17
|
+
Margin **ingest API** (`POST /api/ingest/calls` / `/api/ingest/outcomes`),
|
|
18
|
+
authenticated with a per-project ingest key. This is the customer-shaped path —
|
|
19
|
+
the same SDK a stranger drops in — not the in-process meter Margin runs on
|
|
20
|
+
itself.
|
|
21
|
+
|
|
22
|
+
- **Standalone + stdlib-only.** No dependency on Margin's server code and no
|
|
23
|
+
third-party deps. The default transport is `urllib`.
|
|
24
|
+
- **Fail-safe.** A failed emit returns an `IngestResult(ok=False, …)` instead of
|
|
25
|
+
crashing your app. Pass `raise_on_error=True` for strict/CI behaviour.
|
|
26
|
+
- **Provenance-honest.** `is_simulated` is carried through untouched; the written
|
|
27
|
+
`source` is forced server-side to your key's project — you cannot spoof
|
|
28
|
+
another project's economics.
|
|
29
|
+
|
|
30
|
+
## Install
|
|
31
|
+
|
|
32
|
+
Published as a git-installable subpath of the Margin repo (no PyPI account
|
|
33
|
+
needed pre-launch):
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install "git+https://github.com/subhsubh24/Margin.ai.git#subdirectory=sdk/python"
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Configure
|
|
40
|
+
|
|
41
|
+
Two environment variables — the deployed API base and your project's key:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
export MARGIN_INGEST_URL="https://margin-ai-rho.vercel.app"
|
|
45
|
+
export MARGIN_INGEST_KEY="mgk_…" # issued by the Margin owner (see below)
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The Margin owner issues your project a key with the provisioning CLI in the
|
|
49
|
+
Margin repo:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
python3 scripts/issue_ingest_key.py <your-project-slug>
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
The raw `mgk_…` key is shown **once** — only its hash is stored. Give it to your
|
|
56
|
+
project as `MARGIN_INGEST_KEY`.
|
|
57
|
+
|
|
58
|
+
## Use
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
from margin_meter import MarginMeter
|
|
62
|
+
|
|
63
|
+
meter = MarginMeter() # reads MARGIN_INGEST_URL + MARGIN_INGEST_KEY
|
|
64
|
+
|
|
65
|
+
# 1) Wrap the LLM call — latency is timed automatically, cost computed server-side.
|
|
66
|
+
with meter.measure(workflow_id="fit-scoring", provider="google",
|
|
67
|
+
model="gemini-2.5-flash") as m:
|
|
68
|
+
resp = call_the_model(...)
|
|
69
|
+
m.set_tokens(input_tokens=1200, output_tokens=300, cache_read_tokens=800)
|
|
70
|
+
|
|
71
|
+
# 2) Record the outcome it produced (the unit of productivity).
|
|
72
|
+
meter.record_outcome(workflow_id="fit-scoring", passed=True,
|
|
73
|
+
quality_score=0.94, quality_method="ground_truth")
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Or record a call directly (when you already have the token counts):
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
res = meter.record_call(
|
|
80
|
+
workflow_id="fit-scoring", provider="google", model="gemini-2.5-flash",
|
|
81
|
+
input_tokens=1200, output_tokens=300, cache_read_tokens=800,
|
|
82
|
+
)
|
|
83
|
+
if not res.ok:
|
|
84
|
+
log.warning("margin ingest failed: %s (%s)", res.error, res.status_code)
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## API
|
|
88
|
+
|
|
89
|
+
| Method | Emits to | Notes |
|
|
90
|
+
| --- | --- | --- |
|
|
91
|
+
| `record_call(...)` | `POST /api/ingest/calls` | cost computed from the pricing table when `cost_usd` omitted |
|
|
92
|
+
| `record_outcome(...)` | `POST /api/ingest/outcomes` | `quality_method` records HOW the score was graded |
|
|
93
|
+
| `measure(...)` | `record_call` on exit | context manager; times latency, `status="error"` on exception |
|
|
94
|
+
|
|
95
|
+
Every method returns an `IngestResult(ok, status_code, body, error)`. `ok` is
|
|
96
|
+
True only on HTTP 200; `body` holds `call_id`/`outcome_id` + `source`.
|
|
97
|
+
|
|
98
|
+
## Testing against a live app (no network)
|
|
99
|
+
|
|
100
|
+
The network boundary is a single `transport.post(path, json=..., headers=...)`
|
|
101
|
+
protocol, which a FastAPI `TestClient` satisfies exactly — so you can exercise
|
|
102
|
+
the real ingest endpoints hermetically by injecting one:
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
from fastapi.testclient import TestClient
|
|
106
|
+
import asgi
|
|
107
|
+
from margin_meter import MarginMeter
|
|
108
|
+
|
|
109
|
+
meter = MarginMeter(api_key=raw_key, transport=TestClient(asgi.app),
|
|
110
|
+
raise_on_error=True)
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
## Rate + validation bounds
|
|
114
|
+
|
|
115
|
+
Ingest is auth'd, validated, and rate-bounded server-side. A bad key → 401, a
|
|
116
|
+
malformed/implausible row → 422, and a full rolling per-project window → 429.
|
|
117
|
+
In fail-safe mode these come back as `IngestResult(ok=False, status_code=…)`.
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# margin-meter (Python SDK)
|
|
2
|
+
|
|
3
|
+
The tiny client a **Python** project imports to connect to Margin. It wraps your
|
|
4
|
+
LLM calls and records their outcomes, emitting each one **over HTTP** to a
|
|
5
|
+
Margin **ingest API** (`POST /api/ingest/calls` / `/api/ingest/outcomes`),
|
|
6
|
+
authenticated with a per-project ingest key. This is the customer-shaped path —
|
|
7
|
+
the same SDK a stranger drops in — not the in-process meter Margin runs on
|
|
8
|
+
itself.
|
|
9
|
+
|
|
10
|
+
- **Standalone + stdlib-only.** No dependency on Margin's server code and no
|
|
11
|
+
third-party deps. The default transport is `urllib`.
|
|
12
|
+
- **Fail-safe.** A failed emit returns an `IngestResult(ok=False, …)` instead of
|
|
13
|
+
crashing your app. Pass `raise_on_error=True` for strict/CI behaviour.
|
|
14
|
+
- **Provenance-honest.** `is_simulated` is carried through untouched; the written
|
|
15
|
+
`source` is forced server-side to your key's project — you cannot spoof
|
|
16
|
+
another project's economics.
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
Published as a git-installable subpath of the Margin repo (no PyPI account
|
|
21
|
+
needed pre-launch):
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install "git+https://github.com/subhsubh24/Margin.ai.git#subdirectory=sdk/python"
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Configure
|
|
28
|
+
|
|
29
|
+
Two environment variables — the deployed API base and your project's key:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
export MARGIN_INGEST_URL="https://margin-ai-rho.vercel.app"
|
|
33
|
+
export MARGIN_INGEST_KEY="mgk_…" # issued by the Margin owner (see below)
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
The Margin owner issues your project a key with the provisioning CLI in the
|
|
37
|
+
Margin repo:
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
python3 scripts/issue_ingest_key.py <your-project-slug>
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
The raw `mgk_…` key is shown **once** — only its hash is stored. Give it to your
|
|
44
|
+
project as `MARGIN_INGEST_KEY`.
|
|
45
|
+
|
|
46
|
+
## Use
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from margin_meter import MarginMeter
|
|
50
|
+
|
|
51
|
+
meter = MarginMeter() # reads MARGIN_INGEST_URL + MARGIN_INGEST_KEY
|
|
52
|
+
|
|
53
|
+
# 1) Wrap the LLM call — latency is timed automatically, cost computed server-side.
|
|
54
|
+
with meter.measure(workflow_id="fit-scoring", provider="google",
|
|
55
|
+
model="gemini-2.5-flash") as m:
|
|
56
|
+
resp = call_the_model(...)
|
|
57
|
+
m.set_tokens(input_tokens=1200, output_tokens=300, cache_read_tokens=800)
|
|
58
|
+
|
|
59
|
+
# 2) Record the outcome it produced (the unit of productivity).
|
|
60
|
+
meter.record_outcome(workflow_id="fit-scoring", passed=True,
|
|
61
|
+
quality_score=0.94, quality_method="ground_truth")
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Or record a call directly (when you already have the token counts):
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
res = meter.record_call(
|
|
68
|
+
workflow_id="fit-scoring", provider="google", model="gemini-2.5-flash",
|
|
69
|
+
input_tokens=1200, output_tokens=300, cache_read_tokens=800,
|
|
70
|
+
)
|
|
71
|
+
if not res.ok:
|
|
72
|
+
log.warning("margin ingest failed: %s (%s)", res.error, res.status_code)
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## API
|
|
76
|
+
|
|
77
|
+
| Method | Emits to | Notes |
|
|
78
|
+
| --- | --- | --- |
|
|
79
|
+
| `record_call(...)` | `POST /api/ingest/calls` | cost computed from the pricing table when `cost_usd` omitted |
|
|
80
|
+
| `record_outcome(...)` | `POST /api/ingest/outcomes` | `quality_method` records HOW the score was graded |
|
|
81
|
+
| `measure(...)` | `record_call` on exit | context manager; times latency, `status="error"` on exception |
|
|
82
|
+
|
|
83
|
+
Every method returns an `IngestResult(ok, status_code, body, error)`. `ok` is
|
|
84
|
+
True only on HTTP 200; `body` holds `call_id`/`outcome_id` + `source`.
|
|
85
|
+
|
|
86
|
+
## Testing against a live app (no network)
|
|
87
|
+
|
|
88
|
+
The network boundary is a single `transport.post(path, json=..., headers=...)`
|
|
89
|
+
protocol, which a FastAPI `TestClient` satisfies exactly — so you can exercise
|
|
90
|
+
the real ingest endpoints hermetically by injecting one:
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
from fastapi.testclient import TestClient
|
|
94
|
+
import asgi
|
|
95
|
+
from margin_meter import MarginMeter
|
|
96
|
+
|
|
97
|
+
meter = MarginMeter(api_key=raw_key, transport=TestClient(asgi.app),
|
|
98
|
+
raise_on_error=True)
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## Rate + validation bounds
|
|
102
|
+
|
|
103
|
+
Ingest is auth'd, validated, and rate-bounded server-side. A bad key → 401, a
|
|
104
|
+
malformed/implausible row → 422, and a full rolling per-project window → 429.
|
|
105
|
+
In fail-safe mode these come back as `IngestResult(ok=False, status_code=…)`.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""margin_meter — the Python SDK a project imports to connect to Margin.
|
|
2
|
+
|
|
3
|
+
A tiny, stdlib-only client that wraps an LLM call and emits its measured
|
|
4
|
+
economics (spend + outcome) to a Margin ingest API over HTTP. See
|
|
5
|
+
:mod:`margin_meter.client` for the full contract.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from margin_meter.client import (
|
|
9
|
+
CALLS_PATH,
|
|
10
|
+
OUTCOMES_PATH,
|
|
11
|
+
IngestResult,
|
|
12
|
+
MarginConfigError,
|
|
13
|
+
MarginIngestError,
|
|
14
|
+
MarginMeter,
|
|
15
|
+
MarginMeterError,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"MarginMeter",
|
|
20
|
+
"IngestResult",
|
|
21
|
+
"MarginMeterError",
|
|
22
|
+
"MarginConfigError",
|
|
23
|
+
"MarginIngestError",
|
|
24
|
+
"CALLS_PATH",
|
|
25
|
+
"OUTCOMES_PATH",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,373 @@
|
|
|
1
|
+
"""The Python Margin meter client — emits call + outcome economics OVER HTTP.
|
|
2
|
+
|
|
3
|
+
This is the importable library a Python sibling (JobScraper, LLM-Quant, or any
|
|
4
|
+
real customer) drops in to connect to Margin. Unlike ``margin.meter`` (the
|
|
5
|
+
in-process, DB-direct meter Margin uses on itself), this client is a *standalone*
|
|
6
|
+
package with NO dependency on the ``margin`` server code: it wraps an LLM call
|
|
7
|
+
and POSTs the measured row to Margin's ingest API (``POST /api/ingest/calls`` and
|
|
8
|
+
``/api/ingest/outcomes``), authenticated with a per-project ingest key. That is
|
|
9
|
+
the customer-shaped path — the same SDK a stranger would install.
|
|
10
|
+
|
|
11
|
+
Design goals that shaped this module:
|
|
12
|
+
|
|
13
|
+
- **Standalone + stdlib-only.** A sibling installs ``margin-meter`` and nothing
|
|
14
|
+
else — no server code, no heavy deps. The default transport is ``urllib``.
|
|
15
|
+
- **Pluggable transport (hermetically testable).** The network boundary is a
|
|
16
|
+
single ``transport.post(path, json=..., headers=...) -> response`` protocol,
|
|
17
|
+
so a test can inject a FastAPI ``TestClient`` (which satisfies it exactly) and
|
|
18
|
+
exercise the real ingest endpoints with no socket. Production uses urllib.
|
|
19
|
+
- **Fail-safe by default.** Telemetry must never crash the host app, so a failed
|
|
20
|
+
emit returns an :class:`IngestResult` with ``ok=False`` instead of raising.
|
|
21
|
+
Pass ``raise_on_error=True`` for strict/testing behaviour.
|
|
22
|
+
- **Provenance-honest.** ``is_simulated`` is carried through untouched; a real
|
|
23
|
+
call stays real. The written ``source`` is forced server-side to the key's
|
|
24
|
+
project slug — the client cannot spoof another project's economics.
|
|
25
|
+
|
|
26
|
+
Usage (a sibling wrapping its own call)::
|
|
27
|
+
|
|
28
|
+
from margin_meter import MarginMeter
|
|
29
|
+
|
|
30
|
+
meter = MarginMeter() # reads MARGIN_INGEST_URL + MARGIN_INGEST_KEY
|
|
31
|
+
|
|
32
|
+
with meter.measure(workflow_id="fit-scoring", provider="google",
|
|
33
|
+
model="gemini-2.5-flash") as m:
|
|
34
|
+
resp = call_the_model(...)
|
|
35
|
+
m.set_tokens(input_tokens=1200, output_tokens=300, cache_read_tokens=800)
|
|
36
|
+
|
|
37
|
+
meter.record_outcome(workflow_id="fit-scoring", passed=True,
|
|
38
|
+
quality_score=0.94, quality_method="ground_truth")
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
from __future__ import annotations
|
|
42
|
+
|
|
43
|
+
import json as _json
|
|
44
|
+
import os
|
|
45
|
+
import time
|
|
46
|
+
import urllib.error
|
|
47
|
+
import urllib.request
|
|
48
|
+
from dataclasses import dataclass
|
|
49
|
+
from types import TracebackType
|
|
50
|
+
from typing import Any, Optional
|
|
51
|
+
|
|
52
|
+
CALLS_PATH = "/api/ingest/calls"
|
|
53
|
+
OUTCOMES_PATH = "/api/ingest/outcomes"
|
|
54
|
+
KEY_HEADER = "X-Margin-Key"
|
|
55
|
+
|
|
56
|
+
DEFAULT_INGEST_URL = "http://127.0.0.1:8000"
|
|
57
|
+
DEFAULT_TIMEOUT_S = 10.0
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class MarginMeterError(RuntimeError):
|
|
61
|
+
"""Base error for the SDK."""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class MarginConfigError(MarginMeterError):
|
|
65
|
+
"""The client is missing required configuration (ingest URL or key)."""
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class MarginIngestError(MarginMeterError):
|
|
69
|
+
"""A row was rejected by the ingest API (only raised when strict).
|
|
70
|
+
|
|
71
|
+
``status_code`` is the HTTP status (401 bad key, 422 invalid row, 429 rate
|
|
72
|
+
cap) and ``detail`` the server's reason, so a strict caller gets an
|
|
73
|
+
actionable error instead of a silent drop.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
def __init__(self, status_code: int, detail: str) -> None:
|
|
77
|
+
self.status_code = status_code
|
|
78
|
+
self.detail = detail
|
|
79
|
+
super().__init__(f"ingest rejected ({status_code}): {detail}")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclass
|
|
83
|
+
class IngestResult:
|
|
84
|
+
"""The outcome of one emit. Returned by every record call.
|
|
85
|
+
|
|
86
|
+
``ok`` is True only on an HTTP 200. On failure ``status_code`` (0 for a
|
|
87
|
+
transport/network failure) and ``error`` carry the reason; ``body`` holds the
|
|
88
|
+
parsed success payload (``call_id``/``outcome_id`` + ``source``) when ok.
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
ok: bool
|
|
92
|
+
status_code: int
|
|
93
|
+
body: Optional[dict] = None
|
|
94
|
+
error: Optional[str] = None
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class _HttpResponse:
|
|
98
|
+
"""Minimal response shape the urllib transport returns (mirrors TestClient)."""
|
|
99
|
+
|
|
100
|
+
def __init__(self, status_code: int, body: bytes) -> None:
|
|
101
|
+
self.status_code = status_code
|
|
102
|
+
self._body = body
|
|
103
|
+
|
|
104
|
+
def json(self) -> Any:
|
|
105
|
+
if not self._body:
|
|
106
|
+
return {}
|
|
107
|
+
return _json.loads(self._body.decode("utf-8"))
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class _UrllibTransport:
|
|
111
|
+
"""The default transport: POST JSON over HTTP with the stdlib, no deps.
|
|
112
|
+
|
|
113
|
+
Satisfies the same ``post(path, json=..., headers=...)`` protocol a FastAPI
|
|
114
|
+
``TestClient`` does, so the two are interchangeable — production uses this,
|
|
115
|
+
tests inject the TestClient.
|
|
116
|
+
"""
|
|
117
|
+
|
|
118
|
+
def __init__(self, base_url: str, timeout: float = DEFAULT_TIMEOUT_S) -> None:
|
|
119
|
+
self.base_url = base_url.rstrip("/")
|
|
120
|
+
self.timeout = timeout
|
|
121
|
+
|
|
122
|
+
def post(self, path: str, json: dict, headers: dict) -> _HttpResponse:
|
|
123
|
+
url = self.base_url + path
|
|
124
|
+
data = _json.dumps(json).encode("utf-8")
|
|
125
|
+
req_headers = {"Content-Type": "application/json", **headers}
|
|
126
|
+
req = urllib.request.Request(url, data=data, headers=req_headers, method="POST")
|
|
127
|
+
try:
|
|
128
|
+
with urllib.request.urlopen(req, timeout=self.timeout) as resp:
|
|
129
|
+
return _HttpResponse(resp.status, resp.read())
|
|
130
|
+
except urllib.error.HTTPError as exc:
|
|
131
|
+
# A 4xx/5xx from the API — surface the status + body, don't raise.
|
|
132
|
+
return _HttpResponse(exc.code, exc.read())
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class MarginMeter:
|
|
136
|
+
"""A per-project client that emits measured economics to a Margin ingest API.
|
|
137
|
+
|
|
138
|
+
Configuration is read from the environment when not passed explicitly:
|
|
139
|
+
``MARGIN_INGEST_URL`` (the deployed API base) and ``MARGIN_INGEST_KEY`` (the
|
|
140
|
+
project's ``mgk_…`` key). A sibling typically constructs ``MarginMeter()``
|
|
141
|
+
with no args and lets the env drive it.
|
|
142
|
+
"""
|
|
143
|
+
|
|
144
|
+
def __init__(
|
|
145
|
+
self,
|
|
146
|
+
*,
|
|
147
|
+
ingest_url: str | None = None,
|
|
148
|
+
api_key: str | None = None,
|
|
149
|
+
transport: Any = None,
|
|
150
|
+
timeout: float = DEFAULT_TIMEOUT_S,
|
|
151
|
+
raise_on_error: bool = False,
|
|
152
|
+
) -> None:
|
|
153
|
+
self.ingest_url = ingest_url or os.environ.get("MARGIN_INGEST_URL") or DEFAULT_INGEST_URL
|
|
154
|
+
self.api_key = api_key or os.environ.get("MARGIN_INGEST_KEY")
|
|
155
|
+
self.raise_on_error = raise_on_error
|
|
156
|
+
# An injected transport (e.g. a TestClient) wins; else build the default.
|
|
157
|
+
self._transport = transport or _UrllibTransport(self.ingest_url, timeout=timeout)
|
|
158
|
+
|
|
159
|
+
# ------------------------------------------------------------------ #
|
|
160
|
+
# Core emit
|
|
161
|
+
# ------------------------------------------------------------------ #
|
|
162
|
+
def _emit(self, path: str, payload: dict) -> IngestResult:
|
|
163
|
+
if not self.api_key:
|
|
164
|
+
result = IngestResult(
|
|
165
|
+
ok=False,
|
|
166
|
+
status_code=0,
|
|
167
|
+
error="no ingest key (set MARGIN_INGEST_KEY or pass api_key)",
|
|
168
|
+
)
|
|
169
|
+
if self.raise_on_error:
|
|
170
|
+
raise MarginConfigError(result.error)
|
|
171
|
+
return result
|
|
172
|
+
|
|
173
|
+
headers = {KEY_HEADER: self.api_key}
|
|
174
|
+
try:
|
|
175
|
+
resp = self._transport.post(path, json=payload, headers=headers)
|
|
176
|
+
except Exception as exc: # network/transport failure — fail-safe
|
|
177
|
+
result = IngestResult(ok=False, status_code=0, error=f"transport error: {exc}")
|
|
178
|
+
if self.raise_on_error:
|
|
179
|
+
raise MarginIngestError(0, str(exc)) from exc
|
|
180
|
+
return result
|
|
181
|
+
|
|
182
|
+
status = resp.status_code
|
|
183
|
+
if status == 200:
|
|
184
|
+
return IngestResult(ok=True, status_code=200, body=resp.json())
|
|
185
|
+
|
|
186
|
+
detail = self._extract_detail(resp)
|
|
187
|
+
if self.raise_on_error:
|
|
188
|
+
raise MarginIngestError(status, detail)
|
|
189
|
+
return IngestResult(ok=False, status_code=status, error=detail)
|
|
190
|
+
|
|
191
|
+
@staticmethod
|
|
192
|
+
def _extract_detail(resp: Any) -> str:
|
|
193
|
+
try:
|
|
194
|
+
body = resp.json()
|
|
195
|
+
except Exception:
|
|
196
|
+
return "unparseable error body"
|
|
197
|
+
if isinstance(body, dict) and "detail" in body:
|
|
198
|
+
return str(body["detail"])
|
|
199
|
+
return str(body)
|
|
200
|
+
|
|
201
|
+
# ------------------------------------------------------------------ #
|
|
202
|
+
# Public surface — mirrors margin.meter
|
|
203
|
+
# ------------------------------------------------------------------ #
|
|
204
|
+
def record_call(
|
|
205
|
+
self,
|
|
206
|
+
*,
|
|
207
|
+
workflow_id: str,
|
|
208
|
+
provider: str,
|
|
209
|
+
model: str,
|
|
210
|
+
input_tokens: int = 0,
|
|
211
|
+
output_tokens: int = 0,
|
|
212
|
+
cache_read_tokens: int = 0,
|
|
213
|
+
latency_ms: int = 0,
|
|
214
|
+
status: str = "ok",
|
|
215
|
+
is_retry: bool = False,
|
|
216
|
+
session_id: str | None = None,
|
|
217
|
+
prompt_id: str | None = None,
|
|
218
|
+
cost_usd: float | None = None,
|
|
219
|
+
is_simulated: bool = False,
|
|
220
|
+
) -> IngestResult:
|
|
221
|
+
"""Emit one measured LLM call to ``POST /api/ingest/calls``.
|
|
222
|
+
|
|
223
|
+
Cost is computed server-side from the pricing table when ``cost_usd`` is
|
|
224
|
+
omitted, so a sibling need only report tokens. Returns an
|
|
225
|
+
:class:`IngestResult` (never raises unless ``raise_on_error``).
|
|
226
|
+
"""
|
|
227
|
+
payload: dict[str, Any] = {
|
|
228
|
+
"workflow_id": workflow_id,
|
|
229
|
+
"provider": provider,
|
|
230
|
+
"model": model,
|
|
231
|
+
"input_tokens": input_tokens,
|
|
232
|
+
"output_tokens": output_tokens,
|
|
233
|
+
"cache_read_tokens": cache_read_tokens,
|
|
234
|
+
"latency_ms": latency_ms,
|
|
235
|
+
"status": status,
|
|
236
|
+
"is_retry": is_retry,
|
|
237
|
+
"is_simulated": is_simulated,
|
|
238
|
+
}
|
|
239
|
+
if session_id is not None:
|
|
240
|
+
payload["session_id"] = session_id
|
|
241
|
+
if prompt_id is not None:
|
|
242
|
+
payload["prompt_id"] = prompt_id
|
|
243
|
+
if cost_usd is not None:
|
|
244
|
+
payload["cost_usd"] = cost_usd
|
|
245
|
+
return self._emit(CALLS_PATH, payload)
|
|
246
|
+
|
|
247
|
+
def record_outcome(
|
|
248
|
+
self,
|
|
249
|
+
*,
|
|
250
|
+
workflow_id: str,
|
|
251
|
+
passed: bool,
|
|
252
|
+
quality_score: float | None = None,
|
|
253
|
+
quality_method: str | None = None,
|
|
254
|
+
link: str | None = None,
|
|
255
|
+
is_simulated: bool = False,
|
|
256
|
+
) -> IngestResult:
|
|
257
|
+
"""Emit one outcome (a unit of productivity) to ``POST /api/ingest/outcomes``.
|
|
258
|
+
|
|
259
|
+
``quality_method`` records HOW ``quality_score`` was graded (e.g.
|
|
260
|
+
``ground_truth``, ``llm_judge``) so a self-report is never mistaken for a
|
|
261
|
+
graded number. Returns an :class:`IngestResult`.
|
|
262
|
+
"""
|
|
263
|
+
payload: dict[str, Any] = {
|
|
264
|
+
"workflow_id": workflow_id,
|
|
265
|
+
"passed": passed,
|
|
266
|
+
"is_simulated": is_simulated,
|
|
267
|
+
}
|
|
268
|
+
if quality_score is not None:
|
|
269
|
+
payload["quality_score"] = quality_score
|
|
270
|
+
if quality_method is not None:
|
|
271
|
+
payload["quality_method"] = quality_method
|
|
272
|
+
if link is not None:
|
|
273
|
+
payload["link"] = link
|
|
274
|
+
return self._emit(OUTCOMES_PATH, payload)
|
|
275
|
+
|
|
276
|
+
def measure(
|
|
277
|
+
self,
|
|
278
|
+
*,
|
|
279
|
+
workflow_id: str,
|
|
280
|
+
provider: str,
|
|
281
|
+
model: str,
|
|
282
|
+
is_retry: bool = False,
|
|
283
|
+
session_id: str | None = None,
|
|
284
|
+
prompt_id: str | None = None,
|
|
285
|
+
is_simulated: bool = False,
|
|
286
|
+
) -> "_Measure":
|
|
287
|
+
"""A context manager that times a call and emits it on exit.
|
|
288
|
+
|
|
289
|
+
Mirrors ``margin.meter.measure`` but over HTTP. Set token counts inside
|
|
290
|
+
the block via :meth:`_Measure.set_tokens`; latency is timed
|
|
291
|
+
automatically. On an exception the call is still emitted with
|
|
292
|
+
``status="error"`` and the exception re-raised. The emit's
|
|
293
|
+
:class:`IngestResult` is available as ``.result`` after the block.
|
|
294
|
+
"""
|
|
295
|
+
return _Measure(
|
|
296
|
+
meter=self,
|
|
297
|
+
workflow_id=workflow_id,
|
|
298
|
+
provider=provider,
|
|
299
|
+
model=model,
|
|
300
|
+
is_retry=is_retry,
|
|
301
|
+
session_id=session_id,
|
|
302
|
+
prompt_id=prompt_id,
|
|
303
|
+
is_simulated=is_simulated,
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
class _Measure:
|
|
308
|
+
"""Context manager returned by :meth:`MarginMeter.measure`."""
|
|
309
|
+
|
|
310
|
+
def __init__(
|
|
311
|
+
self,
|
|
312
|
+
*,
|
|
313
|
+
meter: MarginMeter,
|
|
314
|
+
workflow_id: str,
|
|
315
|
+
provider: str,
|
|
316
|
+
model: str,
|
|
317
|
+
is_retry: bool,
|
|
318
|
+
session_id: str | None,
|
|
319
|
+
prompt_id: str | None,
|
|
320
|
+
is_simulated: bool,
|
|
321
|
+
) -> None:
|
|
322
|
+
self._meter = meter
|
|
323
|
+
self.workflow_id = workflow_id
|
|
324
|
+
self.provider = provider
|
|
325
|
+
self.model = model
|
|
326
|
+
self.is_retry = is_retry
|
|
327
|
+
self.session_id = session_id
|
|
328
|
+
self.prompt_id = prompt_id
|
|
329
|
+
self.is_simulated = is_simulated
|
|
330
|
+
self.input_tokens = 0
|
|
331
|
+
self.output_tokens = 0
|
|
332
|
+
self.cache_read_tokens = 0
|
|
333
|
+
self.result: IngestResult | None = None
|
|
334
|
+
self._start = 0.0
|
|
335
|
+
|
|
336
|
+
def set_tokens(
|
|
337
|
+
self,
|
|
338
|
+
*,
|
|
339
|
+
input_tokens: int = 0,
|
|
340
|
+
output_tokens: int = 0,
|
|
341
|
+
cache_read_tokens: int = 0,
|
|
342
|
+
) -> None:
|
|
343
|
+
self.input_tokens = input_tokens
|
|
344
|
+
self.output_tokens = output_tokens
|
|
345
|
+
self.cache_read_tokens = cache_read_tokens
|
|
346
|
+
|
|
347
|
+
def __enter__(self) -> "_Measure":
|
|
348
|
+
self._start = time.perf_counter()
|
|
349
|
+
return self
|
|
350
|
+
|
|
351
|
+
def __exit__(
|
|
352
|
+
self,
|
|
353
|
+
exc_type: Optional[type[BaseException]],
|
|
354
|
+
exc: Optional[BaseException],
|
|
355
|
+
tb: Optional[TracebackType],
|
|
356
|
+
) -> None:
|
|
357
|
+
latency_ms = int((time.perf_counter() - self._start) * 1000)
|
|
358
|
+
self.result = self._meter.record_call(
|
|
359
|
+
workflow_id=self.workflow_id,
|
|
360
|
+
provider=self.provider,
|
|
361
|
+
model=self.model,
|
|
362
|
+
input_tokens=self.input_tokens,
|
|
363
|
+
output_tokens=self.output_tokens,
|
|
364
|
+
cache_read_tokens=self.cache_read_tokens,
|
|
365
|
+
latency_ms=latency_ms,
|
|
366
|
+
status="error" if exc_type is not None else "ok",
|
|
367
|
+
is_retry=self.is_retry,
|
|
368
|
+
session_id=self.session_id,
|
|
369
|
+
prompt_id=self.prompt_id,
|
|
370
|
+
is_simulated=self.is_simulated,
|
|
371
|
+
)
|
|
372
|
+
# Never suppress the host's exception.
|
|
373
|
+
return None
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: margin-meter
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Tiny stdlib-only client that emits LLM call + outcome economics to a Margin ingest API.
|
|
5
|
+
Author: Margin
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/subhsubh24/Margin.ai
|
|
8
|
+
Project-URL: Source, https://github.com/subhsubh24/Margin.ai/tree/main/sdk/python
|
|
9
|
+
Keywords: llm,cost,observability,finops,margin
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
|
|
13
|
+
# margin-meter (Python SDK)
|
|
14
|
+
|
|
15
|
+
The tiny client a **Python** project imports to connect to Margin. It wraps your
|
|
16
|
+
LLM calls and records their outcomes, emitting each one **over HTTP** to a
|
|
17
|
+
Margin **ingest API** (`POST /api/ingest/calls` / `/api/ingest/outcomes`),
|
|
18
|
+
authenticated with a per-project ingest key. This is the customer-shaped path —
|
|
19
|
+
the same SDK a stranger drops in — not the in-process meter Margin runs on
|
|
20
|
+
itself.
|
|
21
|
+
|
|
22
|
+
- **Standalone + stdlib-only.** No dependency on Margin's server code and no
|
|
23
|
+
third-party deps. The default transport is `urllib`.
|
|
24
|
+
- **Fail-safe.** A failed emit returns an `IngestResult(ok=False, …)` instead of
|
|
25
|
+
crashing your app. Pass `raise_on_error=True` for strict/CI behaviour.
|
|
26
|
+
- **Provenance-honest.** `is_simulated` is carried through untouched; the written
|
|
27
|
+
`source` is forced server-side to your key's project — you cannot spoof
|
|
28
|
+
another project's economics.
|
|
29
|
+
|
|
30
|
+
## Install
|
|
31
|
+
|
|
32
|
+
Published as a git-installable subpath of the Margin repo (no PyPI account
|
|
33
|
+
needed pre-launch):
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install "git+https://github.com/subhsubh24/Margin.ai.git#subdirectory=sdk/python"
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Configure
|
|
40
|
+
|
|
41
|
+
Two environment variables — the deployed API base and your project's key:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
export MARGIN_INGEST_URL="https://margin-ai-rho.vercel.app"
|
|
45
|
+
export MARGIN_INGEST_KEY="mgk_…" # issued by the Margin owner (see below)
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The Margin owner issues your project a key with the provisioning CLI in the
|
|
49
|
+
Margin repo:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
python3 scripts/issue_ingest_key.py <your-project-slug>
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
The raw `mgk_…` key is shown **once** — only its hash is stored. Give it to your
|
|
56
|
+
project as `MARGIN_INGEST_KEY`.
|
|
57
|
+
|
|
58
|
+
## Use
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
from margin_meter import MarginMeter
|
|
62
|
+
|
|
63
|
+
meter = MarginMeter() # reads MARGIN_INGEST_URL + MARGIN_INGEST_KEY
|
|
64
|
+
|
|
65
|
+
# 1) Wrap the LLM call — latency is timed automatically, cost computed server-side.
|
|
66
|
+
with meter.measure(workflow_id="fit-scoring", provider="google",
|
|
67
|
+
model="gemini-2.5-flash") as m:
|
|
68
|
+
resp = call_the_model(...)
|
|
69
|
+
m.set_tokens(input_tokens=1200, output_tokens=300, cache_read_tokens=800)
|
|
70
|
+
|
|
71
|
+
# 2) Record the outcome it produced (the unit of productivity).
|
|
72
|
+
meter.record_outcome(workflow_id="fit-scoring", passed=True,
|
|
73
|
+
quality_score=0.94, quality_method="ground_truth")
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Or record a call directly (when you already have the token counts):
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
res = meter.record_call(
|
|
80
|
+
workflow_id="fit-scoring", provider="google", model="gemini-2.5-flash",
|
|
81
|
+
input_tokens=1200, output_tokens=300, cache_read_tokens=800,
|
|
82
|
+
)
|
|
83
|
+
if not res.ok:
|
|
84
|
+
log.warning("margin ingest failed: %s (%s)", res.error, res.status_code)
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## API
|
|
88
|
+
|
|
89
|
+
| Method | Emits to | Notes |
|
|
90
|
+
| --- | --- | --- |
|
|
91
|
+
| `record_call(...)` | `POST /api/ingest/calls` | cost computed from the pricing table when `cost_usd` omitted |
|
|
92
|
+
| `record_outcome(...)` | `POST /api/ingest/outcomes` | `quality_method` records HOW the score was graded |
|
|
93
|
+
| `measure(...)` | `record_call` on exit | context manager; times latency, `status="error"` on exception |
|
|
94
|
+
|
|
95
|
+
Every method returns an `IngestResult(ok, status_code, body, error)`. `ok` is
|
|
96
|
+
True only on HTTP 200; `body` holds `call_id`/`outcome_id` + `source`.
|
|
97
|
+
|
|
98
|
+
## Testing against a live app (no network)
|
|
99
|
+
|
|
100
|
+
The network boundary is a single `transport.post(path, json=..., headers=...)`
|
|
101
|
+
protocol, which a FastAPI `TestClient` satisfies exactly — so you can exercise
|
|
102
|
+
the real ingest endpoints hermetically by injecting one:
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
from fastapi.testclient import TestClient
|
|
106
|
+
import asgi
|
|
107
|
+
from margin_meter import MarginMeter
|
|
108
|
+
|
|
109
|
+
meter = MarginMeter(api_key=raw_key, transport=TestClient(asgi.app),
|
|
110
|
+
raise_on_error=True)
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
## Rate + validation bounds
|
|
114
|
+
|
|
115
|
+
Ingest is auth'd, validated, and rate-bounded server-side. A bad key → 401, a
|
|
116
|
+
malformed/implausible row → 422, and a full rolling per-project window → 429.
|
|
117
|
+
In fail-safe mode these come back as `IngestResult(ok=False, status_code=…)`.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
margin_meter
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "margin-meter"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Tiny stdlib-only client that emits LLM call + outcome economics to a Margin ingest API."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Margin" }]
|
|
13
|
+
keywords = ["llm", "cost", "observability", "finops", "margin"]
|
|
14
|
+
dependencies = []
|
|
15
|
+
|
|
16
|
+
[project.urls]
|
|
17
|
+
Homepage = "https://github.com/subhsubh24/Margin.ai"
|
|
18
|
+
Source = "https://github.com/subhsubh24/Margin.ai/tree/main/sdk/python"
|
|
19
|
+
|
|
20
|
+
[tool.setuptools.packages.find]
|
|
21
|
+
where = ["."]
|
|
22
|
+
include = ["margin_meter*"]
|