margin-meter 0.6.1__tar.gz → 0.6.2__tar.gz

{margin_meter-0.6.1/margin_meter.egg-info → margin_meter-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: margin-meter
-Version: 0.6.1
+Version: 0.6.2
 Summary: Tiny stdlib-only client that auto-instruments OpenAI/Anthropic/Gemini and emits LLM call + outcome economics to a Margin ingest API.
 Author: Margin
 License: MIT
@@ -12,6 +12,12 @@ Description-Content-Type: text/markdown
 
 # margin-meter (Python SDK)
 
+> **You probably do not need this to start.** Margin's first step installs nothing: sign in at
+> [trymargin.io/console](https://trymargin.io/console), connect the repository your agents live
+> in (or drop the folder), and press Execute. Each agent runs on the evals beside it and every
+> call is metered. This SDK is the **optional** route for metering your live production traffic,
+> once you want that as well ([`/start#production`](https://trymargin.io/start#production)).
+
 The tiny client a **Python** project imports to connect to Margin. It wraps your
 LLM calls and records their outcomes, emitting each one **over HTTP** to a
 Margin **ingest API** (`POST /api/ingest/calls` / `/api/ingest/outcomes`),

margin_meter-0.6.1/PKG-INFO → margin_meter-0.6.2/README.md

@@ -1,17 +1,11 @@
-Metadata-Version: 2.4
-Name: margin-meter
-Version: 0.6.1
-Summary: Tiny stdlib-only client that auto-instruments OpenAI/Anthropic/Gemini and emits LLM call + outcome economics to a Margin ingest API.
-Author: Margin
-License: MIT
-Project-URL: Homepage, https://trymargin.io
-Project-URL: Source, https://pypi.org/project/margin-meter/
-Keywords: llm,cost,observability,finops,margin
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-
 # margin-meter (Python SDK)
 
+> **You probably do not need this to start.** Margin's first step installs nothing: sign in at
+> [trymargin.io/console](https://trymargin.io/console), connect the repository your agents live
+> in (or drop the folder), and press Execute. Each agent runs on the evals beside it and every
+> call is metered. This SDK is the **optional** route for metering your live production traffic,
+> once you want that as well ([`/start#production`](https://trymargin.io/start#production)).
+
 The tiny client a **Python** project imports to connect to Margin. It wraps your
 LLM calls and records their outcomes, emitting each one **over HTTP** to a
 Margin **ingest API** (`POST /api/ingest/calls` / `/api/ingest/outcomes`),

{margin_meter-0.6.1 → margin_meter-0.6.2}/margin_meter/__init__.py

@@ -106,4 +106,4 @@ __all__ = [
 # closing the invoice-reconcile coverage gap is "ship the fix, wait for the seed's
 # dependency refresh, re-run the reconcile", and a runtime that misreports which
 # SDK metered a call makes that unanswerable.
-__version__ = "0.6.1"
+__version__ = "0.6.2"

{margin_meter-0.6.1 → margin_meter-0.6.2}/margin_meter/__main__.py

@@ -13,6 +13,10 @@ Subcommands:
 * ``langfuse``  — report Langfuse scores as outcomes (rung 2).
 * ``langfuse-calls`` — report Langfuse GENERATION observations as metered calls
   (rung 2, the AI-in plug — Langfuse is the one source that carries both).
+* ``evals``     — draft eval criteria from the connected repo, list them, or open a PR
+  with the accepted ones (MAR-1167).
+* ``mcp``       — the same eval actions as an MCP server on stdio.
+* ``values``    — push what each outcome is worth from your own records (MAR-1496).
 
 Kept as a dispatcher so the surface is stable as more adapters/diagnostics arrive.
 """
@@ -26,10 +30,12 @@ from margin_meter import (
     braintrust,
     deepeval,
     doctor,
+    evals,
     langfuse,
     langsmith,
     promptfoo,
     ragas,
+    values,
 )
 
 _USAGE = """margin-meter — Margin's metering SDK
@@ -55,6 +61,11 @@ commands:
   langfuse-calls
               report Langfuse GENERATION observations to Margin as metered calls
               (cost from calculatedTotalCost, else priced from tokens)
+  evals       draft | list | pr: draft eval criteria from the connected repo,
+              list them, or open a PR with the ones a person accepted
+  mcp         serve the evals actions as MCP tools over stdio
+  values      push what each outcome is worth from your own records
+              (event_id,value_usd CSV on stdin or a file; --source, --field)
 
 run `margin-meter <command> --help` for a command's options.
 """
@@ -82,6 +93,12 @@ def main(argv: Sequence[str] | None = None) -> int:
         return langfuse.main(rest)
     if command == "langfuse-calls":
         return langfuse.main_calls(rest)
+    if command == "evals":
+        return evals.main(rest)
+    if command == "mcp":
+        return evals.serve(rest)
+    if command == "values":
+        return values.main(rest)
     print(f"margin-meter: unknown command {command!r}\n", file=sys.stderr)
     print(_USAGE, file=sys.stderr)
     return 2

{margin_meter-0.6.1 → margin_meter-0.6.2}/margin_meter/braintrust.py

@@ -36,8 +36,8 @@ The mapping, one Braintrust result → one outcome:
   the grade is an LLM judge (rung 6 provenance); omitted otherwise. A bare
   ``model`` field is NOT used — in a Braintrust record it is usually the
   generation model under test, not the judge.
-* ``link``            ← the case ``id`` / ``span_id``, else a label from its
-  ``input``, else ``braintrust:<index>``.
+* ``link``            ← the case ``id`` / ``span_id``, else ``braintrust:<index>`` —
+  never its ``input`` (MAR-1548).
 * ``value_usd``       ← opt-in via ``metadata.margin_value`` — the only route to
   value-per-outcome (rung 8). Absent is the honest cost-per-outcome default,
   never defaulted to 0.
@@ -101,7 +101,7 @@ _JUDGE_MODEL_KEY = "margin_judge_model"
 # Mirror ``src/margin/ingest.MAX_VALUE_USD`` (kept as a literal so the SDK has no
 # dependency on the server package). A value above this is rejected by ingest and
 # would take the whole outcome row down with it, so the adapter drops the value
-# and keeps the row — the same treatment a negative value gets.
+# and keeps the row — the same treatment a malformed value gets.
 _MAX_VALUE_USD = 10_000_000.0
 
 # Model ids that name no real model — a custom/deterministic scorer leaves the
@@ -296,28 +296,22 @@ def value_from_result(result: dict) -> Optional[float]:
         raise ValueError(f"{_VALUE_KEY} must be a number, got {raw!r}")
     if not math.isfinite(val):
         raise ValueError(f"{_VALUE_KEY} must be finite, got {raw!r}")
-    if val < 0:
-        raise ValueError(f"{_VALUE_KEY} must be >= 0, got {val!r}")
-    if val > _MAX_VALUE_USD:
-        raise ValueError(f"{_VALUE_KEY} implausible (> {_MAX_VALUE_USD}), got {val!r}")
+    # A loss is a real value (MAR-1198): ingest takes any finite amount within +/-MAX.
+    if abs(val) > _MAX_VALUE_USD:
+        raise ValueError(f"{_VALUE_KEY} implausible (|value| > {_MAX_VALUE_USD}), got {val!r}")
     return val
 
 
 def link_from_result(result: dict, index: int) -> str:
-    """A traceable link back to the exact eval case: its id/span id, else a short
-    label built from the input, else the row index."""
+    """A traceable link back to the exact eval case: its id/span id, else the row
+    index.
+
+    ⛔ Never the case's ``input`` (MAR-1548): /security says no prompt reaches us.
+    """
     for key in ("id", "span_id", "root_span_id"):
         val = result.get(key)
         if isinstance(val, str) and val.strip():
             return val.strip()[:_MAX_LINK_LEN]
-    inp = result.get("input")
-    if isinstance(inp, str) and inp.strip():
-        return inp.strip()[:_MAX_LINK_LEN]
-    if isinstance(inp, (dict, list)):
-        try:
-            return json.dumps(inp, ensure_ascii=False, sort_keys=True)[:_MAX_LINK_LEN]
-        except (TypeError, ValueError):
-            pass
     return f"braintrust:{index}"
 
 

{margin_meter-0.6.1 → margin_meter-0.6.2}/margin_meter/client.py

@@ -46,6 +46,8 @@ import functools
 import json as _json
 import contextlib
 import logging
+import contextvars
+import inspect
 import os
 import random
 import time
@@ -623,6 +625,44 @@ def _retry_after_seconds(resp: Any) -> Optional[float]:
         return None
 
 
+_SDK_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+#: The calling file, set on the CALLING thread by a wrapper whose metering record may be written
+#: on another one (MAR-1230). litellm writes its success callbacks from a thread pool but runs
+#: them in a copy of the caller's context, so a value set here reaches that thread.
+_SITE_HINT: "contextvars.ContextVar[str]" = contextvars.ContextVar("margin_call_site", default="")
+#: Set as the hint by a callback that has no calling thread to read (MAR-1236): the record then
+#: names no file rather than whatever unrelated frame sits at the bottom of that thread. A
+#: litellm stream, sync or async, is logged after the call has returned, so it names no file.
+_NO_SITE = "\0"
+
+
+def _call_site() -> str:
+    """The file in the working tree that made this call, relative to it (MAR-1227): the
+    innermost frame that is neither this package nor an installed library. It rides on a SINK
+    line only, next to ``case``, so a harness can split a case's spend by the agent file that
+    spent it; ingest never sees it. Empty when no such frame exists."""
+    import sys
+    hint = _SITE_HINT.get()
+    if hint:
+        return "" if hint == _NO_SITE else hint
+    try:
+        root = os.path.realpath(os.getcwd())
+    except OSError:  # the working directory was removed: no site, but keep the call metered
+        return ""
+    frame = sys._getframe(1)
+    while frame is not None:
+        name = frame.f_code.co_filename
+        # "<frozen runpy>", "<string>": not files, though realpath would place them in cwd.
+        path = "" if name.startswith("<") else os.path.realpath(name)
+        if (path.startswith(root + os.sep) and not path.startswith(_SDK_DIR)
+                and "site-packages" not in path and "dist-packages" not in path):
+            return os.path.relpath(path, root).replace(os.sep, "/")
+        frame = frame.f_back
+    return ""
+
+
 def _backoff_seconds(attempt: int, retry_after: Optional[float] = None) -> float:
     """FULL jitter — ``random(0, base * 2**attempt)``, capped, or the server's wait.
 
@@ -1146,8 +1186,23 @@ class MarginMeter:
     def _write_sink(self, path: str, payload: dict) -> IngestResult:
         """Append one record to the JSONL sink for later forwarding. Fail-safe."""
         try:
+            record = {"path": path, "payload": payload}
+            # The pytest case this record was made in (MAR-1207). It rides on the SINK line
+            # only, never in the payload, so a harness can tell a case that called a model
+            # from one that did not; ingest never sees it.
+            case = os.environ.get("PYTEST_CURRENT_TEST") or ""
+            for phase in (" (setup)", " (call)", " (teardown)"):
+                if case.endswith(phase):
+                    case = case[: -len(phase)]
+            if case:
+                record["case"] = case
+            # The calling file rides on every sink line (MAR-1230), cased or not, so a face-off
+            # scoped to one agent can prove which of its calls were swapped.
+            site = _call_site()
+            if site:
+                record["site"] = site
             with open(self.ingest_file, "a", encoding="utf-8") as fh:
-                fh.write(_json.dumps({"path": path, "payload": payload}) + "\n")
+                fh.write(_json.dumps(record) + "\n")
         except Exception as exc:  # a sink write must never break the wrapped call
             self._count(path, ok=False, reason="sink_write")
             result = IngestResult(
@@ -1812,6 +1867,9 @@ class MarginMeter:
             cache_creation_1h_tokens=usage.cache_creation_1h_tokens,
             reasoning_tokens=usage.reasoning_tokens,
             web_search_requests=usage.web_search_requests,
+            # OpenRouter's billed cost for a fan-out slug (MAR-1309); None everywhere
+            # else, which leaves the server to price tokens off its table as before.
+            cost_usd=usage.billed_cost_usd,
             latency_ms=latency_ms,
             status=status,
             is_retry=is_retry,
@@ -2018,6 +2076,9 @@ class MarginMeter:
                         # accumulator max-merges it, so forward it or a streamed search
                         # drops its flat $10/1,000 charge and under-bills (MAR-499).
                         web_search_requests=usage.web_search_requests,
+                        # A streamed fan-out's billed cost (MAR-1309), merged off the
+                        # final usage chunk; None for every other stream.
+                        cost_usd=usage.billed_cost_usd,
                         latency_ms=latency_ms,
                         status="ok",
                         is_retry=is_retry,
@@ -2093,7 +2154,16 @@ class MarginMeter:
         # from the supply chain — even though the host, which awaits the coroutine,
         # keeps working. Detect a coroutine function and return an async wrapper that
         # AWAITS the real response before metering, timing the full await.
-        if asyncio.iscoroutinefunction(fn):
+        # ⛔ openai 2.x and anthropic DECORATE their async create (``required_args``), so the
+        # attribute itself is a plain function returning a coroutine and
+        # ``iscoroutinefunction`` says False. Measured 2026-09-25 (MAR-1236): every
+        # AsyncOpenAI chat call and every AsyncAnthropic messages call went unmetered. The
+        # decorators keep ``__wrapped__``, so the real ``async def`` is found by unwrapping.
+        try:
+            real = inspect.unwrap(fn)
+        except ValueError:  # a __wrapped__ cycle (mock.call): classify the callable as given
+            real = fn
+        if asyncio.iscoroutinefunction(fn) or inspect.iscoroutinefunction(real):
 
             @functools.wraps(fn)
             async def _awrapped(*args: Any, **kwargs: Any) -> Any:

{margin_meter-0.6.1 → margin_meter-0.6.2}/margin_meter/deepeval.py

@@ -56,6 +56,9 @@ _MAX_JUDGE_ID_LEN = 128
 
 _VALUE_KEY = "margin_value"
 
+# Mirror ``src/margin/ingest.MAX_VALUE_USD``: ingest refuses a value past this either way.
+_MAX_VALUE_USD = 10_000_000.0
+
 # ``evaluationModel`` values that name no real model — a deterministic/custom
 # metric leaves it empty or a placeholder. Normalized lower-case for the check.
 _NON_MODELS = frozenset({"", "n/a", "na", "none", "null"})
@@ -158,8 +161,9 @@ def value_from_case(case: dict) -> Optional[float]:
         raise ValueError(f"{_VALUE_KEY} must be a number, got {raw!r}")
     if not math.isfinite(val):
         raise ValueError(f"{_VALUE_KEY} must be finite, got {raw!r}")
-    if val < 0:
-        raise ValueError(f"{_VALUE_KEY} must be >= 0, got {val!r}")
+    # A loss is a real value (MAR-1198): ingest takes any finite amount within +/-MAX.
+    if abs(val) > _MAX_VALUE_USD:
+        raise ValueError(f"{_VALUE_KEY} implausible (|value| > {_MAX_VALUE_USD}), got {val!r}")
     return val
 
 

margin_meter-0.6.2/margin_meter/evals.py

@@ -0,0 +1,218 @@
+"""Evals from the command line and from MCP (MAR-1167).
+
+The console is one door to the eval builder. This module opens two more for developers
+who would rather stay in a terminal or an agent:
+
+* ``margin-meter evals draft``  — draft criteria from the project's connected repository
+* ``margin-meter evals list``   — the project's criteria, drafts and accepted
+* ``margin-meter evals pr``     — open a PR carrying the ACCEPTED criteria into the repo
+* ``margin-meter mcp``          — the same three as MCP tools over stdio
+
+All of them call the same project-scoped HTTP routes the console calls, authenticated with
+the project's ingest key (``MARGIN_INGEST_KEY``) against ``MARGIN_INGEST_URL``. Nothing here
+can accept a criterion: acceptance needs a named person in a console session, so a PR opened
+from a script still carries only rules a person on the customer's side agreed to.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+from typing import Any, Callable, Optional, Sequence
+
+DRAFT_PATH = "/api/console/evals/from-repo"
+LIST_PATH = "/api/console/evals"
+PR_PATH = "/api/console/evals/pull-request"
+TIMEOUT_S = 180.0  # a draft reads and scans the whole repo; it is not an ingest call
+
+# (method, path, key) -> (status, body). Tests inject a TestClient-backed one.
+Transport = Callable[[str, str, str], "tuple[int, Any]"]
+
+
+def http_transport(base_url: str) -> Transport:
+    base = base_url.rstrip("/")
+
+    def send(method: str, path: str, key: str) -> "tuple[int, Any]":
+        req = urllib.request.Request(
+            base + path,
+            data=b"{}" if method == "POST" else None,
+            method=method,
+            headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=TIMEOUT_S) as resp:
+                raw, status = resp.read(), resp.status
+        except urllib.error.HTTPError as exc:
+            try:
+                raw = exc.read()
+            except Exception:  # an HTTPError without a body must not mask the status
+                raw = b""
+            status = exc.code
+        except (urllib.error.URLError, OSError) as exc:  # unreachable: report, never crash
+            raise EvalsError(f"could not reach {base}: {exc}") from exc
+        try:
+            return status, json.loads(raw or b"null")
+        except ValueError:
+            return status, raw.decode("utf-8", "replace")
+
+    return send
+
+
+class EvalsError(RuntimeError):
+    """The API refused; the message is the API's own reason."""
+
+
+def _call(send: Transport, method: str, path: str, key: str) -> dict:
+    status, body = send(method, path, key)
+    if status >= 400:
+        detail = body.get("detail") if isinstance(body, dict) else body
+        raise EvalsError(f"HTTP {status}: {detail}")
+    return body if isinstance(body, dict) else {"result": body}
+
+
+def draft(send: Transport, key: str) -> dict:
+    return _call(send, "POST", DRAFT_PATH, key)
+
+
+def list_criteria(send: Transport, key: str) -> dict:
+    return _call(send, "GET", LIST_PATH, key)
+
+
+def open_pr(send: Transport, key: str) -> dict:
+    return _call(send, "POST", PR_PATH, key)
+
+
+_ACTIONS = {"draft": draft, "list": list_criteria, "pr": open_pr}
+
+
+def _config(url: Optional[str], key: Optional[str]) -> "tuple[str, str]":
+    url = url or os.environ.get("MARGIN_INGEST_URL") or ""
+    key = key or os.environ.get("MARGIN_INGEST_KEY") or ""
+    if not url or not key:
+        raise EvalsError("set MARGIN_INGEST_URL and MARGIN_INGEST_KEY (or pass --url/--key)")
+    return url, key
+
+
+def _print_human(action: str, out: dict) -> None:
+    if action == "pr":
+        verb = "updated" if out.get("reused") else "opened"
+        print(f"{verb} PR #{out.get('number')} on {out.get('repo')}: {out.get('url')}")
+        return
+    for c in out.get("criteria", []):
+        where = ", ".join(c.get("evidence") or []) or "-"
+        print(f"[{c.get('label', c.get('status'))}] {c.get('text')}  ({where})")
+    for h in out.get("harnesses", []) or []:
+        print(f"existing harness: {h}")
+    if out.get("why"):
+        print(out["why"])
+
+
+def main(argv: Optional[Sequence[str]] = None, send: Optional[Transport] = None) -> int:
+    p = argparse.ArgumentParser(prog="margin-meter evals",
+                                description="Draft, list and ship eval criteria.")
+    p.add_argument("action", choices=sorted(_ACTIONS))
+    p.add_argument("--url")
+    p.add_argument("--key")
+    p.add_argument("--json", action="store_true", help="print the raw API response")
+    a = p.parse_args(argv)
+    try:
+        url, key = _config(a.url, a.key)
+        out = _ACTIONS[a.action](send or http_transport(url), key)
+    except EvalsError as exc:
+        print(f"margin-meter evals: {exc}", file=sys.stderr)
+        return 1
+    if a.json:
+        print(json.dumps(out, indent=2))
+    else:
+        _print_human(a.action, out)
+    return 0
+
+
+# --------------------------------------------------------------------------- #
+# MCP over stdio: newline-delimited JSON-RPC 2.0, the three actions as tools.
+# --------------------------------------------------------------------------- #
+PROTOCOL_VERSION = "2025-06-18"
+_TOOLS = [
+    {"name": "margin_evals_draft_from_repo",
+     "description": ("Read the project's connected GitHub repository and draft eval criteria "
+                     "from the instructions its agent is given. Every criterion cites the "
+                     "path:line it came from and stays a draft until a person accepts it "
+                     "in the Margin console."),
+     "annotations": {"readOnlyHint": False, "destructiveHint": False}},
+    {"name": "margin_evals_list",
+     "description": "List the project's eval criteria with their status (draft or accepted).",
+     "annotations": {"readOnlyHint": True}},
+    {"name": "margin_evals_open_pr",
+     "description": ("Open (or update) a pull request that adds the project's ACCEPTED "
+                     "criteria to margin/evals.json in its repository. Refused when nothing "
+                     "is accepted yet. The customer reviews and merges it."),
+     "annotations": {"readOnlyHint": False, "destructiveHint": False}},
+]
+_TOOL_ACTION = {"margin_evals_draft_from_repo": draft, "margin_evals_list": list_criteria,
+                "margin_evals_open_pr": open_pr}
+for _t in _TOOLS:
+    _t["inputSchema"] = {"type": "object", "properties": {}, "additionalProperties": False}
+
+
+def handle(msg: dict, send: Transport, key: str) -> Optional[dict]:
+    """One JSON-RPC message → its response (None for a notification)."""
+    if not isinstance(msg, dict):
+        return {"jsonrpc": "2.0", "id": None,
+                "error": {"code": -32600, "message": "invalid request"}}
+    if "id" not in msg:
+        return None
+    mid, method = msg["id"], msg.get("method")
+    if method == "initialize":
+        result: Any = {"protocolVersion": PROTOCOL_VERSION, "capabilities": {"tools": {}},
+                       "serverInfo": {"name": "margin-evals", "version": "1"}}
+    elif method == "tools/list":
+        result = {"tools": _TOOLS}
+    elif method == "tools/call":
+        name = (msg.get("params") or {}).get("name")
+        action = _TOOL_ACTION.get(name)
+        if action is None:
+            return {"jsonrpc": "2.0", "id": mid,
+                    "error": {"code": -32602, "message": f"unknown tool {name!r}"}}
+        try:
+            text, err = json.dumps(action(send, key), indent=2), False
+        except EvalsError as exc:
+            text, err = str(exc), True
+        result = {"content": [{"type": "text", "text": text}], "isError": err}
+    elif method == "ping":
+        result = {}
+    else:
+        return {"jsonrpc": "2.0", "id": mid,
+                "error": {"code": -32601, "message": f"method not found: {method}"}}
+    return {"jsonrpc": "2.0", "id": mid, "result": result}
+
+
+def serve(argv: Optional[Sequence[str]] = None) -> int:
+    p = argparse.ArgumentParser(prog="margin-meter mcp",
+                                description="Margin's eval tools as an MCP server on stdio.")
+    p.add_argument("--url")
+    p.add_argument("--key")
+    a = p.parse_args(argv)
+    try:
+        url, key = _config(a.url, a.key)
+    except EvalsError as exc:
+        print(f"margin-meter mcp: {exc}", file=sys.stderr)
+        return 1
+    send = http_transport(url)
+    for line in sys.stdin:
+        if not line.strip():
+            continue
+        try:
+            msg = json.loads(line)
+        except ValueError:
+            out: Optional[dict] = {"jsonrpc": "2.0", "id": None,
+                                   "error": {"code": -32700, "message": "parse error"}}
+        else:
+            out = handle(msg, send, key)
+        if out is not None:
+            sys.stdout.write(json.dumps(out) + "\n")
+            sys.stdout.flush()
+    return 0