margin-meter 0.6.3__tar.gz → 0.6.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {margin_meter-0.6.3 → margin_meter-0.6.4}/PKG-INFO +1 -1
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/__init__.py +1 -1
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/client.py +41 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/pytest_plugin.py +10 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter.egg-info/PKG-INFO +1 -1
- {margin_meter-0.6.3 → margin_meter-0.6.4}/README.md +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/__main__.py +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/batching.py +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/braintrust.py +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/deepeval.py +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/doctor.py +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/evals.py +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/instrument.py +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/langfuse.py +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/langsmith.py +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/model_override.py +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/promptfoo.py +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/providers.py +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/ragas.py +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/substrate.py +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter/values.py +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter.egg-info/SOURCES.txt +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter.egg-info/dependency_links.txt +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter.egg-info/entry_points.txt +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/margin_meter.egg-info/top_level.txt +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/pyproject.toml +0 -0
- {margin_meter-0.6.3 → margin_meter-0.6.4}/setup.cfg +0 -0
|
@@ -106,4 +106,4 @@ __all__ = [
|
|
|
106
106
|
# closing the invoice-reconcile coverage gap is "ship the fix, wait for the seed's
|
|
107
107
|
# dependency refresh, re-run the reconcile", and a runtime that misreports which
|
|
108
108
|
# SDK metered a call makes that unanswerable.
|
|
109
|
-
__version__ = "0.6.
|
|
109
|
+
__version__ = "0.6.4"
|
|
@@ -1479,6 +1479,8 @@ class MarginMeter:
|
|
|
1479
1479
|
harness: str | None = None,
|
|
1480
1480
|
finish_reason: str | None = None,
|
|
1481
1481
|
provider_call_id: str | None = None,
|
|
1482
|
+
case_id: str | None = None,
|
|
1483
|
+
prompt_hash: str | None = None,
|
|
1482
1484
|
) -> IngestResult:
|
|
1483
1485
|
"""Emit one measured LLM call to ``POST /api/ingest/calls``.
|
|
1484
1486
|
|
|
@@ -1630,6 +1632,18 @@ class MarginMeter:
|
|
|
1630
1632
|
harness = harness[:64]
|
|
1631
1633
|
if harness is not None:
|
|
1632
1634
|
payload["harness"] = harness
|
|
1635
|
+
# WHICH EVAL CASE this call ran under (MAR-1782). Read from MARGIN_CASE_ID AT CALL
|
|
1636
|
+
# TIME, because the case changes per test inside one process; the pytest plugin sets
|
|
1637
|
+
# it around each test. Blank reads as absent. ⛔ It is NOT ``task_key``: that is the
|
|
1638
|
+
# task CLASS the parity gate buckets on, and a unique case id there would give every
|
|
1639
|
+
# call its own class. Only repeat detection reads this. A dimension, never money.
|
|
1640
|
+
case_id = (case_id or os.environ.get("MARGIN_CASE_ID") or "").strip()[:128] or None
|
|
1641
|
+
if case_id is not None:
|
|
1642
|
+
payload["case_id"] = case_id
|
|
1643
|
+
# sha256 hex of the request, never the text. Lets repeat detection tell an identical
|
|
1644
|
+
# re-send from a different request of similar size.
|
|
1645
|
+
if prompt_hash:
|
|
1646
|
+
payload["prompt_hash"] = prompt_hash
|
|
1633
1647
|
if cost_usd is not None:
|
|
1634
1648
|
payload["cost_usd"] = cost_usd
|
|
1635
1649
|
# Only send a non-default tier: a synchronous call omits it (the server
|
|
@@ -1856,6 +1870,7 @@ class MarginMeter:
|
|
|
1856
1870
|
substrate: str | None = None,
|
|
1857
1871
|
task_key: str | None = None,
|
|
1858
1872
|
provider_call_id: str | None = None,
|
|
1873
|
+
prompt_hash: str | None = None,
|
|
1859
1874
|
) -> IngestResult:
|
|
1860
1875
|
"""Meter one call by reading tokens straight off its provider ``response``.
|
|
1861
1876
|
|
|
@@ -1901,6 +1916,7 @@ class MarginMeter:
|
|
|
1901
1916
|
# forwarded as finish_reason (MAR-209). A signal, never a billing verdict.
|
|
1902
1917
|
finish_reason=usage.stop_reason,
|
|
1903
1918
|
is_simulated=is_simulated,
|
|
1919
|
+
prompt_hash=prompt_hash,
|
|
1904
1920
|
)
|
|
1905
1921
|
|
|
1906
1922
|
def wrap(
|
|
@@ -2020,6 +2036,7 @@ class MarginMeter:
|
|
|
2020
2036
|
is_retry=is_retry,
|
|
2021
2037
|
session_id=session_id,
|
|
2022
2038
|
prompt_id=prompt_id_from(kwargs),
|
|
2039
|
+
prompt_hash=prompt_hash_from(kwargs),
|
|
2023
2040
|
operation=operation,
|
|
2024
2041
|
substrate=call_substrate,
|
|
2025
2042
|
task_key=call_task_key,
|
|
@@ -2047,6 +2064,7 @@ class MarginMeter:
|
|
|
2047
2064
|
is_retry=is_retry,
|
|
2048
2065
|
session_id=session_id,
|
|
2049
2066
|
prompt_id=prompt_id_from(kwargs),
|
|
2067
|
+
prompt_hash=prompt_hash_from(kwargs),
|
|
2050
2068
|
operation=operation,
|
|
2051
2069
|
substrate=call_substrate,
|
|
2052
2070
|
task_key=call_task_key,
|
|
@@ -2103,6 +2121,7 @@ class MarginMeter:
|
|
|
2103
2121
|
is_retry=is_retry,
|
|
2104
2122
|
session_id=session_id,
|
|
2105
2123
|
prompt_id=prompt_id_from(kwargs),
|
|
2124
|
+
prompt_hash=prompt_hash_from(kwargs),
|
|
2106
2125
|
operation=operation,
|
|
2107
2126
|
substrate=call_substrate,
|
|
2108
2127
|
task_key=call_task_key,
|
|
@@ -2242,6 +2261,28 @@ class MarginMeter:
|
|
|
2242
2261
|
return _wrapped
|
|
2243
2262
|
|
|
2244
2263
|
|
|
2264
|
+
_PROMPT_FIELDS = ("messages", "input", "contents", "prompt", "system", "tools")
|
|
2265
|
+
|
|
2266
|
+
|
|
2267
|
+
def prompt_hash_from(kwargs: dict) -> str | None:
|
|
2268
|
+
"""sha256 hex of the request's prompt-bearing kwargs, or ``None``. Never the text.
|
|
2269
|
+
|
|
2270
|
+
Two calls that send the same messages to the same model hash alike, which is all
|
|
2271
|
+
repeat detection needs. Best-effort: an unserialisable request yields ``None``.
|
|
2272
|
+
"""
|
|
2273
|
+
import hashlib
|
|
2274
|
+
import json
|
|
2275
|
+
|
|
2276
|
+
part = {k: kwargs[k] for k in _PROMPT_FIELDS if kwargs.get(k) is not None}
|
|
2277
|
+
if not part:
|
|
2278
|
+
return None
|
|
2279
|
+
try:
|
|
2280
|
+
blob = json.dumps(part, sort_keys=True, default=repr).encode()
|
|
2281
|
+
except Exception:
|
|
2282
|
+
return None
|
|
2283
|
+
return hashlib.sha256(blob).hexdigest()
|
|
2284
|
+
|
|
2285
|
+
|
|
2245
2286
|
def prompt_id_from(kwargs: dict) -> str | None:
|
|
2246
2287
|
"""Pull a caller-supplied ``margin_prompt_id`` out of call kwargs, if present.
|
|
2247
2288
|
|
|
@@ -291,6 +291,12 @@ def pytest_runtest_setup(item):
|
|
|
291
291
|
nothing and is NOT warned about here. A warning raised in the setup phase becomes an ERROR
|
|
292
292
|
under ``-W error``, which would flip a passing test to a fail in a suite that only has the
|
|
293
293
|
SDK installed (the adapter, when enabled, reports a bad marker itself)."""
|
|
294
|
+
# Name the eval case for every call this test meters (MAR-1782), unconditionally like
|
|
295
|
+
# the marker below. Cleared in teardown so a call outside a test carries no stale case.
|
|
296
|
+
try:
|
|
297
|
+
os.environ["MARGIN_CASE_ID"] = str(item.nodeid)[:128]
|
|
298
|
+
except Exception:
|
|
299
|
+
pass
|
|
294
300
|
try:
|
|
295
301
|
value = value_from_marker(item)
|
|
296
302
|
except ValueError:
|
|
@@ -299,6 +305,10 @@ def pytest_runtest_setup(item):
|
|
|
299
305
|
item.user_properties.append((VALUE_PROPERTY, value))
|
|
300
306
|
|
|
301
307
|
|
|
308
|
+
def pytest_runtest_teardown(item):
|
|
309
|
+
os.environ.pop("MARGIN_CASE_ID", None)
|
|
310
|
+
|
|
311
|
+
|
|
302
312
|
def pytest_configure(config):
|
|
303
313
|
# Register the marker unconditionally so a suite that uses it without the
|
|
304
314
|
# flag does not trip PytestUnknownMarkWarning.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|