metergraph 0.2.0__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. {metergraph-0.2.0 → metergraph-0.2.1}/PKG-INFO +4 -1
  2. {metergraph-0.2.0 → metergraph-0.2.1}/pyproject.toml +2 -2
  3. {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/_capture.py +102 -51
  4. {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/_config.py +13 -1
  5. metergraph-0.2.1/src/metergraph/_failure_log.py +44 -0
  6. {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/_transport.py +23 -1
  7. {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/_version.py +1 -1
  8. {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph.egg-info/PKG-INFO +4 -1
  9. {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph.egg-info/SOURCES.txt +4 -1
  10. metergraph-0.2.1/src/metergraph.egg-info/requires.txt +6 -0
  11. metergraph-0.2.1/tests/test_real_client_integration.py +226 -0
  12. {metergraph-0.2.0 → metergraph-0.2.1}/tests/test_sdk.py +294 -1
  13. metergraph-0.2.1/tests/test_seam_reality.py +35 -0
  14. metergraph-0.2.0/src/metergraph.egg-info/requires.txt +0 -3
  15. {metergraph-0.2.0 → metergraph-0.2.1}/README.md +0 -0
  16. {metergraph-0.2.0 → metergraph-0.2.1}/setup.cfg +0 -0
  17. {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/__init__.py +0 -0
  18. {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/_context.py +0 -0
  19. {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/_template.py +0 -0
  20. {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/_track.py +0 -0
  21. {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph.egg-info/dependency_links.txt +0 -0
  22. {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: metergraph
3
- Version: 0.2.0
3
+ Version: 0.2.1
4
4
  Summary: Fire-and-forget LLM spend capture for Metergraph
5
5
  Author: Pioneer Square Labs
6
6
  License-Expression: Apache-2.0
@@ -15,6 +15,9 @@ Requires-Python: >=3.10
15
15
  Description-Content-Type: text/markdown
16
16
  Provides-Extra: dev
17
17
  Requires-Dist: pytest>=8; extra == "dev"
18
+ Requires-Dist: openai>=2.50.0; extra == "dev"
19
+ Requires-Dist: anthropic>=0.40; extra == "dev"
20
+ Requires-Dist: google-genai>=1; extra == "dev"
18
21
 
19
22
  # metergraph (Python)
20
23
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "metergraph"
3
- version = "0.2.0"
3
+ version = "0.2.1"
4
4
  description = "Fire-and-forget LLM spend capture for Metergraph"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.10"
@@ -20,7 +20,7 @@ Repository = "https://github.com/PioneerSquareLabs/metergraphsdk"
20
20
  Issues = "https://github.com/PioneerSquareLabs/metergraphsdk/issues"
21
21
 
22
22
  [project.optional-dependencies]
23
- dev = ["pytest>=8"]
23
+ dev = ["pytest>=8", "openai>=2.50.0", "anthropic>=0.40", "google-genai>=1"]
24
24
 
25
25
  [build-system]
26
26
  requires = ["setuptools>=68"]
@@ -1082,61 +1082,85 @@ def _finish_or_stream(
1082
1082
  return result
1083
1083
 
1084
1084
 
1085
- def wrap(client: Any, *, provider: str | None = None) -> Any:
1086
- """Patch supported resource methods on an OpenAI, Anthropic, or Google client."""
1087
- if provider is None:
1088
- if hasattr(getattr(client, "models", None), "generate_content"):
1089
- provider = "google"
1090
- elif hasattr(client, "chat") or hasattr(client, "responses"):
1091
- provider = "openai"
1092
- else:
1093
- provider = "anthropic"
1094
- seams: list[tuple[Any, str, str]] = []
1095
- if provider == "google":
1096
- for models in (
1097
- getattr(client, "models", None),
1098
- getattr(getattr(client, "aio", None), "models", None),
1099
- ):
1100
- if models is not None:
1101
- seams.extend(
1102
- (
1103
- (models, "generate_content", "models.generate_content"),
1104
- (
1105
- models,
1106
- "generate_content_stream",
1107
- "models.generate_content.stream",
1108
- ),
1109
- )
1110
- )
1111
- chat = getattr(getattr(client, "chat", None), "completions", None)
1112
- if chat is not None:
1113
- seams.append((chat, "create", "chat.completions"))
1114
- seams.append((chat, "parse", "chat.completions.parse"))
1115
- beta_chat = getattr(getattr(getattr(client, "beta", None), "chat", None), "completions", None)
1116
- if beta_chat is not None:
1117
- seams.append((beta_chat, "parse", "chat.completions.parse"))
1118
- responses = getattr(client, "responses", None)
1119
- if responses is not None:
1120
- seams.extend(
1121
- (
1122
- (responses, "create", "responses"),
1123
- (responses, "stream", "responses.stream"),
1124
- )
1125
- )
1126
- messages = getattr(client, "messages", None)
1127
- if messages is not None:
1128
- seams.extend(
1129
- ((messages, "create", "messages"), (messages, "stream", "messages.stream"))
1130
- )
1131
- patched = sum(
1132
- _patch(owner, method, provider, endpoint) for owner, method, endpoint in seams
1133
- )
1085
+ @dataclass(frozen=True)
1086
+ class Seam:
1087
+ """A single instrumentable method, addressed by a dotted attribute path
1088
+ from the client root (e.g. "beta.chat.completions")."""
1089
+
1090
+ path: str
1091
+ method: str
1092
+ endpoint: str
1093
+
1094
+
1095
+ OPENAI_SEAMS: tuple[Seam, ...] = (
1096
+ Seam("chat.completions", "create", "chat.completions"),
1097
+ Seam("chat.completions", "parse", "chat.completions.parse"),
1098
+ Seam("beta.chat.completions", "parse", "chat.completions.parse"),
1099
+ Seam("responses", "create", "responses"),
1100
+ Seam("responses", "stream", "responses.stream"),
1101
+ Seam("responses", "parse", "responses.parse"),
1102
+ Seam("beta.responses", "create", "responses"),
1103
+ # Note: client.beta.responses has no .parse method (verified against
1104
+ # openai==2.50.0) — do not add one here without re-verifying first.
1105
+ )
1106
+
1107
+ ANTHROPIC_SEAMS: tuple[Seam, ...] = (
1108
+ Seam("messages", "create", "messages"),
1109
+ Seam("messages", "stream", "messages.stream"),
1110
+ )
1111
+
1112
+ GOOGLE_SEAMS: tuple[Seam, ...] = (
1113
+ Seam("models", "generate_content", "models.generate_content"),
1114
+ Seam("models", "generate_content_stream", "models.generate_content.stream"),
1115
+ Seam("aio.models", "generate_content", "models.generate_content"),
1116
+ Seam("aio.models", "generate_content_stream", "models.generate_content.stream"),
1117
+ )
1118
+
1119
+ SEAM_TABLES: dict[str, tuple[Seam, ...]] = {
1120
+ "openai": OPENAI_SEAMS,
1121
+ "anthropic": ANTHROPIC_SEAMS,
1122
+ "google": GOOGLE_SEAMS,
1123
+ }
1124
+
1125
+
1126
+ def _detect_provider(client: Any) -> str:
1127
+ if hasattr(getattr(client, "models", None), "generate_content"):
1128
+ return "google"
1129
+ if hasattr(client, "chat") or hasattr(client, "responses"):
1130
+ return "openai"
1131
+ return "anthropic"
1132
+
1133
+
1134
+ def _resolve(client: Any, path: str) -> Any:
1135
+ obj = client
1136
+ for part in path.split("."):
1137
+ obj = getattr(obj, part, None)
1138
+ if obj is None:
1139
+ return None
1140
+ return obj
1141
+
1142
+
1143
+ def _apply_seams(client: Any, provider: str) -> list[str]:
1144
+ patched: list[str] = []
1145
+ for seam in SEAM_TABLES.get(provider, ()):
1146
+ try:
1147
+ owner = _resolve(client, seam.path)
1148
+ except Exception:
1149
+ continue # a pathological client property must never break wrap()
1150
+ if owner is not None and _patch(owner, seam.method, provider, seam.endpoint):
1151
+ patched.append(f"{seam.path}.{seam.method}")
1152
+ return patched
1153
+
1154
+
1155
+ def _apply_batch_extras(client: Any, provider: str) -> int:
1156
+ patched = 0
1134
1157
  if provider == "openai":
1135
1158
  files = getattr(client, "files", None)
1136
1159
  if files is not None:
1137
1160
  patched += int(_patch_openai_batch_content(files, "content"))
1138
1161
  patched += int(_patch_openai_batch_content(files, "retrieve_content"))
1139
1162
  elif provider == "anthropic":
1163
+ messages = getattr(client, "messages", None)
1140
1164
  batch_owners = [getattr(messages, "batches", None)]
1141
1165
  beta_messages = getattr(getattr(client, "beta", None), "messages", None)
1142
1166
  batch_owners.append(getattr(beta_messages, "batches", None))
@@ -1145,6 +1169,33 @@ def wrap(client: Any, *, provider: str | None = None) -> Any:
1145
1169
  for owner in batch_owners
1146
1170
  if owner is not None
1147
1171
  )
1148
- if not patched:
1149
- log.warning("Metergraph found no supported methods on %s client", provider)
1172
+ return patched
1173
+
1174
+
1175
+ def wrap(client: Any, *, provider: str | None = None) -> Any:
1176
+ """Patch supported resource methods on an OpenAI, Anthropic, or Google client.
1177
+
1178
+ Never raises: an unrecognized client shape, or an exception while probing
1179
+ it, results in an unmodified, uninstrumented client — not a crash.
1180
+ """
1181
+ try:
1182
+ resolved_provider = provider or _detect_provider(client)
1183
+ patched = _apply_seams(client, resolved_provider)
1184
+ patched_count = len(patched) + _apply_batch_extras(client, resolved_provider)
1185
+ if not patched_count:
1186
+ log.warning(
1187
+ "Metergraph found no supported methods on %s client", resolved_provider
1188
+ )
1189
+ else:
1190
+ log.info(
1191
+ "Metergraph patched %d seam(s) on %s client: %s",
1192
+ patched_count,
1193
+ resolved_provider,
1194
+ ", ".join(patched) or "(batch-only)",
1195
+ )
1196
+ except Exception:
1197
+ log.warning(
1198
+ "Metergraph wrap() failed; client is unmodified and uninstrumented",
1199
+ exc_info=True,
1200
+ )
1150
1201
  return client
@@ -11,6 +11,8 @@ import urllib.request
11
11
  from collections.abc import Mapping
12
12
  from typing import Any
13
13
 
14
+ from ._failure_log import FailureLogger
15
+
14
16
 
15
17
  log = logging.getLogger("metergraph")
16
18
 
@@ -75,6 +77,7 @@ class ConfigPoller:
75
77
  self._routes: dict[str, dict[str, Any]] = {}
76
78
  self._last_success = 0.0
77
79
  self._lock = threading.Lock()
80
+ self._failure_log = FailureLogger()
78
81
  self._stop = threading.Event()
79
82
  self._thread = threading.Thread(
80
83
  target=self._run, name="metergraph-config", daemon=True
@@ -124,8 +127,17 @@ class ConfigPoller:
124
127
  "Metergraph config authentication failed; using default models"
125
128
  )
126
129
  self._stop.set()
130
+ return False
131
+ self._failure_log.report(
132
+ "config_poll_error",
133
+ f"config poll to {self._url} failed with HTTP {exc.code}",
134
+ )
127
135
  return False
128
- except Exception:
136
+ except Exception as exc:
137
+ self._failure_log.report(
138
+ "config_poll_error",
139
+ f"config poll to {self._url} failed: {type(exc).__name__}: {exc}",
140
+ )
129
141
  return False
130
142
 
131
143
  def model_for(self, route: str, default: str, session_key: str | None) -> str:
@@ -0,0 +1,44 @@
1
+ """Rate-limited failure logging.
2
+
3
+ Logs the first occurrence of a failure kind immediately, then suppresses
4
+ repeats within a quiet window and reports how many were suppressed on the
5
+ next log line for that kind. Keeps log volume bounded under sustained
6
+ failure without ever going completely silent.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import logging
12
+ import time
13
+ from typing import Callable
14
+
15
+
16
+ log = logging.getLogger("metergraph")
17
+
18
+
19
+ class FailureLogger:
20
+ def __init__(
21
+ self,
22
+ quiet_seconds: float = 60.0,
23
+ *,
24
+ clock: Callable[[], float] = time.monotonic,
25
+ ) -> None:
26
+ self._quiet_seconds = quiet_seconds
27
+ self._clock = clock
28
+ self._last_logged: dict[str, float] = {}
29
+ self._suppressed: dict[str, int] = {}
30
+
31
+ def report(self, kind: str, message: str, level: int = logging.WARNING) -> None:
32
+ now = self._clock()
33
+ last = self._last_logged.get(kind)
34
+ if last is not None and now - last < self._quiet_seconds:
35
+ self._suppressed[kind] = self._suppressed.get(kind, 0) + 1
36
+ return
37
+ suppressed = self._suppressed.pop(kind, 0)
38
+ suffix = (
39
+ f" ({suppressed} more suppressed in the last {self._quiet_seconds:.0f}s)"
40
+ if suppressed
41
+ else ""
42
+ )
43
+ log.log(level, "metergraph: %s%s", message, suffix)
44
+ self._last_logged[kind] = now
@@ -13,6 +13,7 @@ import urllib.error
13
13
  import urllib.request
14
14
  from typing import Any
15
15
 
16
+ from ._failure_log import FailureLogger
16
17
  from ._version import SDK_VERSION
17
18
 
18
19
 
@@ -43,6 +44,7 @@ class Writer:
43
44
  self._errors = 0
44
45
  self._backoff = 1.0
45
46
  self._retry_at = 0.0
47
+ self._failure_log = FailureLogger()
46
48
  self._thread = self._new_thread()
47
49
  self._thread.start()
48
50
  if hasattr(os, "register_at_fork"):
@@ -144,11 +146,31 @@ class Writer:
144
146
  log.warning(
145
147
  "Metergraph authentication failed; capture disabled for this process"
146
148
  )
149
+ elif exc.code == 413 and len(rows) > 1:
150
+ midpoint = len(rows) // 2
151
+ left = self._deliver(rows[:midpoint])
152
+ right = self._deliver(rows[midpoint:])
153
+ return left and right
154
+ elif exc.code in (400, 404, 413, 422):
155
+ self._dropped += len(rows)
156
+ self._failure_log.report(
157
+ "client_error",
158
+ f"ingest rejected batch with HTTP {exc.code} against {self._url}; "
159
+ "dropping this batch (payload-specific, not a process-wide failure)",
160
+ )
147
161
  else:
148
162
  self._failed(len(rows))
163
+ self._failure_log.report(
164
+ "server_error",
165
+ f"ingest request failed with HTTP {exc.code} against {self._url}",
166
+ )
149
167
  return False
150
- except Exception:
168
+ except Exception as exc:
151
169
  self._failed(len(rows))
170
+ self._failure_log.report(
171
+ "transport_error",
172
+ f"ingest request to {self._url} failed: {type(exc).__name__}: {exc}",
173
+ )
152
174
  return False
153
175
 
154
176
  def _failed(self, count: int) -> None:
@@ -8,4 +8,4 @@ import importlib.metadata
8
8
  try:
9
9
  SDK_VERSION = importlib.metadata.version("metergraph")
10
10
  except Exception:
11
- SDK_VERSION = "0.2.0"
11
+ SDK_VERSION = "0.2.1"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: metergraph
3
- Version: 0.2.0
3
+ Version: 0.2.1
4
4
  Summary: Fire-and-forget LLM spend capture for Metergraph
5
5
  Author: Pioneer Square Labs
6
6
  License-Expression: Apache-2.0
@@ -15,6 +15,9 @@ Requires-Python: >=3.10
15
15
  Description-Content-Type: text/markdown
16
16
  Provides-Extra: dev
17
17
  Requires-Dist: pytest>=8; extra == "dev"
18
+ Requires-Dist: openai>=2.50.0; extra == "dev"
19
+ Requires-Dist: anthropic>=0.40; extra == "dev"
20
+ Requires-Dist: google-genai>=1; extra == "dev"
18
21
 
19
22
  # metergraph (Python)
20
23
 
@@ -4,6 +4,7 @@ src/metergraph/__init__.py
4
4
  src/metergraph/_capture.py
5
5
  src/metergraph/_config.py
6
6
  src/metergraph/_context.py
7
+ src/metergraph/_failure_log.py
7
8
  src/metergraph/_template.py
8
9
  src/metergraph/_track.py
9
10
  src/metergraph/_transport.py
@@ -13,4 +14,6 @@ src/metergraph.egg-info/SOURCES.txt
13
14
  src/metergraph.egg-info/dependency_links.txt
14
15
  src/metergraph.egg-info/requires.txt
15
16
  src/metergraph.egg-info/top_level.txt
16
- tests/test_sdk.py
17
+ tests/test_real_client_integration.py
18
+ tests/test_sdk.py
19
+ tests/test_seam_reality.py
@@ -0,0 +1,6 @@
1
+
2
+ [dev]
3
+ pytest>=8
4
+ openai>=2.50.0
5
+ anthropic>=0.40
6
+ google-genai>=1
@@ -0,0 +1,226 @@
1
+ """Wrap the real, unmodified provider SDK clients and drive a call through
2
+ their actual request-building/response-parsing code — with the network
3
+ replaced by a mocked transport, not the SDK itself.
4
+
5
+ test_seam_reality.py proves a seam *exists* on the real client. The
6
+ behavioral tests in test_sdk.py prove wrap() *works*, but only against a
7
+ hand-built fake that mimics the real client's shape. Neither proves that
8
+ wrapping the real client and calling a real method actually produces a
9
+ captured row — which is exactly the gap that let the original
10
+ chat.completions.parse capture bug ship unnoticed. These tests close it,
11
+ without needing live API keys or network access.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import asyncio
17
+ import json
18
+
19
+ import httpx
20
+ import metergraph
21
+ from anthropic import AsyncAnthropic
22
+ from google import genai
23
+ from metergraph import _capture
24
+ from metergraph._capture import Options, Runtime
25
+ from openai import AsyncOpenAI
26
+ from pydantic import BaseModel
27
+
28
+
29
+ class Rows:
30
+ def __init__(self):
31
+ self.rows = []
32
+
33
+ def enqueue(self, row):
34
+ self.rows.append(row)
35
+ return True
36
+
37
+
38
+ class Answer(BaseModel):
39
+ text: str
40
+
41
+
42
+ def test_wrap_captures_openai_parse_through_a_real_client(tmp_path):
43
+ """.parse() is the exact seam the original production bug missed."""
44
+ rows = Rows()
45
+ _capture.set_runtime(Runtime(rows, Options(app_root=str(tmp_path))))
46
+
47
+ def handler(request: httpx.Request) -> httpx.Response:
48
+ return httpx.Response(
49
+ 200,
50
+ json={
51
+ "id": "chatcmpl-test",
52
+ "object": "chat.completion",
53
+ "created": 0,
54
+ "model": "gpt-4o-mini",
55
+ "choices": [
56
+ {
57
+ "index": 0,
58
+ "message": {
59
+ "role": "assistant",
60
+ "content": json.dumps({"text": "hi"}),
61
+ },
62
+ "finish_reason": "stop",
63
+ }
64
+ ],
65
+ "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
66
+ },
67
+ )
68
+
69
+ client = AsyncOpenAI(
70
+ api_key="test",
71
+ http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
72
+ )
73
+ metergraph.wrap(client, provider="openai")
74
+
75
+ async def run():
76
+ with metergraph.route("real-client-test"):
77
+ return await client.chat.completions.parse(
78
+ model="gpt-4o-mini",
79
+ messages=[{"role": "user", "content": "hi"}],
80
+ response_format=Answer,
81
+ )
82
+
83
+ response = asyncio.run(run())
84
+
85
+ assert response.choices[0].message.parsed == Answer(text="hi")
86
+ assert len(rows.rows) == 1
87
+ row = rows.rows[0]
88
+ assert row["endpoint"] == "chat.completions.parse"
89
+ assert row["provider"] == "openai"
90
+ assert row["model"] == "gpt-4o-mini"
91
+ assert row["input_tokens"] == 5
92
+ assert row["output_tokens"] == 2
93
+ _capture.set_runtime(None)
94
+
95
+
96
+ def test_wrap_captures_openai_create_through_a_real_client(tmp_path):
97
+ rows = Rows()
98
+ _capture.set_runtime(Runtime(rows, Options(app_root=str(tmp_path))))
99
+
100
+ def handler(request: httpx.Request) -> httpx.Response:
101
+ return httpx.Response(
102
+ 200,
103
+ json={
104
+ "id": "chatcmpl-test2",
105
+ "object": "chat.completion",
106
+ "created": 0,
107
+ "model": "gpt-4o-mini",
108
+ "choices": [
109
+ {
110
+ "index": 0,
111
+ "message": {"role": "assistant", "content": "hi"},
112
+ "finish_reason": "stop",
113
+ }
114
+ ],
115
+ "usage": {"prompt_tokens": 4, "completion_tokens": 1, "total_tokens": 5},
116
+ },
117
+ )
118
+
119
+ client = AsyncOpenAI(
120
+ api_key="test",
121
+ http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
122
+ )
123
+ metergraph.wrap(client, provider="openai")
124
+
125
+ async def run():
126
+ return await client.chat.completions.create(
127
+ model="gpt-4o-mini", messages=[{"role": "user", "content": "hi"}]
128
+ )
129
+
130
+ response = asyncio.run(run())
131
+
132
+ assert response.choices[0].message.content == "hi"
133
+ assert len(rows.rows) == 1
134
+ assert rows.rows[0]["endpoint"] == "chat.completions"
135
+ _capture.set_runtime(None)
136
+
137
+
138
+ def test_wrap_captures_anthropic_messages_create_through_a_real_client(tmp_path):
139
+ rows = Rows()
140
+ _capture.set_runtime(Runtime(rows, Options(app_root=str(tmp_path))))
141
+
142
+ def handler(request: httpx.Request) -> httpx.Response:
143
+ return httpx.Response(
144
+ 200,
145
+ json={
146
+ "id": "msg_test",
147
+ "type": "message",
148
+ "role": "assistant",
149
+ "model": "claude-haiku-4-5-20251001",
150
+ "content": [{"type": "text", "text": "hi"}],
151
+ "stop_reason": "end_turn",
152
+ "usage": {"input_tokens": 5, "output_tokens": 2},
153
+ },
154
+ )
155
+
156
+ client = AsyncAnthropic(
157
+ api_key="test",
158
+ http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
159
+ )
160
+ metergraph.wrap(client, provider="anthropic")
161
+
162
+ async def run():
163
+ return await client.messages.create(
164
+ model="claude-haiku-4-5-20251001",
165
+ max_tokens=10,
166
+ messages=[{"role": "user", "content": "hi"}],
167
+ )
168
+
169
+ response = asyncio.run(run())
170
+
171
+ assert response.content[0].text == "hi"
172
+ assert len(rows.rows) == 1
173
+ row = rows.rows[0]
174
+ assert row["endpoint"] == "messages"
175
+ assert row["provider"] == "anthropic"
176
+ assert row["input_tokens"] == 5
177
+ assert row["output_tokens"] == 2
178
+ _capture.set_runtime(None)
179
+
180
+
181
+ def test_wrap_captures_google_generate_content_through_a_real_client(tmp_path):
182
+ """genai.Client has no constructor hook for a custom transport, so the
183
+ mocked httpx client is swapped in on the internal _api_client after
184
+ construction — everything from there on (request building, response
185
+ parsing) is the real SDK's own code."""
186
+ rows = Rows()
187
+ _capture.set_runtime(Runtime(rows, Options(app_root=str(tmp_path))))
188
+
189
+ def handler(request: httpx.Request) -> httpx.Response:
190
+ return httpx.Response(
191
+ 200,
192
+ json={
193
+ "candidates": [
194
+ {
195
+ "content": {"parts": [{"text": "hi"}], "role": "model"},
196
+ "finishReason": "STOP",
197
+ }
198
+ ],
199
+ "usageMetadata": {
200
+ "promptTokenCount": 5,
201
+ "candidatesTokenCount": 2,
202
+ "totalTokenCount": 7,
203
+ },
204
+ "modelVersion": "gemini-2.5-flash",
205
+ },
206
+ )
207
+
208
+ client = genai.Client(api_key="test")
209
+ client._api_client._async_httpx_client = httpx.AsyncClient(
210
+ transport=httpx.MockTransport(handler)
211
+ )
212
+ metergraph.wrap(client, provider="google")
213
+
214
+ async def run():
215
+ return await client.aio.models.generate_content(
216
+ model="gemini-2.5-flash", contents="hi"
217
+ )
218
+
219
+ response = asyncio.run(run())
220
+
221
+ assert response.text == "hi"
222
+ assert len(rows.rows) == 1
223
+ row = rows.rows[0]
224
+ assert row["endpoint"] == "models.generate_content"
225
+ assert row["provider"] == "google"
226
+ _capture.set_runtime(None)
@@ -3,6 +3,7 @@ from __future__ import annotations
3
3
  import asyncio
4
4
  import gzip
5
5
  import json
6
+ import logging
6
7
  import os
7
8
  import threading
8
9
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -12,7 +13,8 @@ from types import SimpleNamespace
12
13
  import metergraph
13
14
  from metergraph import _capture
14
15
  from metergraph._capture import Options, Runtime
15
- from metergraph._config import choose_model
16
+ from metergraph._config import ConfigPoller, choose_model
17
+ from metergraph._failure_log import FailureLogger
16
18
  from metergraph._template import template_hash
17
19
  from metergraph._transport import Writer
18
20
 
@@ -35,6 +37,18 @@ def test_hosted_default_is_https():
35
37
  assert metergraph.DEFAULT_INGEST_URL == "https://d2xus7mp8zdv6t.cloudfront.net"
36
38
 
37
39
 
40
+ def test_python_seam_endpoints_match_shared_fixture():
41
+ fixture_path = Path(__file__).parent / "fixtures" / "seam_endpoints.json"
42
+ expected = json.loads(fixture_path.read_text())
43
+ actual = {
44
+ "openai": sorted({seam.endpoint for seam in _capture.OPENAI_SEAMS}),
45
+ "anthropic": sorted({seam.endpoint for seam in _capture.ANTHROPIC_SEAMS}),
46
+ "google": sorted({seam.endpoint for seam in _capture.GOOGLE_SEAMS}),
47
+ }
48
+ for provider, endpoints in expected.items():
49
+ assert actual[provider] == sorted(endpoints), provider
50
+
51
+
38
52
  def response(text="done"):
39
53
  usage = SimpleNamespace(
40
54
  prompt_tokens=12,
@@ -144,6 +158,90 @@ def test_wrap_patches_create_and_parse_on_both_chat_and_beta_chat(tmp_path):
144
158
  _capture.set_runtime(None)
145
159
 
146
160
 
161
+ def test_wrap_patches_responses_parse_and_beta_responses_create(tmp_path):
162
+ rows = Rows()
163
+ _capture.set_runtime(
164
+ Runtime(rows, Options(app_root=str(Path(__file__).parents[1])))
165
+ )
166
+
167
+ class Responses:
168
+ def create(self, **kwargs):
169
+ return response()
170
+
171
+ def parse(self, **kwargs):
172
+ return response()
173
+
174
+ class BetaResponses:
175
+ # client.beta.responses has .create but, as of openai>=1.x, no .parse —
176
+ # verified directly against the installed SDK; do not add a .parse here.
177
+ def create(self, **kwargs):
178
+ return response()
179
+
180
+ client = SimpleNamespace(
181
+ responses=Responses(),
182
+ beta=SimpleNamespace(responses=BetaResponses()),
183
+ )
184
+ metergraph.wrap(client, provider="openai")
185
+
186
+ client.responses.create(model="gpt-test")
187
+ client.responses.parse(model="gpt-test")
188
+ client.beta.responses.create(model="gpt-test")
189
+
190
+ assert [row["endpoint"] for row in rows.rows] == [
191
+ "responses",
192
+ "responses.parse",
193
+ "responses",
194
+ ]
195
+ _capture.set_runtime(None)
196
+
197
+
198
+ def test_wrap_skips_missing_nested_attribute_without_raising():
199
+ client = SimpleNamespace(beta=SimpleNamespace()) # beta.responses does not exist
200
+ result = metergraph.wrap(client, provider="openai")
201
+ assert result is client
202
+
203
+
204
+ def test_wrap_skips_one_broken_seam_without_affecting_others(tmp_path):
205
+ rows = Rows()
206
+ _capture.set_runtime(
207
+ Runtime(rows, Options(app_root=str(Path(__file__).parents[1])))
208
+ )
209
+
210
+ class Completions:
211
+ def create(self, **kwargs):
212
+ return response()
213
+
214
+ class Chat:
215
+ completions = Completions()
216
+
217
+ class Client:
218
+ chat = Chat()
219
+
220
+ @property
221
+ def responses(self):
222
+ raise RuntimeError("boom: not ready yet")
223
+
224
+ client = Client()
225
+ metergraph.wrap(client, provider="openai")
226
+ client.chat.completions.create(model="gpt-test")
227
+
228
+ assert [row["endpoint"] for row in rows.rows] == ["chat.completions"]
229
+ _capture.set_runtime(None)
230
+
231
+
232
+ def test_wrap_never_raises_even_if_client_attribute_access_raises(caplog):
233
+ class Explosive:
234
+ @property
235
+ def responses(self):
236
+ raise RuntimeError("boom: not ready yet")
237
+
238
+ client = Explosive()
239
+ with caplog.at_level(logging.WARNING, logger="metergraph"):
240
+ result = metergraph.wrap(client) # no provider override: exercises auto-detection
241
+ assert result is client
242
+ assert any("wrap() failed" in r.getMessage() for r in caplog.records)
243
+
244
+
147
245
  def gemini_response(text="gemini done"):
148
246
  return SimpleNamespace(
149
247
  text=text,
@@ -757,6 +855,31 @@ def test_canary_assignment_is_sticky_and_fail_open():
757
855
  assert choose_model("route-a", "fallback", "session-1", None) == "fallback"
758
856
 
759
857
 
858
+ def test_config_poller_logs_generic_failures_via_failure_logger(caplog):
859
+ class Handler(BaseHTTPRequestHandler):
860
+ def do_GET(self):
861
+ self.send_response(500)
862
+ self.end_headers()
863
+
864
+ def log_message(self, *args):
865
+ pass
866
+
867
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
868
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
869
+ thread.start()
870
+ poller = ConfigPoller(
871
+ "mg_test", f"http://127.0.0.1:{server.server_port}",
872
+ poll_seconds=60, hard_ttl_seconds=120,
873
+ )
874
+ with caplog.at_level(logging.WARNING, logger="metergraph"):
875
+ ok = poller.poll_once()
876
+ poller.stop()
877
+ server.shutdown()
878
+
879
+ assert ok is False
880
+ assert any("config poll to" in r.getMessage() for r in caplog.records)
881
+
882
+
760
883
  def test_record_outcome_uses_the_async_content_free_channel(monkeypatch):
761
884
  rows = Rows()
762
885
  monkeypatch.setattr(metergraph, "_writer", rows)
@@ -858,3 +981,173 @@ def test_writer_splits_wire_batches_at_512_kib():
858
981
  assert len(wire_lengths) > 1
859
982
  assert max(wire_lengths) <= 512 * 1024
860
983
  assert sorted(row["index"] for row in delivered_rows) == list(range(6))
984
+
985
+
986
+ def test_failure_logger_logs_first_occurrence_and_suppresses_repeats(caplog):
987
+ now = [0.0]
988
+ logger = FailureLogger(quiet_seconds=60.0, clock=lambda: now[0])
989
+ with caplog.at_level(logging.WARNING, logger="metergraph"):
990
+ logger.report("transport_error", "boom 1")
991
+ now[0] = 10.0
992
+ logger.report("transport_error", "boom 2")
993
+ now[0] = 20.0
994
+ logger.report("transport_error", "boom 3")
995
+ messages = [r.getMessage() for r in caplog.records]
996
+ assert len(messages) == 1
997
+ assert "boom 1" in messages[0]
998
+
999
+
1000
+ def test_failure_logger_reports_suppressed_count_after_quiet_window(caplog):
1001
+ now = [0.0]
1002
+ logger = FailureLogger(quiet_seconds=60.0, clock=lambda: now[0])
1003
+ with caplog.at_level(logging.WARNING, logger="metergraph"):
1004
+ logger.report("transport_error", "boom 1")
1005
+ now[0] = 10.0
1006
+ logger.report("transport_error", "boom 2") # suppressed
1007
+ now[0] = 70.0
1008
+ logger.report("transport_error", "boom 3")
1009
+ messages = [r.getMessage() for r in caplog.records]
1010
+ assert len(messages) == 2
1011
+ assert "1 more suppressed" in messages[1]
1012
+ assert "boom 3" in messages[1]
1013
+
1014
+
1015
+ def test_failure_logger_tracks_kinds_independently(caplog):
1016
+ now = [0.0]
1017
+ logger = FailureLogger(quiet_seconds=60.0, clock=lambda: now[0])
1018
+ with caplog.at_level(logging.WARNING, logger="metergraph"):
1019
+ logger.report("transport_error", "t1")
1020
+ logger.report("client_error", "c1")
1021
+ assert len(caplog.records) == 2
1022
+
1023
+
1024
+ def test_failure_logger_bounds_log_volume_under_sustained_failure(caplog):
1025
+ now = [0.0]
1026
+ logger = FailureLogger(quiet_seconds=60.0, clock=lambda: now[0])
1027
+ with caplog.at_level(logging.WARNING, logger="metergraph"):
1028
+ for _ in range(1000):
1029
+ logger.report("transport_error", "boom")
1030
+ assert len(caplog.records) == 1
1031
+
1032
+
1033
+ def test_writer_auth_failure_is_fatal_and_logged_once(caplog):
1034
+ class Handler(BaseHTTPRequestHandler):
1035
+ def do_POST(self):
1036
+ self.rfile.read(int(self.headers["Content-Length"]))
1037
+ self.send_response(401)
1038
+ self.end_headers()
1039
+
1040
+ def log_message(self, *args):
1041
+ pass
1042
+
1043
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
1044
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
1045
+ thread.start()
1046
+ writer = Writer("mg_test", f"http://127.0.0.1:{server.server_port}", flush_seconds=5)
1047
+ with caplog.at_level(logging.WARNING, logger="metergraph"):
1048
+ writer.enqueue({"payload": "one"})
1049
+ writer.flush(2)
1050
+ writer.enqueue({"payload": "two"})
1051
+ writer.flush(2)
1052
+ writer.shutdown()
1053
+ server.shutdown()
1054
+
1055
+ auth_warnings = [r for r in caplog.records if "authentication failed" in r.getMessage()]
1056
+ assert len(auth_warnings) == 1
1057
+ assert writer.dropped >= 1
1058
+
1059
+
1060
+ def test_writer_permanent_client_error_drops_batch_but_writer_stays_alive(caplog):
1061
+ attempts = []
1062
+
1063
+ class Handler(BaseHTTPRequestHandler):
1064
+ def do_POST(self):
1065
+ attempts.append(1)
1066
+ self.rfile.read(int(self.headers["Content-Length"]))
1067
+ self.send_response(400)
1068
+ self.end_headers()
1069
+
1070
+ def log_message(self, *args):
1071
+ pass
1072
+
1073
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
1074
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
1075
+ thread.start()
1076
+ writer = Writer("mg_test", f"http://127.0.0.1:{server.server_port}", flush_seconds=5)
1077
+ with caplog.at_level(logging.WARNING, logger="metergraph"):
1078
+ writer.enqueue({"payload": "one"})
1079
+ writer.flush(2)
1080
+ writer.enqueue({"payload": "two"})
1081
+ writer.flush(2)
1082
+ writer.shutdown()
1083
+ server.shutdown()
1084
+
1085
+ # A 400 is specific to the rejected batch, not the whole connection: the
1086
+ # writer must still attempt the second, unrelated batch.
1087
+ assert len(attempts) == 2
1088
+ assert writer._fatal is False
1089
+ assert writer.dropped >= 2
1090
+ assert any("HTTP 400" in r.getMessage() for r in caplog.records)
1091
+
1092
+
1093
+ def test_writer_413_splits_oversized_batch_and_delivers_the_pieces():
1094
+ received_batches = []
1095
+
1096
+ class Handler(BaseHTTPRequestHandler):
1097
+ def do_POST(self):
1098
+ body = self.rfile.read(int(self.headers["Content-Length"]))
1099
+ if self.headers.get("Content-Encoding") == "gzip":
1100
+ body = gzip.decompress(body)
1101
+ rows = json.loads(body)["rows"]
1102
+ received_batches.append(len(rows))
1103
+ if len(rows) > 1:
1104
+ self.send_response(413)
1105
+ else:
1106
+ self.send_response(202)
1107
+ self.end_headers()
1108
+
1109
+ def log_message(self, *args):
1110
+ pass
1111
+
1112
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
1113
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
1114
+ thread.start()
1115
+ writer = Writer("mg_test", f"http://127.0.0.1:{server.server_port}", flush_seconds=5)
1116
+ for i in range(4):
1117
+ writer.enqueue({"index": i})
1118
+ assert writer.flush(10)
1119
+ writer.shutdown()
1120
+ server.shutdown()
1121
+
1122
+ assert any(size > 1 for size in received_batches) # a multi-row batch hit 413 at least once
1123
+ assert received_batches.count(1) == 4 # every row was eventually delivered as its own batch
1124
+ assert writer.dropped == 0
1125
+ assert writer._fatal is False
1126
+
1127
+
1128
+ def test_writer_server_error_retries_and_is_not_fatal(caplog):
1129
+ attempts = []
1130
+
1131
+ class Handler(BaseHTTPRequestHandler):
1132
+ def do_POST(self):
1133
+ self.rfile.read(int(self.headers["Content-Length"]))
1134
+ attempts.append(1)
1135
+ self.send_response(500)
1136
+ self.end_headers()
1137
+
1138
+ def log_message(self, *args):
1139
+ pass
1140
+
1141
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
1142
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
1143
+ thread.start()
1144
+ writer = Writer("mg_test", f"http://127.0.0.1:{server.server_port}", flush_seconds=5)
1145
+ with caplog.at_level(logging.WARNING, logger="metergraph"):
1146
+ writer.enqueue({"payload": "one"})
1147
+ writer.flush(2)
1148
+ writer.shutdown()
1149
+ server.shutdown()
1150
+
1151
+ assert len(attempts) == 1
1152
+ assert writer._fatal is False
1153
+ assert any("HTTP 500" in r.getMessage() for r in caplog.records)
@@ -0,0 +1,35 @@
1
+ """Assert every seam-table entry resolves against the real, installed
2
+ provider SDKs — not just against hand-built fakes, which can only ever
3
+ verify the code against what its author believed the SDK looked like."""
4
+
5
+ from __future__ import annotations
6
+
7
+ from anthropic import Anthropic
8
+ from google import genai
9
+ from openai import OpenAI
10
+
11
+ from metergraph._capture import ANTHROPIC_SEAMS, GOOGLE_SEAMS, OPENAI_SEAMS, _resolve
12
+
13
+
14
+ def _missing_seams(client, seams) -> list[str]:
15
+ missing = []
16
+ for seam in seams:
17
+ owner = _resolve(client, seam.path)
18
+ if owner is None or not callable(getattr(owner, seam.method, None)):
19
+ missing.append(f"{seam.path}.{seam.method}")
20
+ return missing
21
+
22
+
23
+ def test_openai_seams_exist_on_the_real_sdk():
24
+ client = OpenAI(api_key="test")
25
+ assert _missing_seams(client, OPENAI_SEAMS) == []
26
+
27
+
28
+ def test_anthropic_seams_exist_on_the_real_sdk():
29
+ client = Anthropic(api_key="test")
30
+ assert _missing_seams(client, ANTHROPIC_SEAMS) == []
31
+
32
+
33
+ def test_google_seams_exist_on_the_real_sdk():
34
+ client = genai.Client(api_key="test")
35
+ assert _missing_seams(client, GOOGLE_SEAMS) == []
@@ -1,3 +0,0 @@
1
-
2
- [dev]
3
- pytest>=8
File without changes
File without changes