metergraph 0.2.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {metergraph-0.2.0 → metergraph-0.2.1}/PKG-INFO +4 -1
- {metergraph-0.2.0 → metergraph-0.2.1}/pyproject.toml +2 -2
- {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/_capture.py +102 -51
- {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/_config.py +13 -1
- metergraph-0.2.1/src/metergraph/_failure_log.py +44 -0
- {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/_transport.py +23 -1
- {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/_version.py +1 -1
- {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph.egg-info/PKG-INFO +4 -1
- {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph.egg-info/SOURCES.txt +4 -1
- metergraph-0.2.1/src/metergraph.egg-info/requires.txt +6 -0
- metergraph-0.2.1/tests/test_real_client_integration.py +226 -0
- {metergraph-0.2.0 → metergraph-0.2.1}/tests/test_sdk.py +294 -1
- metergraph-0.2.1/tests/test_seam_reality.py +35 -0
- metergraph-0.2.0/src/metergraph.egg-info/requires.txt +0 -3
- {metergraph-0.2.0 → metergraph-0.2.1}/README.md +0 -0
- {metergraph-0.2.0 → metergraph-0.2.1}/setup.cfg +0 -0
- {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/__init__.py +0 -0
- {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/_context.py +0 -0
- {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/_template.py +0 -0
- {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph/_track.py +0 -0
- {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph.egg-info/dependency_links.txt +0 -0
- {metergraph-0.2.0 → metergraph-0.2.1}/src/metergraph.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: metergraph
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Fire-and-forget LLM spend capture for Metergraph
|
|
5
5
|
Author: Pioneer Square Labs
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -15,6 +15,9 @@ Requires-Python: >=3.10
|
|
|
15
15
|
Description-Content-Type: text/markdown
|
|
16
16
|
Provides-Extra: dev
|
|
17
17
|
Requires-Dist: pytest>=8; extra == "dev"
|
|
18
|
+
Requires-Dist: openai>=2.50.0; extra == "dev"
|
|
19
|
+
Requires-Dist: anthropic>=0.40; extra == "dev"
|
|
20
|
+
Requires-Dist: google-genai>=1; extra == "dev"
|
|
18
21
|
|
|
19
22
|
# metergraph (Python)
|
|
20
23
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "metergraph"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.1"
|
|
4
4
|
description = "Fire-and-forget LLM spend capture for Metergraph"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.10"
|
|
@@ -20,7 +20,7 @@ Repository = "https://github.com/PioneerSquareLabs/metergraphsdk"
|
|
|
20
20
|
Issues = "https://github.com/PioneerSquareLabs/metergraphsdk/issues"
|
|
21
21
|
|
|
22
22
|
[project.optional-dependencies]
|
|
23
|
-
dev = ["pytest>=8"]
|
|
23
|
+
dev = ["pytest>=8", "openai>=2.50.0", "anthropic>=0.40", "google-genai>=1"]
|
|
24
24
|
|
|
25
25
|
[build-system]
|
|
26
26
|
requires = ["setuptools>=68"]
|
|
@@ -1082,61 +1082,85 @@ def _finish_or_stream(
|
|
|
1082
1082
|
return result
|
|
1083
1083
|
|
|
1084
1084
|
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
if
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1085
|
+
@dataclass(frozen=True)
|
|
1086
|
+
class Seam:
|
|
1087
|
+
"""A single instrumentable method, addressed by a dotted attribute path
|
|
1088
|
+
from the client root (e.g. "beta.chat.completions")."""
|
|
1089
|
+
|
|
1090
|
+
path: str
|
|
1091
|
+
method: str
|
|
1092
|
+
endpoint: str
|
|
1093
|
+
|
|
1094
|
+
|
|
1095
|
+
OPENAI_SEAMS: tuple[Seam, ...] = (
|
|
1096
|
+
Seam("chat.completions", "create", "chat.completions"),
|
|
1097
|
+
Seam("chat.completions", "parse", "chat.completions.parse"),
|
|
1098
|
+
Seam("beta.chat.completions", "parse", "chat.completions.parse"),
|
|
1099
|
+
Seam("responses", "create", "responses"),
|
|
1100
|
+
Seam("responses", "stream", "responses.stream"),
|
|
1101
|
+
Seam("responses", "parse", "responses.parse"),
|
|
1102
|
+
Seam("beta.responses", "create", "responses"),
|
|
1103
|
+
# Note: client.beta.responses has no .parse method (verified against
|
|
1104
|
+
# openai==2.50.0) — do not add one here without re-verifying first.
|
|
1105
|
+
)
|
|
1106
|
+
|
|
1107
|
+
ANTHROPIC_SEAMS: tuple[Seam, ...] = (
|
|
1108
|
+
Seam("messages", "create", "messages"),
|
|
1109
|
+
Seam("messages", "stream", "messages.stream"),
|
|
1110
|
+
)
|
|
1111
|
+
|
|
1112
|
+
GOOGLE_SEAMS: tuple[Seam, ...] = (
|
|
1113
|
+
Seam("models", "generate_content", "models.generate_content"),
|
|
1114
|
+
Seam("models", "generate_content_stream", "models.generate_content.stream"),
|
|
1115
|
+
Seam("aio.models", "generate_content", "models.generate_content"),
|
|
1116
|
+
Seam("aio.models", "generate_content_stream", "models.generate_content.stream"),
|
|
1117
|
+
)
|
|
1118
|
+
|
|
1119
|
+
SEAM_TABLES: dict[str, tuple[Seam, ...]] = {
|
|
1120
|
+
"openai": OPENAI_SEAMS,
|
|
1121
|
+
"anthropic": ANTHROPIC_SEAMS,
|
|
1122
|
+
"google": GOOGLE_SEAMS,
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
def _detect_provider(client: Any) -> str:
|
|
1127
|
+
if hasattr(getattr(client, "models", None), "generate_content"):
|
|
1128
|
+
return "google"
|
|
1129
|
+
if hasattr(client, "chat") or hasattr(client, "responses"):
|
|
1130
|
+
return "openai"
|
|
1131
|
+
return "anthropic"
|
|
1132
|
+
|
|
1133
|
+
|
|
1134
|
+
def _resolve(client: Any, path: str) -> Any:
|
|
1135
|
+
obj = client
|
|
1136
|
+
for part in path.split("."):
|
|
1137
|
+
obj = getattr(obj, part, None)
|
|
1138
|
+
if obj is None:
|
|
1139
|
+
return None
|
|
1140
|
+
return obj
|
|
1141
|
+
|
|
1142
|
+
|
|
1143
|
+
def _apply_seams(client: Any, provider: str) -> list[str]:
|
|
1144
|
+
patched: list[str] = []
|
|
1145
|
+
for seam in SEAM_TABLES.get(provider, ()):
|
|
1146
|
+
try:
|
|
1147
|
+
owner = _resolve(client, seam.path)
|
|
1148
|
+
except Exception:
|
|
1149
|
+
continue # a pathological client property must never break wrap()
|
|
1150
|
+
if owner is not None and _patch(owner, seam.method, provider, seam.endpoint):
|
|
1151
|
+
patched.append(f"{seam.path}.{seam.method}")
|
|
1152
|
+
return patched
|
|
1153
|
+
|
|
1154
|
+
|
|
1155
|
+
def _apply_batch_extras(client: Any, provider: str) -> int:
|
|
1156
|
+
patched = 0
|
|
1134
1157
|
if provider == "openai":
|
|
1135
1158
|
files = getattr(client, "files", None)
|
|
1136
1159
|
if files is not None:
|
|
1137
1160
|
patched += int(_patch_openai_batch_content(files, "content"))
|
|
1138
1161
|
patched += int(_patch_openai_batch_content(files, "retrieve_content"))
|
|
1139
1162
|
elif provider == "anthropic":
|
|
1163
|
+
messages = getattr(client, "messages", None)
|
|
1140
1164
|
batch_owners = [getattr(messages, "batches", None)]
|
|
1141
1165
|
beta_messages = getattr(getattr(client, "beta", None), "messages", None)
|
|
1142
1166
|
batch_owners.append(getattr(beta_messages, "batches", None))
|
|
@@ -1145,6 +1169,33 @@ def wrap(client: Any, *, provider: str | None = None) -> Any:
|
|
|
1145
1169
|
for owner in batch_owners
|
|
1146
1170
|
if owner is not None
|
|
1147
1171
|
)
|
|
1148
|
-
|
|
1149
|
-
|
|
1172
|
+
return patched
|
|
1173
|
+
|
|
1174
|
+
|
|
1175
|
+
def wrap(client: Any, *, provider: str | None = None) -> Any:
|
|
1176
|
+
"""Patch supported resource methods on an OpenAI, Anthropic, or Google client.
|
|
1177
|
+
|
|
1178
|
+
Never raises: an unrecognized client shape, or an exception while probing
|
|
1179
|
+
it, results in an unmodified, uninstrumented client — not a crash.
|
|
1180
|
+
"""
|
|
1181
|
+
try:
|
|
1182
|
+
resolved_provider = provider or _detect_provider(client)
|
|
1183
|
+
patched = _apply_seams(client, resolved_provider)
|
|
1184
|
+
patched_count = len(patched) + _apply_batch_extras(client, resolved_provider)
|
|
1185
|
+
if not patched_count:
|
|
1186
|
+
log.warning(
|
|
1187
|
+
"Metergraph found no supported methods on %s client", resolved_provider
|
|
1188
|
+
)
|
|
1189
|
+
else:
|
|
1190
|
+
log.info(
|
|
1191
|
+
"Metergraph patched %d seam(s) on %s client: %s",
|
|
1192
|
+
patched_count,
|
|
1193
|
+
resolved_provider,
|
|
1194
|
+
", ".join(patched) or "(batch-only)",
|
|
1195
|
+
)
|
|
1196
|
+
except Exception:
|
|
1197
|
+
log.warning(
|
|
1198
|
+
"Metergraph wrap() failed; client is unmodified and uninstrumented",
|
|
1199
|
+
exc_info=True,
|
|
1200
|
+
)
|
|
1150
1201
|
return client
|
|
@@ -11,6 +11,8 @@ import urllib.request
|
|
|
11
11
|
from collections.abc import Mapping
|
|
12
12
|
from typing import Any
|
|
13
13
|
|
|
14
|
+
from ._failure_log import FailureLogger
|
|
15
|
+
|
|
14
16
|
|
|
15
17
|
log = logging.getLogger("metergraph")
|
|
16
18
|
|
|
@@ -75,6 +77,7 @@ class ConfigPoller:
|
|
|
75
77
|
self._routes: dict[str, dict[str, Any]] = {}
|
|
76
78
|
self._last_success = 0.0
|
|
77
79
|
self._lock = threading.Lock()
|
|
80
|
+
self._failure_log = FailureLogger()
|
|
78
81
|
self._stop = threading.Event()
|
|
79
82
|
self._thread = threading.Thread(
|
|
80
83
|
target=self._run, name="metergraph-config", daemon=True
|
|
@@ -124,8 +127,17 @@ class ConfigPoller:
|
|
|
124
127
|
"Metergraph config authentication failed; using default models"
|
|
125
128
|
)
|
|
126
129
|
self._stop.set()
|
|
130
|
+
return False
|
|
131
|
+
self._failure_log.report(
|
|
132
|
+
"config_poll_error",
|
|
133
|
+
f"config poll to {self._url} failed with HTTP {exc.code}",
|
|
134
|
+
)
|
|
127
135
|
return False
|
|
128
|
-
except Exception:
|
|
136
|
+
except Exception as exc:
|
|
137
|
+
self._failure_log.report(
|
|
138
|
+
"config_poll_error",
|
|
139
|
+
f"config poll to {self._url} failed: {type(exc).__name__}: {exc}",
|
|
140
|
+
)
|
|
129
141
|
return False
|
|
130
142
|
|
|
131
143
|
def model_for(self, route: str, default: str, session_key: str | None) -> str:
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Rate-limited failure logging.
|
|
2
|
+
|
|
3
|
+
Logs the first occurrence of a failure kind immediately, then suppresses
|
|
4
|
+
repeats within a quiet window and reports how many were suppressed on the
|
|
5
|
+
next log line for that kind. Keeps log volume bounded under sustained
|
|
6
|
+
failure without ever going completely silent.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import logging
|
|
12
|
+
import time
|
|
13
|
+
from typing import Callable
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
log = logging.getLogger("metergraph")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class FailureLogger:
|
|
20
|
+
def __init__(
|
|
21
|
+
self,
|
|
22
|
+
quiet_seconds: float = 60.0,
|
|
23
|
+
*,
|
|
24
|
+
clock: Callable[[], float] = time.monotonic,
|
|
25
|
+
) -> None:
|
|
26
|
+
self._quiet_seconds = quiet_seconds
|
|
27
|
+
self._clock = clock
|
|
28
|
+
self._last_logged: dict[str, float] = {}
|
|
29
|
+
self._suppressed: dict[str, int] = {}
|
|
30
|
+
|
|
31
|
+
def report(self, kind: str, message: str, level: int = logging.WARNING) -> None:
|
|
32
|
+
now = self._clock()
|
|
33
|
+
last = self._last_logged.get(kind)
|
|
34
|
+
if last is not None and now - last < self._quiet_seconds:
|
|
35
|
+
self._suppressed[kind] = self._suppressed.get(kind, 0) + 1
|
|
36
|
+
return
|
|
37
|
+
suppressed = self._suppressed.pop(kind, 0)
|
|
38
|
+
suffix = (
|
|
39
|
+
f" ({suppressed} more suppressed in the last {self._quiet_seconds:.0f}s)"
|
|
40
|
+
if suppressed
|
|
41
|
+
else ""
|
|
42
|
+
)
|
|
43
|
+
log.log(level, "metergraph: %s%s", message, suffix)
|
|
44
|
+
self._last_logged[kind] = now
|
|
@@ -13,6 +13,7 @@ import urllib.error
|
|
|
13
13
|
import urllib.request
|
|
14
14
|
from typing import Any
|
|
15
15
|
|
|
16
|
+
from ._failure_log import FailureLogger
|
|
16
17
|
from ._version import SDK_VERSION
|
|
17
18
|
|
|
18
19
|
|
|
@@ -43,6 +44,7 @@ class Writer:
|
|
|
43
44
|
self._errors = 0
|
|
44
45
|
self._backoff = 1.0
|
|
45
46
|
self._retry_at = 0.0
|
|
47
|
+
self._failure_log = FailureLogger()
|
|
46
48
|
self._thread = self._new_thread()
|
|
47
49
|
self._thread.start()
|
|
48
50
|
if hasattr(os, "register_at_fork"):
|
|
@@ -144,11 +146,31 @@ class Writer:
|
|
|
144
146
|
log.warning(
|
|
145
147
|
"Metergraph authentication failed; capture disabled for this process"
|
|
146
148
|
)
|
|
149
|
+
elif exc.code == 413 and len(rows) > 1:
|
|
150
|
+
midpoint = len(rows) // 2
|
|
151
|
+
left = self._deliver(rows[:midpoint])
|
|
152
|
+
right = self._deliver(rows[midpoint:])
|
|
153
|
+
return left and right
|
|
154
|
+
elif exc.code in (400, 404, 413, 422):
|
|
155
|
+
self._dropped += len(rows)
|
|
156
|
+
self._failure_log.report(
|
|
157
|
+
"client_error",
|
|
158
|
+
f"ingest rejected batch with HTTP {exc.code} against {self._url}; "
|
|
159
|
+
"dropping this batch (payload-specific, not a process-wide failure)",
|
|
160
|
+
)
|
|
147
161
|
else:
|
|
148
162
|
self._failed(len(rows))
|
|
163
|
+
self._failure_log.report(
|
|
164
|
+
"server_error",
|
|
165
|
+
f"ingest request failed with HTTP {exc.code} against {self._url}",
|
|
166
|
+
)
|
|
149
167
|
return False
|
|
150
|
-
except Exception:
|
|
168
|
+
except Exception as exc:
|
|
151
169
|
self._failed(len(rows))
|
|
170
|
+
self._failure_log.report(
|
|
171
|
+
"transport_error",
|
|
172
|
+
f"ingest request to {self._url} failed: {type(exc).__name__}: {exc}",
|
|
173
|
+
)
|
|
152
174
|
return False
|
|
153
175
|
|
|
154
176
|
def _failed(self, count: int) -> None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: metergraph
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Fire-and-forget LLM spend capture for Metergraph
|
|
5
5
|
Author: Pioneer Square Labs
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -15,6 +15,9 @@ Requires-Python: >=3.10
|
|
|
15
15
|
Description-Content-Type: text/markdown
|
|
16
16
|
Provides-Extra: dev
|
|
17
17
|
Requires-Dist: pytest>=8; extra == "dev"
|
|
18
|
+
Requires-Dist: openai>=2.50.0; extra == "dev"
|
|
19
|
+
Requires-Dist: anthropic>=0.40; extra == "dev"
|
|
20
|
+
Requires-Dist: google-genai>=1; extra == "dev"
|
|
18
21
|
|
|
19
22
|
# metergraph (Python)
|
|
20
23
|
|
|
@@ -4,6 +4,7 @@ src/metergraph/__init__.py
|
|
|
4
4
|
src/metergraph/_capture.py
|
|
5
5
|
src/metergraph/_config.py
|
|
6
6
|
src/metergraph/_context.py
|
|
7
|
+
src/metergraph/_failure_log.py
|
|
7
8
|
src/metergraph/_template.py
|
|
8
9
|
src/metergraph/_track.py
|
|
9
10
|
src/metergraph/_transport.py
|
|
@@ -13,4 +14,6 @@ src/metergraph.egg-info/SOURCES.txt
|
|
|
13
14
|
src/metergraph.egg-info/dependency_links.txt
|
|
14
15
|
src/metergraph.egg-info/requires.txt
|
|
15
16
|
src/metergraph.egg-info/top_level.txt
|
|
16
|
-
tests/
|
|
17
|
+
tests/test_real_client_integration.py
|
|
18
|
+
tests/test_sdk.py
|
|
19
|
+
tests/test_seam_reality.py
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""Wrap the real, unmodified provider SDK clients and drive a call through
|
|
2
|
+
their actual request-building/response-parsing code — with the network
|
|
3
|
+
replaced by a mocked transport, not the SDK itself.
|
|
4
|
+
|
|
5
|
+
test_seam_reality.py proves a seam *exists* on the real client. The
|
|
6
|
+
behavioral tests in test_sdk.py prove wrap() *works*, but only against a
|
|
7
|
+
hand-built fake that mimics the real client's shape. Neither proves that
|
|
8
|
+
wrapping the real client and calling a real method actually produces a
|
|
9
|
+
captured row — which is exactly the gap that let the original
|
|
10
|
+
chat.completions.parse capture bug ship unnoticed. These tests close it,
|
|
11
|
+
without needing live API keys or network access.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import asyncio
|
|
17
|
+
import json
|
|
18
|
+
|
|
19
|
+
import httpx
|
|
20
|
+
import metergraph
|
|
21
|
+
from anthropic import AsyncAnthropic
|
|
22
|
+
from google import genai
|
|
23
|
+
from metergraph import _capture
|
|
24
|
+
from metergraph._capture import Options, Runtime
|
|
25
|
+
from openai import AsyncOpenAI
|
|
26
|
+
from pydantic import BaseModel
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Rows:
|
|
30
|
+
def __init__(self):
|
|
31
|
+
self.rows = []
|
|
32
|
+
|
|
33
|
+
def enqueue(self, row):
|
|
34
|
+
self.rows.append(row)
|
|
35
|
+
return True
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Answer(BaseModel):
|
|
39
|
+
text: str
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_wrap_captures_openai_parse_through_a_real_client(tmp_path):
|
|
43
|
+
""".parse() is the exact seam the original production bug missed."""
|
|
44
|
+
rows = Rows()
|
|
45
|
+
_capture.set_runtime(Runtime(rows, Options(app_root=str(tmp_path))))
|
|
46
|
+
|
|
47
|
+
def handler(request: httpx.Request) -> httpx.Response:
|
|
48
|
+
return httpx.Response(
|
|
49
|
+
200,
|
|
50
|
+
json={
|
|
51
|
+
"id": "chatcmpl-test",
|
|
52
|
+
"object": "chat.completion",
|
|
53
|
+
"created": 0,
|
|
54
|
+
"model": "gpt-4o-mini",
|
|
55
|
+
"choices": [
|
|
56
|
+
{
|
|
57
|
+
"index": 0,
|
|
58
|
+
"message": {
|
|
59
|
+
"role": "assistant",
|
|
60
|
+
"content": json.dumps({"text": "hi"}),
|
|
61
|
+
},
|
|
62
|
+
"finish_reason": "stop",
|
|
63
|
+
}
|
|
64
|
+
],
|
|
65
|
+
"usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
|
|
66
|
+
},
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
client = AsyncOpenAI(
|
|
70
|
+
api_key="test",
|
|
71
|
+
http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
|
|
72
|
+
)
|
|
73
|
+
metergraph.wrap(client, provider="openai")
|
|
74
|
+
|
|
75
|
+
async def run():
|
|
76
|
+
with metergraph.route("real-client-test"):
|
|
77
|
+
return await client.chat.completions.parse(
|
|
78
|
+
model="gpt-4o-mini",
|
|
79
|
+
messages=[{"role": "user", "content": "hi"}],
|
|
80
|
+
response_format=Answer,
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
response = asyncio.run(run())
|
|
84
|
+
|
|
85
|
+
assert response.choices[0].message.parsed == Answer(text="hi")
|
|
86
|
+
assert len(rows.rows) == 1
|
|
87
|
+
row = rows.rows[0]
|
|
88
|
+
assert row["endpoint"] == "chat.completions.parse"
|
|
89
|
+
assert row["provider"] == "openai"
|
|
90
|
+
assert row["model"] == "gpt-4o-mini"
|
|
91
|
+
assert row["input_tokens"] == 5
|
|
92
|
+
assert row["output_tokens"] == 2
|
|
93
|
+
_capture.set_runtime(None)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_wrap_captures_openai_create_through_a_real_client(tmp_path):
|
|
97
|
+
rows = Rows()
|
|
98
|
+
_capture.set_runtime(Runtime(rows, Options(app_root=str(tmp_path))))
|
|
99
|
+
|
|
100
|
+
def handler(request: httpx.Request) -> httpx.Response:
|
|
101
|
+
return httpx.Response(
|
|
102
|
+
200,
|
|
103
|
+
json={
|
|
104
|
+
"id": "chatcmpl-test2",
|
|
105
|
+
"object": "chat.completion",
|
|
106
|
+
"created": 0,
|
|
107
|
+
"model": "gpt-4o-mini",
|
|
108
|
+
"choices": [
|
|
109
|
+
{
|
|
110
|
+
"index": 0,
|
|
111
|
+
"message": {"role": "assistant", "content": "hi"},
|
|
112
|
+
"finish_reason": "stop",
|
|
113
|
+
}
|
|
114
|
+
],
|
|
115
|
+
"usage": {"prompt_tokens": 4, "completion_tokens": 1, "total_tokens": 5},
|
|
116
|
+
},
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
client = AsyncOpenAI(
|
|
120
|
+
api_key="test",
|
|
121
|
+
http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
|
|
122
|
+
)
|
|
123
|
+
metergraph.wrap(client, provider="openai")
|
|
124
|
+
|
|
125
|
+
async def run():
|
|
126
|
+
return await client.chat.completions.create(
|
|
127
|
+
model="gpt-4o-mini", messages=[{"role": "user", "content": "hi"}]
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
response = asyncio.run(run())
|
|
131
|
+
|
|
132
|
+
assert response.choices[0].message.content == "hi"
|
|
133
|
+
assert len(rows.rows) == 1
|
|
134
|
+
assert rows.rows[0]["endpoint"] == "chat.completions"
|
|
135
|
+
_capture.set_runtime(None)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def test_wrap_captures_anthropic_messages_create_through_a_real_client(tmp_path):
|
|
139
|
+
rows = Rows()
|
|
140
|
+
_capture.set_runtime(Runtime(rows, Options(app_root=str(tmp_path))))
|
|
141
|
+
|
|
142
|
+
def handler(request: httpx.Request) -> httpx.Response:
|
|
143
|
+
return httpx.Response(
|
|
144
|
+
200,
|
|
145
|
+
json={
|
|
146
|
+
"id": "msg_test",
|
|
147
|
+
"type": "message",
|
|
148
|
+
"role": "assistant",
|
|
149
|
+
"model": "claude-haiku-4-5-20251001",
|
|
150
|
+
"content": [{"type": "text", "text": "hi"}],
|
|
151
|
+
"stop_reason": "end_turn",
|
|
152
|
+
"usage": {"input_tokens": 5, "output_tokens": 2},
|
|
153
|
+
},
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
client = AsyncAnthropic(
|
|
157
|
+
api_key="test",
|
|
158
|
+
http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
|
|
159
|
+
)
|
|
160
|
+
metergraph.wrap(client, provider="anthropic")
|
|
161
|
+
|
|
162
|
+
async def run():
|
|
163
|
+
return await client.messages.create(
|
|
164
|
+
model="claude-haiku-4-5-20251001",
|
|
165
|
+
max_tokens=10,
|
|
166
|
+
messages=[{"role": "user", "content": "hi"}],
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
response = asyncio.run(run())
|
|
170
|
+
|
|
171
|
+
assert response.content[0].text == "hi"
|
|
172
|
+
assert len(rows.rows) == 1
|
|
173
|
+
row = rows.rows[0]
|
|
174
|
+
assert row["endpoint"] == "messages"
|
|
175
|
+
assert row["provider"] == "anthropic"
|
|
176
|
+
assert row["input_tokens"] == 5
|
|
177
|
+
assert row["output_tokens"] == 2
|
|
178
|
+
_capture.set_runtime(None)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def test_wrap_captures_google_generate_content_through_a_real_client(tmp_path):
|
|
182
|
+
"""genai.Client has no constructor hook for a custom transport, so the
|
|
183
|
+
mocked httpx client is swapped in on the internal _api_client after
|
|
184
|
+
construction — everything from there on (request building, response
|
|
185
|
+
parsing) is the real SDK's own code."""
|
|
186
|
+
rows = Rows()
|
|
187
|
+
_capture.set_runtime(Runtime(rows, Options(app_root=str(tmp_path))))
|
|
188
|
+
|
|
189
|
+
def handler(request: httpx.Request) -> httpx.Response:
|
|
190
|
+
return httpx.Response(
|
|
191
|
+
200,
|
|
192
|
+
json={
|
|
193
|
+
"candidates": [
|
|
194
|
+
{
|
|
195
|
+
"content": {"parts": [{"text": "hi"}], "role": "model"},
|
|
196
|
+
"finishReason": "STOP",
|
|
197
|
+
}
|
|
198
|
+
],
|
|
199
|
+
"usageMetadata": {
|
|
200
|
+
"promptTokenCount": 5,
|
|
201
|
+
"candidatesTokenCount": 2,
|
|
202
|
+
"totalTokenCount": 7,
|
|
203
|
+
},
|
|
204
|
+
"modelVersion": "gemini-2.5-flash",
|
|
205
|
+
},
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
client = genai.Client(api_key="test")
|
|
209
|
+
client._api_client._async_httpx_client = httpx.AsyncClient(
|
|
210
|
+
transport=httpx.MockTransport(handler)
|
|
211
|
+
)
|
|
212
|
+
metergraph.wrap(client, provider="google")
|
|
213
|
+
|
|
214
|
+
async def run():
|
|
215
|
+
return await client.aio.models.generate_content(
|
|
216
|
+
model="gemini-2.5-flash", contents="hi"
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
response = asyncio.run(run())
|
|
220
|
+
|
|
221
|
+
assert response.text == "hi"
|
|
222
|
+
assert len(rows.rows) == 1
|
|
223
|
+
row = rows.rows[0]
|
|
224
|
+
assert row["endpoint"] == "models.generate_content"
|
|
225
|
+
assert row["provider"] == "google"
|
|
226
|
+
_capture.set_runtime(None)
|
|
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import gzip
|
|
5
5
|
import json
|
|
6
|
+
import logging
|
|
6
7
|
import os
|
|
7
8
|
import threading
|
|
8
9
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
@@ -12,7 +13,8 @@ from types import SimpleNamespace
|
|
|
12
13
|
import metergraph
|
|
13
14
|
from metergraph import _capture
|
|
14
15
|
from metergraph._capture import Options, Runtime
|
|
15
|
-
from metergraph._config import choose_model
|
|
16
|
+
from metergraph._config import ConfigPoller, choose_model
|
|
17
|
+
from metergraph._failure_log import FailureLogger
|
|
16
18
|
from metergraph._template import template_hash
|
|
17
19
|
from metergraph._transport import Writer
|
|
18
20
|
|
|
@@ -35,6 +37,18 @@ def test_hosted_default_is_https():
|
|
|
35
37
|
assert metergraph.DEFAULT_INGEST_URL == "https://d2xus7mp8zdv6t.cloudfront.net"
|
|
36
38
|
|
|
37
39
|
|
|
40
|
+
def test_python_seam_endpoints_match_shared_fixture():
|
|
41
|
+
fixture_path = Path(__file__).parent / "fixtures" / "seam_endpoints.json"
|
|
42
|
+
expected = json.loads(fixture_path.read_text())
|
|
43
|
+
actual = {
|
|
44
|
+
"openai": sorted({seam.endpoint for seam in _capture.OPENAI_SEAMS}),
|
|
45
|
+
"anthropic": sorted({seam.endpoint for seam in _capture.ANTHROPIC_SEAMS}),
|
|
46
|
+
"google": sorted({seam.endpoint for seam in _capture.GOOGLE_SEAMS}),
|
|
47
|
+
}
|
|
48
|
+
for provider, endpoints in expected.items():
|
|
49
|
+
assert actual[provider] == sorted(endpoints), provider
|
|
50
|
+
|
|
51
|
+
|
|
38
52
|
def response(text="done"):
|
|
39
53
|
usage = SimpleNamespace(
|
|
40
54
|
prompt_tokens=12,
|
|
@@ -144,6 +158,90 @@ def test_wrap_patches_create_and_parse_on_both_chat_and_beta_chat(tmp_path):
|
|
|
144
158
|
_capture.set_runtime(None)
|
|
145
159
|
|
|
146
160
|
|
|
161
|
+
def test_wrap_patches_responses_parse_and_beta_responses_create(tmp_path):
|
|
162
|
+
rows = Rows()
|
|
163
|
+
_capture.set_runtime(
|
|
164
|
+
Runtime(rows, Options(app_root=str(Path(__file__).parents[1])))
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
class Responses:
|
|
168
|
+
def create(self, **kwargs):
|
|
169
|
+
return response()
|
|
170
|
+
|
|
171
|
+
def parse(self, **kwargs):
|
|
172
|
+
return response()
|
|
173
|
+
|
|
174
|
+
class BetaResponses:
|
|
175
|
+
# client.beta.responses has .create but, as of openai>=1.x, no .parse —
|
|
176
|
+
# verified directly against the installed SDK; do not add a .parse here.
|
|
177
|
+
def create(self, **kwargs):
|
|
178
|
+
return response()
|
|
179
|
+
|
|
180
|
+
client = SimpleNamespace(
|
|
181
|
+
responses=Responses(),
|
|
182
|
+
beta=SimpleNamespace(responses=BetaResponses()),
|
|
183
|
+
)
|
|
184
|
+
metergraph.wrap(client, provider="openai")
|
|
185
|
+
|
|
186
|
+
client.responses.create(model="gpt-test")
|
|
187
|
+
client.responses.parse(model="gpt-test")
|
|
188
|
+
client.beta.responses.create(model="gpt-test")
|
|
189
|
+
|
|
190
|
+
assert [row["endpoint"] for row in rows.rows] == [
|
|
191
|
+
"responses",
|
|
192
|
+
"responses.parse",
|
|
193
|
+
"responses",
|
|
194
|
+
]
|
|
195
|
+
_capture.set_runtime(None)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def test_wrap_skips_missing_nested_attribute_without_raising():
|
|
199
|
+
client = SimpleNamespace(beta=SimpleNamespace()) # beta.responses does not exist
|
|
200
|
+
result = metergraph.wrap(client, provider="openai")
|
|
201
|
+
assert result is client
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def test_wrap_skips_one_broken_seam_without_affecting_others(tmp_path):
|
|
205
|
+
rows = Rows()
|
|
206
|
+
_capture.set_runtime(
|
|
207
|
+
Runtime(rows, Options(app_root=str(Path(__file__).parents[1])))
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
class Completions:
|
|
211
|
+
def create(self, **kwargs):
|
|
212
|
+
return response()
|
|
213
|
+
|
|
214
|
+
class Chat:
|
|
215
|
+
completions = Completions()
|
|
216
|
+
|
|
217
|
+
class Client:
|
|
218
|
+
chat = Chat()
|
|
219
|
+
|
|
220
|
+
@property
|
|
221
|
+
def responses(self):
|
|
222
|
+
raise RuntimeError("boom: not ready yet")
|
|
223
|
+
|
|
224
|
+
client = Client()
|
|
225
|
+
metergraph.wrap(client, provider="openai")
|
|
226
|
+
client.chat.completions.create(model="gpt-test")
|
|
227
|
+
|
|
228
|
+
assert [row["endpoint"] for row in rows.rows] == ["chat.completions"]
|
|
229
|
+
_capture.set_runtime(None)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def test_wrap_never_raises_even_if_client_attribute_access_raises(caplog):
|
|
233
|
+
class Explosive:
|
|
234
|
+
@property
|
|
235
|
+
def responses(self):
|
|
236
|
+
raise RuntimeError("boom: not ready yet")
|
|
237
|
+
|
|
238
|
+
client = Explosive()
|
|
239
|
+
with caplog.at_level(logging.WARNING, logger="metergraph"):
|
|
240
|
+
result = metergraph.wrap(client) # no provider override: exercises auto-detection
|
|
241
|
+
assert result is client
|
|
242
|
+
assert any("wrap() failed" in r.getMessage() for r in caplog.records)
|
|
243
|
+
|
|
244
|
+
|
|
147
245
|
def gemini_response(text="gemini done"):
|
|
148
246
|
return SimpleNamespace(
|
|
149
247
|
text=text,
|
|
@@ -757,6 +855,31 @@ def test_canary_assignment_is_sticky_and_fail_open():
|
|
|
757
855
|
assert choose_model("route-a", "fallback", "session-1", None) == "fallback"
|
|
758
856
|
|
|
759
857
|
|
|
858
|
+
def test_config_poller_logs_generic_failures_via_failure_logger(caplog):
|
|
859
|
+
class Handler(BaseHTTPRequestHandler):
|
|
860
|
+
def do_GET(self):
|
|
861
|
+
self.send_response(500)
|
|
862
|
+
self.end_headers()
|
|
863
|
+
|
|
864
|
+
def log_message(self, *args):
|
|
865
|
+
pass
|
|
866
|
+
|
|
867
|
+
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
|
868
|
+
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
869
|
+
thread.start()
|
|
870
|
+
poller = ConfigPoller(
|
|
871
|
+
"mg_test", f"http://127.0.0.1:{server.server_port}",
|
|
872
|
+
poll_seconds=60, hard_ttl_seconds=120,
|
|
873
|
+
)
|
|
874
|
+
with caplog.at_level(logging.WARNING, logger="metergraph"):
|
|
875
|
+
ok = poller.poll_once()
|
|
876
|
+
poller.stop()
|
|
877
|
+
server.shutdown()
|
|
878
|
+
|
|
879
|
+
assert ok is False
|
|
880
|
+
assert any("config poll to" in r.getMessage() for r in caplog.records)
|
|
881
|
+
|
|
882
|
+
|
|
760
883
|
def test_record_outcome_uses_the_async_content_free_channel(monkeypatch):
|
|
761
884
|
rows = Rows()
|
|
762
885
|
monkeypatch.setattr(metergraph, "_writer", rows)
|
|
@@ -858,3 +981,173 @@ def test_writer_splits_wire_batches_at_512_kib():
|
|
|
858
981
|
assert len(wire_lengths) > 1
|
|
859
982
|
assert max(wire_lengths) <= 512 * 1024
|
|
860
983
|
assert sorted(row["index"] for row in delivered_rows) == list(range(6))
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
def test_failure_logger_logs_first_occurrence_and_suppresses_repeats(caplog):
|
|
987
|
+
now = [0.0]
|
|
988
|
+
logger = FailureLogger(quiet_seconds=60.0, clock=lambda: now[0])
|
|
989
|
+
with caplog.at_level(logging.WARNING, logger="metergraph"):
|
|
990
|
+
logger.report("transport_error", "boom 1")
|
|
991
|
+
now[0] = 10.0
|
|
992
|
+
logger.report("transport_error", "boom 2")
|
|
993
|
+
now[0] = 20.0
|
|
994
|
+
logger.report("transport_error", "boom 3")
|
|
995
|
+
messages = [r.getMessage() for r in caplog.records]
|
|
996
|
+
assert len(messages) == 1
|
|
997
|
+
assert "boom 1" in messages[0]
|
|
998
|
+
|
|
999
|
+
|
|
1000
|
+
def test_failure_logger_reports_suppressed_count_after_quiet_window(caplog):
|
|
1001
|
+
now = [0.0]
|
|
1002
|
+
logger = FailureLogger(quiet_seconds=60.0, clock=lambda: now[0])
|
|
1003
|
+
with caplog.at_level(logging.WARNING, logger="metergraph"):
|
|
1004
|
+
logger.report("transport_error", "boom 1")
|
|
1005
|
+
now[0] = 10.0
|
|
1006
|
+
logger.report("transport_error", "boom 2") # suppressed
|
|
1007
|
+
now[0] = 70.0
|
|
1008
|
+
logger.report("transport_error", "boom 3")
|
|
1009
|
+
messages = [r.getMessage() for r in caplog.records]
|
|
1010
|
+
assert len(messages) == 2
|
|
1011
|
+
assert "1 more suppressed" in messages[1]
|
|
1012
|
+
assert "boom 3" in messages[1]
|
|
1013
|
+
|
|
1014
|
+
|
|
1015
|
+
def test_failure_logger_tracks_kinds_independently(caplog):
|
|
1016
|
+
now = [0.0]
|
|
1017
|
+
logger = FailureLogger(quiet_seconds=60.0, clock=lambda: now[0])
|
|
1018
|
+
with caplog.at_level(logging.WARNING, logger="metergraph"):
|
|
1019
|
+
logger.report("transport_error", "t1")
|
|
1020
|
+
logger.report("client_error", "c1")
|
|
1021
|
+
assert len(caplog.records) == 2
|
|
1022
|
+
|
|
1023
|
+
|
|
1024
|
+
def test_failure_logger_bounds_log_volume_under_sustained_failure(caplog):
|
|
1025
|
+
now = [0.0]
|
|
1026
|
+
logger = FailureLogger(quiet_seconds=60.0, clock=lambda: now[0])
|
|
1027
|
+
with caplog.at_level(logging.WARNING, logger="metergraph"):
|
|
1028
|
+
for _ in range(1000):
|
|
1029
|
+
logger.report("transport_error", "boom")
|
|
1030
|
+
assert len(caplog.records) == 1
|
|
1031
|
+
|
|
1032
|
+
|
|
1033
|
+
def test_writer_auth_failure_is_fatal_and_logged_once(caplog):
|
|
1034
|
+
class Handler(BaseHTTPRequestHandler):
|
|
1035
|
+
def do_POST(self):
|
|
1036
|
+
self.rfile.read(int(self.headers["Content-Length"]))
|
|
1037
|
+
self.send_response(401)
|
|
1038
|
+
self.end_headers()
|
|
1039
|
+
|
|
1040
|
+
def log_message(self, *args):
|
|
1041
|
+
pass
|
|
1042
|
+
|
|
1043
|
+
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
|
1044
|
+
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
1045
|
+
thread.start()
|
|
1046
|
+
writer = Writer("mg_test", f"http://127.0.0.1:{server.server_port}", flush_seconds=5)
|
|
1047
|
+
with caplog.at_level(logging.WARNING, logger="metergraph"):
|
|
1048
|
+
writer.enqueue({"payload": "one"})
|
|
1049
|
+
writer.flush(2)
|
|
1050
|
+
writer.enqueue({"payload": "two"})
|
|
1051
|
+
writer.flush(2)
|
|
1052
|
+
writer.shutdown()
|
|
1053
|
+
server.shutdown()
|
|
1054
|
+
|
|
1055
|
+
auth_warnings = [r for r in caplog.records if "authentication failed" in r.getMessage()]
|
|
1056
|
+
assert len(auth_warnings) == 1
|
|
1057
|
+
assert writer.dropped >= 1
|
|
1058
|
+
|
|
1059
|
+
|
|
1060
|
+
def test_writer_permanent_client_error_drops_batch_but_writer_stays_alive(caplog):
|
|
1061
|
+
attempts = []
|
|
1062
|
+
|
|
1063
|
+
class Handler(BaseHTTPRequestHandler):
|
|
1064
|
+
def do_POST(self):
|
|
1065
|
+
attempts.append(1)
|
|
1066
|
+
self.rfile.read(int(self.headers["Content-Length"]))
|
|
1067
|
+
self.send_response(400)
|
|
1068
|
+
self.end_headers()
|
|
1069
|
+
|
|
1070
|
+
def log_message(self, *args):
|
|
1071
|
+
pass
|
|
1072
|
+
|
|
1073
|
+
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
|
1074
|
+
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
1075
|
+
thread.start()
|
|
1076
|
+
writer = Writer("mg_test", f"http://127.0.0.1:{server.server_port}", flush_seconds=5)
|
|
1077
|
+
with caplog.at_level(logging.WARNING, logger="metergraph"):
|
|
1078
|
+
writer.enqueue({"payload": "one"})
|
|
1079
|
+
writer.flush(2)
|
|
1080
|
+
writer.enqueue({"payload": "two"})
|
|
1081
|
+
writer.flush(2)
|
|
1082
|
+
writer.shutdown()
|
|
1083
|
+
server.shutdown()
|
|
1084
|
+
|
|
1085
|
+
# A 400 is specific to the rejected batch, not the whole connection: the
|
|
1086
|
+
# writer must still attempt the second, unrelated batch.
|
|
1087
|
+
assert len(attempts) == 2
|
|
1088
|
+
assert writer._fatal is False
|
|
1089
|
+
assert writer.dropped >= 2
|
|
1090
|
+
assert any("HTTP 400" in r.getMessage() for r in caplog.records)
|
|
1091
|
+
|
|
1092
|
+
|
|
1093
|
+
def test_writer_413_splits_oversized_batch_and_delivers_the_pieces():
|
|
1094
|
+
received_batches = []
|
|
1095
|
+
|
|
1096
|
+
class Handler(BaseHTTPRequestHandler):
|
|
1097
|
+
def do_POST(self):
|
|
1098
|
+
body = self.rfile.read(int(self.headers["Content-Length"]))
|
|
1099
|
+
if self.headers.get("Content-Encoding") == "gzip":
|
|
1100
|
+
body = gzip.decompress(body)
|
|
1101
|
+
rows = json.loads(body)["rows"]
|
|
1102
|
+
received_batches.append(len(rows))
|
|
1103
|
+
if len(rows) > 1:
|
|
1104
|
+
self.send_response(413)
|
|
1105
|
+
else:
|
|
1106
|
+
self.send_response(202)
|
|
1107
|
+
self.end_headers()
|
|
1108
|
+
|
|
1109
|
+
def log_message(self, *args):
|
|
1110
|
+
pass
|
|
1111
|
+
|
|
1112
|
+
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
|
1113
|
+
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
1114
|
+
thread.start()
|
|
1115
|
+
writer = Writer("mg_test", f"http://127.0.0.1:{server.server_port}", flush_seconds=5)
|
|
1116
|
+
for i in range(4):
|
|
1117
|
+
writer.enqueue({"index": i})
|
|
1118
|
+
assert writer.flush(10)
|
|
1119
|
+
writer.shutdown()
|
|
1120
|
+
server.shutdown()
|
|
1121
|
+
|
|
1122
|
+
assert any(size > 1 for size in received_batches) # a multi-row batch hit 413 at least once
|
|
1123
|
+
assert received_batches.count(1) == 4 # every row was eventually delivered as its own batch
|
|
1124
|
+
assert writer.dropped == 0
|
|
1125
|
+
assert writer._fatal is False
|
|
1126
|
+
|
|
1127
|
+
|
|
1128
|
+
def test_writer_server_error_retries_and_is_not_fatal(caplog):
|
|
1129
|
+
attempts = []
|
|
1130
|
+
|
|
1131
|
+
class Handler(BaseHTTPRequestHandler):
|
|
1132
|
+
def do_POST(self):
|
|
1133
|
+
self.rfile.read(int(self.headers["Content-Length"]))
|
|
1134
|
+
attempts.append(1)
|
|
1135
|
+
self.send_response(500)
|
|
1136
|
+
self.end_headers()
|
|
1137
|
+
|
|
1138
|
+
def log_message(self, *args):
|
|
1139
|
+
pass
|
|
1140
|
+
|
|
1141
|
+
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
|
1142
|
+
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
1143
|
+
thread.start()
|
|
1144
|
+
writer = Writer("mg_test", f"http://127.0.0.1:{server.server_port}", flush_seconds=5)
|
|
1145
|
+
with caplog.at_level(logging.WARNING, logger="metergraph"):
|
|
1146
|
+
writer.enqueue({"payload": "one"})
|
|
1147
|
+
writer.flush(2)
|
|
1148
|
+
writer.shutdown()
|
|
1149
|
+
server.shutdown()
|
|
1150
|
+
|
|
1151
|
+
assert len(attempts) == 1
|
|
1152
|
+
assert writer._fatal is False
|
|
1153
|
+
assert any("HTTP 500" in r.getMessage() for r in caplog.records)
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Assert every seam-table entry resolves against the real, installed
|
|
2
|
+
provider SDKs — not just against hand-built fakes, which can only ever
|
|
3
|
+
verify the code against what its author believed the SDK looked like."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from anthropic import Anthropic
|
|
8
|
+
from google import genai
|
|
9
|
+
from openai import OpenAI
|
|
10
|
+
|
|
11
|
+
from metergraph._capture import ANTHROPIC_SEAMS, GOOGLE_SEAMS, OPENAI_SEAMS, _resolve
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _missing_seams(client, seams) -> list[str]:
|
|
15
|
+
missing = []
|
|
16
|
+
for seam in seams:
|
|
17
|
+
owner = _resolve(client, seam.path)
|
|
18
|
+
if owner is None or not callable(getattr(owner, seam.method, None)):
|
|
19
|
+
missing.append(f"{seam.path}.{seam.method}")
|
|
20
|
+
return missing
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_openai_seams_exist_on_the_real_sdk():
|
|
24
|
+
client = OpenAI(api_key="test")
|
|
25
|
+
assert _missing_seams(client, OPENAI_SEAMS) == []
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_anthropic_seams_exist_on_the_real_sdk():
|
|
29
|
+
client = Anthropic(api_key="test")
|
|
30
|
+
assert _missing_seams(client, ANTHROPIC_SEAMS) == []
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_google_seams_exist_on_the_real_sdk():
|
|
34
|
+
client = genai.Client(api_key="test")
|
|
35
|
+
assert _missing_seams(client, GOOGLE_SEAMS) == []
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|