spanlight 0.1.0__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {spanlight-0.1.0 → spanlight-0.1.1}/PKG-INFO +5 -1
- {spanlight-0.1.0 → spanlight-0.1.1}/README.md +4 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/pyproject.toml +1 -1
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_experiments.py +4 -1
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_experiments_report.py +15 -3
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_exporter.py +27 -214
- spanlight-0.1.1/src/spanlight/_exporter_delivery.py +268 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_templates.py +10 -2
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_version.py +1 -1
- spanlight-0.1.1/tests/test_docs_snippets.py +155 -0
- spanlight-0.1.1/tests/test_experiments_report.py +106 -0
- spanlight-0.1.1/tests/test_exporter_retry_after.py +33 -0
- spanlight-0.1.1/tests/test_templates_surrogates.py +53 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/uv.lock +1 -1
- {spanlight-0.1.0 → spanlight-0.1.1}/.gitignore +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/LICENSE +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/examples/support_bot.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/__init__.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_client.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_config.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_context.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_decorator.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_experiments_api.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_experiments_parse.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_experiments_runs.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_experiments_setup.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_globals.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_ids.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_prompt_client.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_prompts.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_request_hash.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_scores.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_serialize.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/_span.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/integrations/__init__.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/integrations/_common.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/integrations/anthropic.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/integrations/openai.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/src/spanlight/py.typed +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/tests/conftest.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/tests/fixtures/template_cases.json +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/tests/provider_mocks.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/tests/support.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/tests/test_anthropic.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/tests/test_decorator.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/tests/test_exporter.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/tests/test_openai.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/tests/test_package.py +0 -0
- {spanlight-0.1.0 → spanlight-0.1.1}/tests/test_spans_and_context.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: spanlight
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: Python SDK for Spanlight, open-source observability for LLM applications.
|
|
5
5
|
Author: Dheeraj
|
|
6
6
|
License-Expression: MIT
|
|
@@ -313,6 +313,10 @@ specific function, use `@observe(capture_input=False, capture_output=False)`.
|
|
|
313
313
|
|
|
314
314
|
## API reference
|
|
315
315
|
|
|
316
|
+
The full API, with every signature, default and error, is in the
|
|
317
|
+
[Python SDK reference](https://github.com/dheerajkarwasra/spanlight/blob/main/docs-site/src/content/docs/reference/python-sdk.mdx).
|
|
318
|
+
The sections below are a summary.
|
|
319
|
+
|
|
316
320
|
### `init(**options) -> Spanlight`
|
|
317
321
|
|
|
318
322
|
Configures the global client used by `observe`, `span` and the wrappers.
|
|
@@ -286,6 +286,10 @@ specific function, use `@observe(capture_input=False, capture_output=False)`.
|
|
|
286
286
|
|
|
287
287
|
## API reference
|
|
288
288
|
|
|
289
|
+
The full API, with every signature, default and error, is in the
|
|
290
|
+
[Python SDK reference](https://github.com/dheerajkarwasra/spanlight/blob/main/docs-site/src/content/docs/reference/python-sdk.mdx).
|
|
291
|
+
The sections below are a summary.
|
|
292
|
+
|
|
289
293
|
### `init(**options) -> Spanlight`
|
|
290
294
|
|
|
291
295
|
Configures the global client used by `observe`, `span` and the wrappers.
|
|
@@ -120,9 +120,12 @@ def run_experiment(
|
|
|
120
120
|
experiment, version = _prepare(
|
|
121
121
|
api, dataset, prompt, evaluators, credential_id, model, max_cost_usd
|
|
122
122
|
)
|
|
123
|
-
|
|
123
|
+
experiment_url = (
|
|
124
|
+
f"{host.rstrip('/')}/{org_of(api)}/{project_id}/experiments/{experiment['id']}"
|
|
125
|
+
)
|
|
124
126
|
wait_idle(api, experiment["id"], deadline, poll_s, sleep, clock)
|
|
125
127
|
run = start_run(api, experiment["id"], version.version_id, env)
|
|
128
|
+
url = f"{experiment_url}/runs/{run['id']}"
|
|
126
129
|
try:
|
|
127
130
|
final = wait(api, experiment["id"], run["id"], deadline, poll_s, sleep, clock)
|
|
128
131
|
except ExperimentError as error:
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
from decimal import Decimal
|
|
5
|
+
from decimal import ROUND_HALF_UP, Decimal, InvalidOperation
|
|
6
6
|
from typing import Any
|
|
7
7
|
|
|
8
8
|
_PLACES = Decimal("0.000001")
|
|
@@ -65,6 +65,18 @@ def _number(value: Any) -> str:
|
|
|
65
65
|
return _UNKNOWN if value is None else str(value)
|
|
66
66
|
|
|
67
67
|
|
|
68
|
+
def _milliseconds(value: Any) -> str:
|
|
69
|
+
"""A latency in whole milliseconds; the server sends a raw float such as ``1234.5678``."""
|
|
70
|
+
if value is None:
|
|
71
|
+
return _UNKNOWN
|
|
72
|
+
if isinstance(value, bool) or not isinstance(value, int | float):
|
|
73
|
+
return str(value)
|
|
74
|
+
try:
|
|
75
|
+
return str(int(Decimal(str(value)).quantize(Decimal(1), rounding=ROUND_HALF_UP)))
|
|
76
|
+
except (InvalidOperation, ValueError, OverflowError):
|
|
77
|
+
return _UNKNOWN
|
|
78
|
+
|
|
79
|
+
|
|
68
80
|
def render_markdown(
|
|
69
81
|
*,
|
|
70
82
|
title: str,
|
|
@@ -89,8 +101,8 @@ def render_markdown(
|
|
|
89
101
|
lines += [
|
|
90
102
|
f"Items {_number(totals.get('items'))}, errors {_number(totals.get('errors'))}, "
|
|
91
103
|
f"cost {_number(totals.get('cost_usd'))} USD, "
|
|
92
|
-
f"p50 {
|
|
93
|
-
f"p95 {
|
|
104
|
+
f"p50 {_milliseconds(totals.get('latency_p50_ms'))} ms, "
|
|
105
|
+
f"p95 {_milliseconds(totals.get('latency_p95_ms'))} ms.",
|
|
94
106
|
"",
|
|
95
107
|
*_score_rows(totals, fail_below),
|
|
96
108
|
"",
|
|
@@ -11,6 +11,7 @@ Design goals, in priority order:
|
|
|
11
11
|
3. **Deliver what we can.** Batches are retried with jittered exponential
|
|
12
12
|
backoff on network errors, ``429`` and ``5xx``, honouring ``Retry-After``.
|
|
13
13
|
Other ``4xx`` responses are permanent failures and are logged, not retried.
|
|
14
|
+
The sending itself lives in :mod:`spanlight._exporter_delivery`.
|
|
14
15
|
|
|
15
16
|
Spans and scores share one queue (and so one ``max_queue`` bound). Each cycle
|
|
16
17
|
takes up to ``batch_size`` items, sends the spans first and the scores after,
|
|
@@ -19,9 +20,6 @@ so a score recorded after its span never reaches the server before it.
|
|
|
19
20
|
|
|
20
21
|
from __future__ import annotations
|
|
21
22
|
|
|
22
|
-
import email.utils
|
|
23
|
-
import gzip
|
|
24
|
-
import json
|
|
25
23
|
import logging
|
|
26
24
|
import os
|
|
27
25
|
import random
|
|
@@ -29,32 +27,15 @@ import threading
|
|
|
29
27
|
import time
|
|
30
28
|
from collections import deque
|
|
31
29
|
from collections.abc import Callable
|
|
32
|
-
from dataclasses import dataclass
|
|
33
|
-
from typing import Any
|
|
34
30
|
|
|
35
31
|
import httpx
|
|
36
32
|
|
|
37
|
-
from .
|
|
38
|
-
from ._version import __version__
|
|
39
|
-
|
|
40
|
-
logger = logging.getLogger("spanlight")
|
|
41
|
-
|
|
42
|
-
SpanPayload = dict[str, JsonValue]
|
|
43
|
-
|
|
44
|
-
_MAX_LOGGED_REJECTIONS = 10
|
|
33
|
+
from ._exporter_delivery import SCORES, SPANS, Delivery, SpanPayload, parse_retry_after
|
|
45
34
|
|
|
35
|
+
__all__ = ["BatchExporter", "SpanPayload", "parse_retry_after"]
|
|
46
36
|
|
|
47
|
-
|
|
48
|
-
class _Target:
|
|
49
|
-
"""One endpoint the exporter delivers to."""
|
|
50
|
-
|
|
51
|
-
url: str
|
|
52
|
-
body_key: str # the list's key in the request body, and the plural noun in logs
|
|
53
|
-
id_field: str # the payload field logged when the server rejects an item
|
|
54
|
-
|
|
37
|
+
logger = logging.getLogger("spanlight")
|
|
55
38
|
|
|
56
|
-
_SPANS = "spans"
|
|
57
|
-
_SCORES = "scores"
|
|
58
39
|
_Item = tuple[str, SpanPayload]
|
|
59
40
|
|
|
60
41
|
|
|
@@ -100,42 +81,32 @@ class BatchExporter:
|
|
|
100
81
|
base_backoff: float = 0.5,
|
|
101
82
|
max_backoff: float = 30.0,
|
|
102
83
|
) -> None:
|
|
103
|
-
self.
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
84
|
+
self._delivery = Delivery(
|
|
85
|
+
url=url,
|
|
86
|
+
api_key=api_key,
|
|
87
|
+
scores_url=scores_url,
|
|
88
|
+
gzip_body=gzip_body,
|
|
89
|
+
timeout=timeout,
|
|
90
|
+
max_retries=max_retries,
|
|
91
|
+
transport=transport,
|
|
92
|
+
sleep=sleep,
|
|
93
|
+
jitter=jitter,
|
|
94
|
+
base_backoff=base_backoff,
|
|
95
|
+
max_backoff=max_backoff,
|
|
96
|
+
)
|
|
114
97
|
self._batch_size = batch_size
|
|
115
98
|
self._flush_interval = flush_interval
|
|
116
99
|
self._max_queue = max_queue
|
|
117
|
-
self._gzip = gzip_body
|
|
118
|
-
self._timeout = timeout
|
|
119
|
-
self._max_retries = max_retries
|
|
120
|
-
self._transport = transport
|
|
121
|
-
self._sleep = sleep if sleep is not None else self._interruptible_sleep
|
|
122
|
-
self._jitter = jitter
|
|
123
|
-
self._base_backoff = base_backoff
|
|
124
|
-
self._max_backoff = max_backoff
|
|
125
100
|
|
|
126
101
|
self._condition = threading.Condition()
|
|
127
102
|
self._queue: deque[_Item] = deque()
|
|
128
103
|
self._in_flight = 0
|
|
129
104
|
self._flush_requested = False
|
|
130
105
|
self._closed = False
|
|
131
|
-
self._abort_retries = threading.Event()
|
|
132
106
|
self._worker: threading.Thread | None = None
|
|
133
|
-
self._client: httpx.Client | None = None
|
|
134
107
|
self._pid = os.getpid()
|
|
135
108
|
|
|
136
109
|
self._dropped = 0
|
|
137
|
-
self._failed = 0
|
|
138
|
-
self._sent = 0
|
|
139
110
|
|
|
140
111
|
# ------------------------------------------------------------------ #
|
|
141
112
|
# Public API (called from user threads)
|
|
@@ -149,23 +120,23 @@ class BatchExporter:
|
|
|
149
120
|
@property
|
|
150
121
|
def failed_count(self) -> int:
|
|
151
122
|
"""Spans and scores lost because their batch could not be delivered."""
|
|
152
|
-
return self.
|
|
123
|
+
return self._delivery.failed
|
|
153
124
|
|
|
154
125
|
@property
|
|
155
126
|
def sent_count(self) -> int:
|
|
156
127
|
"""Spans and scores delivered and accepted by the server."""
|
|
157
|
-
return self.
|
|
128
|
+
return self._delivery.sent
|
|
158
129
|
|
|
159
130
|
def export(self, span: SpanPayload) -> None:
|
|
160
131
|
"""Enqueue one serialized span. Never blocks on I/O and never raises."""
|
|
161
|
-
self._enqueue(
|
|
132
|
+
self._enqueue(SPANS, span)
|
|
162
133
|
|
|
163
134
|
def export_score(self, score: SpanPayload) -> None:
|
|
164
135
|
"""Enqueue one serialized score. Never blocks on I/O and never raises."""
|
|
165
|
-
if
|
|
136
|
+
if not self._delivery.accepts(SCORES):
|
|
166
137
|
logger.debug("No score endpoint configured; dropping a score")
|
|
167
138
|
return
|
|
168
|
-
self._enqueue(
|
|
139
|
+
self._enqueue(SCORES, score)
|
|
169
140
|
|
|
170
141
|
def _enqueue(self, kind: str, payload: SpanPayload) -> None:
|
|
171
142
|
try:
|
|
@@ -227,12 +198,11 @@ class BatchExporter:
|
|
|
227
198
|
with self._condition:
|
|
228
199
|
self._closed = True
|
|
229
200
|
self._condition.notify_all()
|
|
230
|
-
self.
|
|
201
|
+
self._delivery.abort_retries()
|
|
231
202
|
worker = self._worker
|
|
232
203
|
if worker is not None and worker is not threading.current_thread():
|
|
233
204
|
worker.join(max(deadline - time.monotonic(), 0.0))
|
|
234
|
-
|
|
235
|
-
self._client.close()
|
|
205
|
+
self._delivery.close()
|
|
236
206
|
except Exception:
|
|
237
207
|
logger.debug("Shutdown failed", exc_info=True)
|
|
238
208
|
|
|
@@ -249,7 +219,7 @@ class BatchExporter:
|
|
|
249
219
|
self._queue = deque()
|
|
250
220
|
self._in_flight = 0
|
|
251
221
|
self._worker = None
|
|
252
|
-
self.
|
|
222
|
+
self._delivery.forget_client()
|
|
253
223
|
|
|
254
224
|
def _ensure_worker(self) -> None:
|
|
255
225
|
with self._condition:
|
|
@@ -270,10 +240,10 @@ class BatchExporter:
|
|
|
270
240
|
if batch is None:
|
|
271
241
|
return
|
|
272
242
|
try:
|
|
273
|
-
for kind in (
|
|
243
|
+
for kind in (SPANS, SCORES):
|
|
274
244
|
payloads = [payload for item_kind, payload in batch if item_kind == kind]
|
|
275
245
|
if payloads:
|
|
276
|
-
self.
|
|
246
|
+
self._delivery.send(kind, payloads)
|
|
277
247
|
finally:
|
|
278
248
|
with self._condition:
|
|
279
249
|
self._in_flight -= len(batch)
|
|
@@ -303,160 +273,3 @@ class BatchExporter:
|
|
|
303
273
|
if self._closed or self._flush_requested:
|
|
304
274
|
return False
|
|
305
275
|
return len(self._queue) < self._batch_size
|
|
306
|
-
|
|
307
|
-
def _http_client(self) -> httpx.Client:
|
|
308
|
-
if self._client is None:
|
|
309
|
-
self._client = httpx.Client(transport=self._transport, timeout=self._timeout)
|
|
310
|
-
return self._client
|
|
311
|
-
|
|
312
|
-
def _send_guarded(self, target: _Target, batch: list[SpanPayload]) -> None:
|
|
313
|
-
"""Send one target's part of a cycle; a crash here never stops the next part."""
|
|
314
|
-
try:
|
|
315
|
-
self._send_batch(target, batch)
|
|
316
|
-
except Exception:
|
|
317
|
-
self._failed += len(batch)
|
|
318
|
-
logger.warning(
|
|
319
|
-
"Unexpected error exporting %d %s", len(batch), target.body_key, exc_info=True
|
|
320
|
-
)
|
|
321
|
-
|
|
322
|
-
def _send_batch(self, target: _Target, batch: list[SpanPayload]) -> None:
|
|
323
|
-
if not batch:
|
|
324
|
-
return
|
|
325
|
-
body = json.dumps({target.body_key: batch}, separators=(",", ":"), default=str).encode()
|
|
326
|
-
if self._gzip:
|
|
327
|
-
body = gzip.compress(body)
|
|
328
|
-
|
|
329
|
-
for attempt in range(self._max_retries + 1):
|
|
330
|
-
retry_after: float | None = None
|
|
331
|
-
try:
|
|
332
|
-
response = self._http_client().post(target.url, content=body, headers=self._headers)
|
|
333
|
-
except httpx.HTTPError as exc:
|
|
334
|
-
failure = f"{type(exc).__name__}: {exc}"
|
|
335
|
-
else:
|
|
336
|
-
status = response.status_code
|
|
337
|
-
if 200 <= status < 300:
|
|
338
|
-
self._handle_success(target, response, batch)
|
|
339
|
-
return
|
|
340
|
-
if status == 413 and len(batch) > 1:
|
|
341
|
-
self._split_and_send(target, batch)
|
|
342
|
-
return
|
|
343
|
-
if not _is_retryable(status):
|
|
344
|
-
self._failed += len(batch)
|
|
345
|
-
self._log_rejected_batch(target, response, len(batch))
|
|
346
|
-
return
|
|
347
|
-
failure = f"HTTP {status}"
|
|
348
|
-
retry_after = parse_retry_after(response.headers.get("Retry-After"))
|
|
349
|
-
|
|
350
|
-
if attempt >= self._max_retries or self._abort_retries.is_set():
|
|
351
|
-
self._failed += len(batch)
|
|
352
|
-
logger.warning(
|
|
353
|
-
"Dropping %d %s after %d attempts (%s)",
|
|
354
|
-
len(batch),
|
|
355
|
-
target.body_key,
|
|
356
|
-
attempt + 1,
|
|
357
|
-
failure,
|
|
358
|
-
)
|
|
359
|
-
return
|
|
360
|
-
delay = self._retry_delay(attempt, retry_after)
|
|
361
|
-
logger.debug("Export failed (%s); retrying in %.2fs", failure, delay)
|
|
362
|
-
self._sleep(delay)
|
|
363
|
-
|
|
364
|
-
def _split_and_send(self, target: _Target, batch: list[SpanPayload]) -> None:
|
|
365
|
-
"""Retry an oversized (HTTP 413) batch as two smaller ones."""
|
|
366
|
-
middle = len(batch) // 2
|
|
367
|
-
self._send_batch(target, batch[:middle])
|
|
368
|
-
self._send_batch(target, batch[middle:])
|
|
369
|
-
|
|
370
|
-
def _log_rejected_batch(self, target: _Target, response: httpx.Response, size: int) -> None:
|
|
371
|
-
if target.body_key == _SCORES and response.status_code == 404:
|
|
372
|
-
# A server older than score ingestion: say so once instead of on every batch.
|
|
373
|
-
if not self._scores_unsupported_logged:
|
|
374
|
-
self._scores_unsupported_logged = True
|
|
375
|
-
logger.warning(
|
|
376
|
-
"This Spanlight server does not accept scores (HTTP 404); dropping them"
|
|
377
|
-
)
|
|
378
|
-
return
|
|
379
|
-
logger.error(
|
|
380
|
-
"Spanlight rejected a batch of %d %s: HTTP %d %s",
|
|
381
|
-
size,
|
|
382
|
-
target.body_key,
|
|
383
|
-
response.status_code,
|
|
384
|
-
_short_body(response),
|
|
385
|
-
)
|
|
386
|
-
|
|
387
|
-
def _handle_success(
|
|
388
|
-
self, target: _Target, response: httpx.Response, batch: list[SpanPayload]
|
|
389
|
-
) -> None:
|
|
390
|
-
batch_size = len(batch)
|
|
391
|
-
try:
|
|
392
|
-
result: Any = response.json()
|
|
393
|
-
except ValueError:
|
|
394
|
-
self._sent += batch_size
|
|
395
|
-
return
|
|
396
|
-
rejected = result.get("rejected") if isinstance(result, dict) else None
|
|
397
|
-
if not isinstance(rejected, list) or not rejected:
|
|
398
|
-
self._sent += batch_size
|
|
399
|
-
return
|
|
400
|
-
|
|
401
|
-
self._sent += batch_size - len(rejected)
|
|
402
|
-
self._failed += len(rejected)
|
|
403
|
-
logger.warning(
|
|
404
|
-
"Spanlight rejected %d of %d %s",
|
|
405
|
-
len(rejected),
|
|
406
|
-
batch_size,
|
|
407
|
-
target.body_key,
|
|
408
|
-
)
|
|
409
|
-
for item in rejected[:_MAX_LOGGED_REJECTIONS]:
|
|
410
|
-
if isinstance(item, dict):
|
|
411
|
-
index = item.get("index")
|
|
412
|
-
sent = batch[index] if isinstance(index, int) and 0 <= index < batch_size else {}
|
|
413
|
-
logger.warning(
|
|
414
|
-
" %s %s (index %s): %s",
|
|
415
|
-
target.body_key[:-1],
|
|
416
|
-
item.get(target.id_field) or sent.get(target.id_field),
|
|
417
|
-
index,
|
|
418
|
-
item.get("reason"),
|
|
419
|
-
)
|
|
420
|
-
|
|
421
|
-
def _retry_delay(self, attempt: int, retry_after: float | None) -> float:
|
|
422
|
-
"""Return seconds to wait before retry number ``attempt + 1``.
|
|
423
|
-
|
|
424
|
-
``Retry-After`` wins when present. Otherwise use "equal jitter"
|
|
425
|
-
exponential backoff: half of the exponential step is fixed and the
|
|
426
|
-
other half random, which spreads retries out without ever retrying
|
|
427
|
-
immediately.
|
|
428
|
-
"""
|
|
429
|
-
if retry_after is not None:
|
|
430
|
-
return min(max(retry_after, 0.0), self._max_backoff)
|
|
431
|
-
step = min(self._max_backoff, self._base_backoff * (2.0**attempt))
|
|
432
|
-
return step / 2 + self._jitter() * step / 2
|
|
433
|
-
|
|
434
|
-
def _interruptible_sleep(self, seconds: float) -> None:
|
|
435
|
-
self._abort_retries.wait(seconds)
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
def _is_retryable(status: int) -> bool:
|
|
439
|
-
return status == 429 or status >= 500
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
def parse_retry_after(value: str | None) -> float | None:
|
|
443
|
-
"""Parse a ``Retry-After`` header (delta-seconds or HTTP-date) into seconds."""
|
|
444
|
-
if value is None:
|
|
445
|
-
return None
|
|
446
|
-
value = value.strip()
|
|
447
|
-
try:
|
|
448
|
-
return max(float(value), 0.0)
|
|
449
|
-
except ValueError:
|
|
450
|
-
pass
|
|
451
|
-
try:
|
|
452
|
-
moment = email.utils.parsedate_to_datetime(value)
|
|
453
|
-
except (TypeError, ValueError):
|
|
454
|
-
return None
|
|
455
|
-
return max(moment.timestamp() - time.time(), 0.0)
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
def _short_body(response: httpx.Response) -> str:
|
|
459
|
-
try:
|
|
460
|
-
return response.text[:500]
|
|
461
|
-
except Exception:
|
|
462
|
-
return "<unreadable body>"
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
"""Delivery of one batch to the ingestion API: retries, backoff, splitting and logging.
|
|
2
|
+
|
|
3
|
+
Runs only on the exporter's worker thread. Batches are retried with jittered
|
|
4
|
+
exponential backoff on network errors, ``429`` and ``5xx``, honouring
|
|
5
|
+
``Retry-After``. Other ``4xx`` responses are permanent failures and are logged,
|
|
6
|
+
not retried. Nothing in this module raises into the caller.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import email.utils
|
|
12
|
+
import gzip
|
|
13
|
+
import json
|
|
14
|
+
import logging
|
|
15
|
+
import math
|
|
16
|
+
import threading
|
|
17
|
+
import time
|
|
18
|
+
from collections.abc import Callable
|
|
19
|
+
from dataclasses import dataclass
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
import httpx
|
|
23
|
+
|
|
24
|
+
from ._serialize import JsonValue
|
|
25
|
+
from ._version import __version__
|
|
26
|
+
|
|
27
|
+
logger = logging.getLogger("spanlight")
|
|
28
|
+
|
|
29
|
+
SpanPayload = dict[str, JsonValue]
|
|
30
|
+
|
|
31
|
+
SPANS = "spans"
|
|
32
|
+
SCORES = "scores"
|
|
33
|
+
|
|
34
|
+
_MAX_LOGGED_REJECTIONS = 10
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class _Target:
|
|
39
|
+
"""One endpoint the exporter delivers to."""
|
|
40
|
+
|
|
41
|
+
url: str
|
|
42
|
+
body_key: str # the list's key in the request body, and the plural noun in logs
|
|
43
|
+
id_field: str # the payload field logged when the server rejects an item
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class Delivery:
|
|
47
|
+
"""Send batches of spans or scores, and count what was accepted and what was lost.
|
|
48
|
+
|
|
49
|
+
The arguments are those of :class:`spanlight._exporter.BatchExporter` that
|
|
50
|
+
concern the network.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
*,
|
|
56
|
+
url: str,
|
|
57
|
+
api_key: str,
|
|
58
|
+
scores_url: str | None,
|
|
59
|
+
gzip_body: bool,
|
|
60
|
+
timeout: float,
|
|
61
|
+
max_retries: int,
|
|
62
|
+
transport: httpx.BaseTransport | None,
|
|
63
|
+
sleep: Callable[[float], None] | None,
|
|
64
|
+
jitter: Callable[[], float],
|
|
65
|
+
base_backoff: float,
|
|
66
|
+
max_backoff: float,
|
|
67
|
+
) -> None:
|
|
68
|
+
self._targets: dict[str, _Target] = {SPANS: _Target(url, SPANS, "span_id")}
|
|
69
|
+
if scores_url is not None:
|
|
70
|
+
self._targets[SCORES] = _Target(scores_url, SCORES, "id")
|
|
71
|
+
self._scores_unsupported_logged = False
|
|
72
|
+
self._headers = {
|
|
73
|
+
"Authorization": f"Bearer {api_key}",
|
|
74
|
+
"Content-Type": "application/json",
|
|
75
|
+
"User-Agent": f"spanlight-python/{__version__}",
|
|
76
|
+
}
|
|
77
|
+
if gzip_body:
|
|
78
|
+
self._headers["Content-Encoding"] = "gzip"
|
|
79
|
+
self._gzip = gzip_body
|
|
80
|
+
self._timeout = timeout
|
|
81
|
+
self._max_retries = max_retries
|
|
82
|
+
self._transport = transport
|
|
83
|
+
self._sleep = sleep if sleep is not None else self._interruptible_sleep
|
|
84
|
+
self._jitter = jitter
|
|
85
|
+
self._base_backoff = base_backoff
|
|
86
|
+
self._max_backoff = max_backoff
|
|
87
|
+
self._abort_retries = threading.Event()
|
|
88
|
+
self._client: httpx.Client | None = None
|
|
89
|
+
self.failed = 0
|
|
90
|
+
self.sent = 0
|
|
91
|
+
|
|
92
|
+
def accepts(self, kind: str) -> bool:
|
|
93
|
+
"""Whether an endpoint is configured for ``kind`` (``SPANS`` or ``SCORES``)."""
|
|
94
|
+
return kind in self._targets
|
|
95
|
+
|
|
96
|
+
def send(self, kind: str, batch: list[SpanPayload]) -> None:
|
|
97
|
+
"""Send one kind's part of a cycle; a crash here never stops the next part."""
|
|
98
|
+
target = self._targets[kind]
|
|
99
|
+
try:
|
|
100
|
+
self._send_batch(target, batch)
|
|
101
|
+
except Exception:
|
|
102
|
+
self.failed += len(batch)
|
|
103
|
+
logger.warning(
|
|
104
|
+
"Unexpected error exporting %d %s", len(batch), target.body_key, exc_info=True
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
def abort_retries(self) -> None:
|
|
108
|
+
"""Stop waiting between retries: the batch in progress gets no further attempt."""
|
|
109
|
+
self._abort_retries.set()
|
|
110
|
+
|
|
111
|
+
def forget_client(self) -> None:
|
|
112
|
+
"""Drop the HTTP client without closing it (it belongs to a parent process)."""
|
|
113
|
+
self._client = None
|
|
114
|
+
|
|
115
|
+
def close(self) -> None:
|
|
116
|
+
"""Close the HTTP client, if one was opened."""
|
|
117
|
+
if self._client is not None:
|
|
118
|
+
self._client.close()
|
|
119
|
+
|
|
120
|
+
def _http_client(self) -> httpx.Client:
|
|
121
|
+
if self._client is None:
|
|
122
|
+
self._client = httpx.Client(transport=self._transport, timeout=self._timeout)
|
|
123
|
+
return self._client
|
|
124
|
+
|
|
125
|
+
def _send_batch(self, target: _Target, batch: list[SpanPayload]) -> None:
|
|
126
|
+
if not batch:
|
|
127
|
+
return
|
|
128
|
+
body = json.dumps({target.body_key: batch}, separators=(",", ":"), default=str).encode()
|
|
129
|
+
if self._gzip:
|
|
130
|
+
body = gzip.compress(body)
|
|
131
|
+
|
|
132
|
+
for attempt in range(self._max_retries + 1):
|
|
133
|
+
retry_after: float | None = None
|
|
134
|
+
try:
|
|
135
|
+
response = self._http_client().post(target.url, content=body, headers=self._headers)
|
|
136
|
+
except httpx.HTTPError as exc:
|
|
137
|
+
failure = f"{type(exc).__name__}: {exc}"
|
|
138
|
+
else:
|
|
139
|
+
status = response.status_code
|
|
140
|
+
if 200 <= status < 300:
|
|
141
|
+
self._handle_success(target, response, batch)
|
|
142
|
+
return
|
|
143
|
+
if status == 413 and len(batch) > 1:
|
|
144
|
+
self._split_and_send(target, batch)
|
|
145
|
+
return
|
|
146
|
+
if not _is_retryable(status):
|
|
147
|
+
self.failed += len(batch)
|
|
148
|
+
self._log_rejected_batch(target, response, len(batch))
|
|
149
|
+
return
|
|
150
|
+
failure = f"HTTP {status}"
|
|
151
|
+
retry_after = parse_retry_after(response.headers.get("Retry-After"))
|
|
152
|
+
|
|
153
|
+
if attempt >= self._max_retries or self._abort_retries.is_set():
|
|
154
|
+
self.failed += len(batch)
|
|
155
|
+
logger.warning(
|
|
156
|
+
"Dropping %d %s after %d attempts (%s)",
|
|
157
|
+
len(batch),
|
|
158
|
+
target.body_key,
|
|
159
|
+
attempt + 1,
|
|
160
|
+
failure,
|
|
161
|
+
)
|
|
162
|
+
return
|
|
163
|
+
delay = self.retry_delay(attempt, retry_after)
|
|
164
|
+
logger.debug("Export failed (%s); retrying in %.2fs", failure, delay)
|
|
165
|
+
self._sleep(delay)
|
|
166
|
+
|
|
167
|
+
def _split_and_send(self, target: _Target, batch: list[SpanPayload]) -> None:
|
|
168
|
+
"""Retry an oversized (HTTP 413) batch as two smaller ones."""
|
|
169
|
+
middle = len(batch) // 2
|
|
170
|
+
self._send_batch(target, batch[:middle])
|
|
171
|
+
self._send_batch(target, batch[middle:])
|
|
172
|
+
|
|
173
|
+
def _log_rejected_batch(self, target: _Target, response: httpx.Response, size: int) -> None:
|
|
174
|
+
if target.body_key == SCORES and response.status_code == 404:
|
|
175
|
+
# A server older than score ingestion: say so once instead of on every batch.
|
|
176
|
+
if not self._scores_unsupported_logged:
|
|
177
|
+
self._scores_unsupported_logged = True
|
|
178
|
+
logger.warning(
|
|
179
|
+
"This Spanlight server does not accept scores (HTTP 404); dropping them"
|
|
180
|
+
)
|
|
181
|
+
return
|
|
182
|
+
logger.error(
|
|
183
|
+
"Spanlight rejected a batch of %d %s: HTTP %d %s",
|
|
184
|
+
size,
|
|
185
|
+
target.body_key,
|
|
186
|
+
response.status_code,
|
|
187
|
+
_short_body(response),
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
def _handle_success(
|
|
191
|
+
self, target: _Target, response: httpx.Response, batch: list[SpanPayload]
|
|
192
|
+
) -> None:
|
|
193
|
+
batch_size = len(batch)
|
|
194
|
+
try:
|
|
195
|
+
result: Any = response.json()
|
|
196
|
+
except ValueError:
|
|
197
|
+
self.sent += batch_size
|
|
198
|
+
return
|
|
199
|
+
rejected = result.get("rejected") if isinstance(result, dict) else None
|
|
200
|
+
if not isinstance(rejected, list) or not rejected:
|
|
201
|
+
self.sent += batch_size
|
|
202
|
+
return
|
|
203
|
+
|
|
204
|
+
self.sent += batch_size - len(rejected)
|
|
205
|
+
self.failed += len(rejected)
|
|
206
|
+
logger.warning(
|
|
207
|
+
"Spanlight rejected %d of %d %s",
|
|
208
|
+
len(rejected),
|
|
209
|
+
batch_size,
|
|
210
|
+
target.body_key,
|
|
211
|
+
)
|
|
212
|
+
for item in rejected[:_MAX_LOGGED_REJECTIONS]:
|
|
213
|
+
if isinstance(item, dict):
|
|
214
|
+
index = item.get("index")
|
|
215
|
+
sent = batch[index] if isinstance(index, int) and 0 <= index < batch_size else {}
|
|
216
|
+
logger.warning(
|
|
217
|
+
" %s %s (index %s): %s",
|
|
218
|
+
target.body_key[:-1],
|
|
219
|
+
item.get(target.id_field) or sent.get(target.id_field),
|
|
220
|
+
index,
|
|
221
|
+
item.get("reason"),
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
def retry_delay(self, attempt: int, retry_after: float | None) -> float:
|
|
225
|
+
"""Return seconds to wait before retry number ``attempt + 1``.
|
|
226
|
+
|
|
227
|
+
``Retry-After`` wins when present. Otherwise use "equal jitter"
|
|
228
|
+
exponential backoff: half of the exponential step is fixed and the
|
|
229
|
+
other half random, which spreads retries out without ever retrying
|
|
230
|
+
immediately.
|
|
231
|
+
"""
|
|
232
|
+
if retry_after is not None:
|
|
233
|
+
return min(max(retry_after, 0.0), self._max_backoff)
|
|
234
|
+
step = min(self._max_backoff, self._base_backoff * (2.0**attempt))
|
|
235
|
+
return step / 2 + self._jitter() * step / 2
|
|
236
|
+
|
|
237
|
+
def _interruptible_sleep(self, seconds: float) -> None:
|
|
238
|
+
self._abort_retries.wait(seconds)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _is_retryable(status: int) -> bool:
|
|
242
|
+
return status == 429 or status >= 500
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def parse_retry_after(value: str | None) -> float | None:
|
|
246
|
+
"""Parse a ``Retry-After`` header (delta-seconds or HTTP-date) into seconds."""
|
|
247
|
+
if value is None:
|
|
248
|
+
return None
|
|
249
|
+
value = value.strip()
|
|
250
|
+
try:
|
|
251
|
+
seconds = float(value)
|
|
252
|
+
except ValueError:
|
|
253
|
+
pass
|
|
254
|
+
else:
|
|
255
|
+
# `nan` and `inf` parse as floats but are not delays; fall back to the backoff.
|
|
256
|
+
return max(seconds, 0.0) if math.isfinite(seconds) else None
|
|
257
|
+
try:
|
|
258
|
+
moment = email.utils.parsedate_to_datetime(value)
|
|
259
|
+
except (TypeError, ValueError):
|
|
260
|
+
return None
|
|
261
|
+
return max(moment.timestamp() - time.time(), 0.0)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _short_body(response: httpx.Response) -> str:
|
|
265
|
+
try:
|
|
266
|
+
return response.text[:500]
|
|
267
|
+
except Exception:
|
|
268
|
+
return "<unreadable body>"
|
|
@@ -127,7 +127,12 @@ def _sources(template: Template) -> list[str]:
|
|
|
127
127
|
blocks = [message.content for message in template.messages]
|
|
128
128
|
else:
|
|
129
129
|
raise InvalidTemplateError(f"unknown template kind: {template.kind!r}")
|
|
130
|
-
|
|
130
|
+
try:
|
|
131
|
+
size = sum(len(block.encode("utf-8")) for block in blocks)
|
|
132
|
+
except UnicodeEncodeError:
|
|
133
|
+
# A lone surrogate: not valid Unicode, and the server would refuse it as a template.
|
|
134
|
+
raise InvalidTemplateError("template text is not valid Unicode") from None
|
|
135
|
+
if size > MAX_TEMPLATE_BYTES:
|
|
131
136
|
raise InvalidTemplateError(f"template exceeds {MAX_TEMPLATE_BYTES} bytes")
|
|
132
137
|
return blocks
|
|
133
138
|
|
|
@@ -156,7 +161,10 @@ def _check_size(parts_per_block: list[list[_Part]], variables: Mapping[str, str]
|
|
|
156
161
|
total += len(part.literal.encode("utf-8"))
|
|
157
162
|
else:
|
|
158
163
|
if part.name not in sizes:
|
|
159
|
-
|
|
164
|
+
# `surrogatepass`: a value may carry a lone surrogate (from `os.fsdecode`,
|
|
165
|
+
# say); count it rather than raise, since the value is the caller's data.
|
|
166
|
+
value = variables[part.name]
|
|
167
|
+
sizes[part.name] = len(value.encode("utf-8", "surrogatepass"))
|
|
160
168
|
total += sizes[part.name]
|
|
161
169
|
if total > MAX_COMPILED_BYTES:
|
|
162
170
|
raise TemplateTooLargeError()
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""The Python examples in the documentation run against the real SDK.
|
|
2
|
+
|
|
3
|
+
A fence marked with ``{/* snippet-test */}`` (MDX) or ``<!-- snippet-test -->`` (Markdown) on
|
|
4
|
+
the line before it is executed here. The SDK points at an in-memory ingestion API, so no
|
|
5
|
+
snippet touches the network or a model provider. A snippet fails the test when it raises or
|
|
6
|
+
when it sends no span to ``POST /v1/traces``.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
from collections.abc import Iterator
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
import httpx
|
|
17
|
+
import pytest
|
|
18
|
+
from support import FakeIngestApi, accept_all, decode_body
|
|
19
|
+
|
|
20
|
+
import spanlight
|
|
21
|
+
|
|
22
|
+
DOCS_ROOT = Path(__file__).resolve().parents[3] / "docs-site" / "src" / "content" / "docs"
|
|
23
|
+
|
|
24
|
+
# The marker on its own line, then a ```python fence: the snippet is the fence body. The fence may
|
|
25
|
+
# be indented (inside <Steps> or <Tabs>); the closing fence must have the same indent, and that
|
|
26
|
+
# indent is removed from every code line.
|
|
27
|
+
_MARKER = r"(?:\{/\*\s*snippet-test\s*\*/\}|<!--\s*snippet-test\s*-->)"
|
|
28
|
+
_MARKER_LINE = re.compile(rf"^[ \t]*{_MARKER}[ \t]*$", re.MULTILINE)
|
|
29
|
+
_SNIPPET = re.compile(
|
|
30
|
+
rf"^[ \t]*{_MARKER}[ \t]*\n(?P<indent>[ \t]*)```python[^\n]*\n"
|
|
31
|
+
rf"(?P<code>.*?)^(?P=indent)```[ \t]*$",
|
|
32
|
+
re.MULTILINE | re.DOTALL,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _extract(text: str) -> list[str]:
|
|
37
|
+
"""The code of every marked fence in ``text``, with the fence's indent removed."""
|
|
38
|
+
return [_dedent(match["code"], match["indent"]) for match in _SNIPPET.finditer(text)]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _dedent(code: str, indent: str) -> str:
|
|
42
|
+
"""Remove exactly ``indent`` from the start of each line (a blank line may lack it)."""
|
|
43
|
+
return "".join(line.removeprefix(indent) for line in code.splitlines(keepends=True))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _pages() -> Iterator[Path]:
|
|
47
|
+
for pattern in ("*.md", "*.mdx"):
|
|
48
|
+
yield from DOCS_ROOT.rglob(pattern)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _snippets() -> list[tuple[str, str]]:
|
|
52
|
+
"""Every marked snippet as ``(id, code)``, id being ``<page>::<index>`` (index from 1)."""
|
|
53
|
+
found: list[tuple[str, str]] = []
|
|
54
|
+
for page in sorted(_pages()):
|
|
55
|
+
text = page.read_text(encoding="utf-8")
|
|
56
|
+
for index, code in enumerate(_extract(text), start=1):
|
|
57
|
+
found.append((f"{page.relative_to(DOCS_ROOT).as_posix()}::{index}", code))
|
|
58
|
+
return found
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
SNIPPETS = _snippets()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _handler(request: httpx.Request) -> httpx.Response:
|
|
65
|
+
path = request.url.path
|
|
66
|
+
if request.method == "GET":
|
|
67
|
+
# Prompts: nothing is stored, so `prompts.get(..., fallback=...)` falls back.
|
|
68
|
+
return httpx.Response(404, json={"title": "Not found"})
|
|
69
|
+
if path.endswith("/v1/scores"):
|
|
70
|
+
return httpx.Response(200, json={"accepted": 1, "rejected": []})
|
|
71
|
+
return accept_all(request)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _traces_posts(ingest: FakeIngestApi) -> list[httpx.Request]:
|
|
75
|
+
return [
|
|
76
|
+
request
|
|
77
|
+
for request in ingest.requests
|
|
78
|
+
if request.method == "POST" and request.url.path == "/v1/traces"
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _sent_spans(ingest: FakeIngestApi) -> list[dict[str, Any]]:
|
|
83
|
+
return [span for request in _traces_posts(ingest) for span in decode_body(request)["spans"]]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@pytest.fixture
|
|
87
|
+
def snippet_sdk(ingest: FakeIngestApi, monkeypatch: pytest.MonkeyPatch) -> FakeIngestApi:
|
|
88
|
+
"""Point the global SDK at ``ingest``, also for snippets that call ``spanlight.init``.
|
|
89
|
+
|
|
90
|
+
The shared ``_isolate_environment`` fixture shuts the global client down after the test, so
|
|
91
|
+
snippets never see one another's state.
|
|
92
|
+
"""
|
|
93
|
+
ingest.handler = _handler
|
|
94
|
+
real_init = spanlight.init
|
|
95
|
+
|
|
96
|
+
def init(**options: Any) -> spanlight.Spanlight:
|
|
97
|
+
options["transport"] = ingest.transport()
|
|
98
|
+
options.setdefault("api_key", "spl_live_test")
|
|
99
|
+
options.setdefault("host", "http://test")
|
|
100
|
+
return real_init(**options)
|
|
101
|
+
|
|
102
|
+
monkeypatch.setattr(spanlight, "init", init)
|
|
103
|
+
spanlight.init()
|
|
104
|
+
return ingest
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_docs_have_marked_snippets():
|
|
108
|
+
markers = sum(len(_MARKER_LINE.findall(page.read_text(encoding="utf-8"))) for page in _pages())
|
|
109
|
+
assert SNIPPETS, "no page marks a Python snippet with snippet-test"
|
|
110
|
+
assert markers == len(SNIPPETS), (
|
|
111
|
+
"a snippet-test marker is not directly followed by a ```python fence "
|
|
112
|
+
f"({markers} markers, {len(SNIPPETS)} snippets)"
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@pytest.mark.parametrize(("snippet_id", "code"), SNIPPETS, ids=[item[0] for item in SNIPPETS])
|
|
117
|
+
def test_documented_snippets_run(snippet_id: str, code: str, snippet_sdk: FakeIngestApi):
|
|
118
|
+
namespace: dict[str, Any] = {"__name__": "__docs_snippet__"}
|
|
119
|
+
exec(compile(code, snippet_id, "exec"), namespace)
|
|
120
|
+
|
|
121
|
+
assert spanlight.flush(timeout=5), f"{snippet_id}: flush timed out"
|
|
122
|
+
assert _traces_posts(snippet_sdk), f"{snippet_id}: nothing was posted to /v1/traces"
|
|
123
|
+
assert _sent_spans(snippet_sdk), f"{snippet_id}: the batches held no span"
|
|
124
|
+
exporter = spanlight.get_client().exporter
|
|
125
|
+
assert exporter is not None
|
|
126
|
+
assert exporter.failed_count == 0, f"{snippet_id}: the server rejected spans"
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def test_indented_fence_is_extracted_and_dedented():
|
|
130
|
+
text = (
|
|
131
|
+
"<Steps>\n\n"
|
|
132
|
+
"1. Trace it.\n\n"
|
|
133
|
+
" {/* snippet-test */}\n"
|
|
134
|
+
" ```python\n"
|
|
135
|
+
" def handle(question):\n"
|
|
136
|
+
" return question\n"
|
|
137
|
+
"\n"
|
|
138
|
+
" handle('hi')\n"
|
|
139
|
+
" ```\n\n"
|
|
140
|
+
"</Steps>\n\n"
|
|
141
|
+
"<!-- snippet-test -->\n"
|
|
142
|
+
"```python\n"
|
|
143
|
+
"x = 1\n"
|
|
144
|
+
"```\n"
|
|
145
|
+
)
|
|
146
|
+
assert _extract(text) == [
|
|
147
|
+
"def handle(question):\n return question\n\nhandle('hi')\n",
|
|
148
|
+
"x = 1\n",
|
|
149
|
+
]
|
|
150
|
+
assert len(_MARKER_LINE.findall(text)) == 2
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def test_fence_closed_at_another_indent_is_not_a_snippet():
|
|
154
|
+
text = " {/* snippet-test */}\n ```python\n x = 1\n```\n"
|
|
155
|
+
assert _extract(text) == []
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""The eval-gate report and the run link it prints."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import httpx
|
|
9
|
+
|
|
10
|
+
from spanlight._experiments import run_experiment
|
|
11
|
+
from spanlight._experiments_report import render_markdown
|
|
12
|
+
|
|
13
|
+
CREDENTIAL = "11111111-2222-3333-4444-555555555555"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _report(totals: dict[str, Any] | None, url: str = "https://x.test/run") -> str:
|
|
17
|
+
return render_markdown(
|
|
18
|
+
title="refunds / answer v1",
|
|
19
|
+
status="done",
|
|
20
|
+
error=None,
|
|
21
|
+
url=url,
|
|
22
|
+
totals=totals,
|
|
23
|
+
fail_below={},
|
|
24
|
+
failures=[],
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_latency_is_rounded_to_whole_milliseconds():
|
|
29
|
+
report = _report(
|
|
30
|
+
{
|
|
31
|
+
"items": 3,
|
|
32
|
+
"errors": 0,
|
|
33
|
+
"cost_usd": "0.0123",
|
|
34
|
+
"latency_p50_ms": 1234.5678901234567,
|
|
35
|
+
"latency_p95_ms": 2999.9999999999995,
|
|
36
|
+
"scores": {},
|
|
37
|
+
}
|
|
38
|
+
)
|
|
39
|
+
assert "p50 1235 ms, p95 3000 ms." in report
|
|
40
|
+
assert "cost 0.0123 USD" in report
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_unknown_latency_stays_the_unknown_marker():
|
|
44
|
+
report = _report(
|
|
45
|
+
{"items": 1, "errors": 1, "cost_usd": None, "latency_p50_ms": None, "scores": {}}
|
|
46
|
+
)
|
|
47
|
+
assert "p50 — ms, p95 — ms." in report
|
|
48
|
+
assert "cost — USD" in report
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_whole_number_latency_has_no_decimals():
|
|
52
|
+
report = _report({"items": 1, "latency_p50_ms": 80, "latency_p95_ms": 80.0, "scores": {}})
|
|
53
|
+
assert "p50 80 ms, p95 80 ms." in report
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_report_links_the_run_page():
|
|
57
|
+
url = "https://x.test/org/proj/experiments/exp/runs/run"
|
|
58
|
+
assert f"[Open the run]({url})" in _report(None, url)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _handler(calls: list[tuple[str, str]]) -> Any:
|
|
62
|
+
def handle(request: httpx.Request) -> httpx.Response:
|
|
63
|
+
path = request.url.path.removeprefix("/api/v1")
|
|
64
|
+
calls.append((request.method, path))
|
|
65
|
+
project = "/projects/proj-1"
|
|
66
|
+
if request.method == "POST" and path == f"{project}/experiments":
|
|
67
|
+
return httpx.Response(
|
|
68
|
+
200, json={"id": "exp-1", "name": json.loads(request.content)["name"]}
|
|
69
|
+
)
|
|
70
|
+
if request.method == "POST" and path == f"{project}/experiments/exp-1/runs":
|
|
71
|
+
return httpx.Response(200, json={"id": "run-1", "status": "queued"})
|
|
72
|
+
routes: dict[str, Any] = {
|
|
73
|
+
project: {"org_id": "org-1"},
|
|
74
|
+
f"{project}/datasets": {"items": [{"id": "ds-1", "name": "refunds"}]},
|
|
75
|
+
f"{project}/prompts/answer": {"latest": {"version": 1}, "labels": []},
|
|
76
|
+
f"{project}/prompts/answer/versions/1": {"id": "pv-1"},
|
|
77
|
+
f"{project}/experiments": {"items": []},
|
|
78
|
+
f"{project}/experiments/exp-1/runs": {"items": []},
|
|
79
|
+
f"{project}/experiments/exp-1/runs/run-1": {"status": "done", "totals": {"scores": {}}},
|
|
80
|
+
}
|
|
81
|
+
if path in routes:
|
|
82
|
+
return httpx.Response(200, json=routes[path])
|
|
83
|
+
return httpx.Response(404, json={"code": "NOT_FOUND"})
|
|
84
|
+
|
|
85
|
+
return handle
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def test_run_experiment_returns_the_run_url():
|
|
89
|
+
calls: list[tuple[str, str]] = []
|
|
90
|
+
outcome = run_experiment(
|
|
91
|
+
host="https://spanlight.example.com/",
|
|
92
|
+
token="tok",
|
|
93
|
+
project_id="proj-1",
|
|
94
|
+
dataset="refunds",
|
|
95
|
+
prompt="answer:1",
|
|
96
|
+
evaluators=[],
|
|
97
|
+
fail_below={},
|
|
98
|
+
credential_id=CREDENTIAL,
|
|
99
|
+
model="claude-haiku-4-5",
|
|
100
|
+
poll_s=1,
|
|
101
|
+
transport=httpx.MockTransport(_handler(calls)),
|
|
102
|
+
environ={},
|
|
103
|
+
)
|
|
104
|
+
assert outcome.run_id == "run-1"
|
|
105
|
+
assert outcome.url == "https://spanlight.example.com/org-1/proj-1/experiments/exp-1/runs/run-1"
|
|
106
|
+
assert outcome.passed
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""A ``Retry-After`` that is not a finite number falls back to the jittered backoff.
|
|
2
|
+
|
|
3
|
+
Failure mode: ``float("nan")`` parses, the retry delay becomes NaN, and ``Event.wait(nan)``
|
|
4
|
+
returns at once, so the exporter retries with no backoff at all.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
|
|
11
|
+
import pytest
|
|
12
|
+
|
|
13
|
+
from spanlight._exporter import BatchExporter, parse_retry_after
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@pytest.mark.parametrize("value", ["nan", "NaN", "inf", "-inf", "infinity"])
|
|
17
|
+
def test_retry_after_nan_is_ignored(value: str) -> None:
|
|
18
|
+
assert parse_retry_after(value) is None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_a_non_finite_retry_after_still_waits_a_finite_backoff() -> None:
|
|
22
|
+
exporter = BatchExporter(url="http://spanlight.test/v1/traces", api_key="k")
|
|
23
|
+
try:
|
|
24
|
+
delay = exporter._delivery.retry_delay(0, parse_retry_after("nan"))
|
|
25
|
+
finally:
|
|
26
|
+
exporter.shutdown()
|
|
27
|
+
|
|
28
|
+
assert math.isfinite(delay) and delay > 0
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_a_numeric_retry_after_is_still_honoured() -> None:
|
|
32
|
+
assert parse_retry_after(" 7 ") == 7.0
|
|
33
|
+
assert parse_retry_after("-3") == 0.0
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Lone surrogates in prompt templates and variables raise only documented errors.
|
|
2
|
+
|
|
3
|
+
Failure mode: the byte-size checks encode strictly as UTF-8, so a value such as ``"caf\\udce9"``
|
|
4
|
+
(what ``os.fsdecode`` gives for undecodable bytes) raises ``UnicodeEncodeError`` out of
|
|
5
|
+
``Prompt.compile`` and ``prompts.get(..., fallback=...)`` instead of a documented error.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import httpx
|
|
13
|
+
import pytest
|
|
14
|
+
|
|
15
|
+
from spanlight import InvalidTemplateError, PromptUnavailableError
|
|
16
|
+
from spanlight._prompt_client import PromptClient
|
|
17
|
+
from spanlight._prompts import fallback_prompt
|
|
18
|
+
|
|
19
|
+
SURROGATE_FALLBACKS: list[Any] = [
|
|
20
|
+
"Hi \ud800 {{name}}",
|
|
21
|
+
[{"role": "user", "content": "\udc00"}],
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_lone_surrogate_variable_does_not_raise_unicode_error() -> None:
|
|
26
|
+
prompt = fallback_prompt("greet", "Hi {{name}}")
|
|
27
|
+
|
|
28
|
+
compiled = prompt.compile(name="caf\udce9")
|
|
29
|
+
|
|
30
|
+
assert compiled.text == "Hi caf\udce9"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@pytest.mark.parametrize("fallback", SURROGATE_FALLBACKS)
|
|
34
|
+
def test_fallback_with_lone_surrogate_is_a_documented_error(fallback: Any) -> None:
|
|
35
|
+
with pytest.raises(InvalidTemplateError):
|
|
36
|
+
fallback_prompt("greet", fallback).compile(name="x")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@pytest.mark.parametrize("fallback", SURROGATE_FALLBACKS)
|
|
40
|
+
def test_get_with_a_lone_surrogate_fallback_is_prompt_unavailable(fallback: Any) -> None:
|
|
41
|
+
def unreachable(request: httpx.Request) -> httpx.Response:
|
|
42
|
+
raise httpx.ConnectError("down", request=request)
|
|
43
|
+
|
|
44
|
+
client = PromptClient(
|
|
45
|
+
base_url="http://spanlight.test/v1/prompts",
|
|
46
|
+
api_key="spl_live_test",
|
|
47
|
+
timeout=1.0,
|
|
48
|
+
cache_ttl=60.0,
|
|
49
|
+
transport=httpx.MockTransport(unreachable),
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
with pytest.raises(PromptUnavailableError, match="fallback is invalid"):
|
|
53
|
+
client.get("greet", fallback=fallback)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|