pytest-rowsmith 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_rowsmith/__init__.py +420 -0
- pytest_rowsmith/plugin.py +36 -0
- pytest_rowsmith-0.1.0.dist-info/METADATA +175 -0
- pytest_rowsmith-0.1.0.dist-info/RECORD +8 -0
- pytest_rowsmith-0.1.0.dist-info/WHEEL +5 -0
- pytest_rowsmith-0.1.0.dist-info/entry_points.txt +2 -0
- pytest_rowsmith-0.1.0.dist-info/licenses/LICENSE +21 -0
- pytest_rowsmith-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
"""pytest-rowsmith: reproducible fake test data from the Rowsmith API, cached for offline test runs.
|
|
2
|
+
|
|
3
|
+
The client sends schemas only (column definitions, or a JSON Schema without its example and default values),
|
|
4
|
+
never data. It uses the standard library alone (urllib),
|
|
5
|
+
and reaches the network only when a generate call misses the cache or the cache is bypassed with
|
|
6
|
+
``--rowsmith-refresh``. Importing this package or loading the plugin opens no connection.
|
|
7
|
+
"""
|
|
8
|
+
import base64
|
|
9
|
+
import csv
|
|
10
|
+
import hashlib
|
|
11
|
+
import io
|
|
12
|
+
import json
|
|
13
|
+
import socket
|
|
14
|
+
import time
|
|
15
|
+
import urllib.error
|
|
16
|
+
import urllib.request
|
|
17
|
+
from http.client import HTTPException
|
|
18
|
+
|
|
19
|
+
# The one place the version is set. pyproject.toml reads it from here, and the User-Agent carries it.
|
|
20
|
+
__version__ = "0.1.0"
|
|
21
|
+
|
|
22
|
+
DEFAULT_URL = "https://testdata-api.re-jam620.workers.dev"
|
|
23
|
+
USER_AGENT = "pytest-rowsmith/" + __version__
|
|
24
|
+
TOOL_HEADER = "pytest"
|
|
25
|
+
TIMEOUT_S = 30.0
|
|
26
|
+
FORMATS = ("csv", "json", "sql")
|
|
27
|
+
# Response headers kept in the cache beside the body bytes.
|
|
28
|
+
KEPT_HEADERS = ("content-type", "x-generator-version", "x-seed", "x-reproduce", "x-schema-version", "x-schema-unmapped",
|
|
29
|
+
"x-request-sha256", "x-output-sha256")
|
|
30
|
+
# Schema keys whose values are examples or defaults: removed before a schema is sent (the service ignores them too).
|
|
31
|
+
# Keys directly inside these maps are property or schema names, and are kept.
|
|
32
|
+
EXAMPLE_KEYS = frozenset(("example", "examples", "default", "const"))
|
|
33
|
+
NAME_MAPS = frozenset(("properties", "patternProperties", "$defs", "definitions", "schemas", "dependentSchemas"))
|
|
34
|
+
CACHE_PREFIX = "rowsmith/v1/"
|
|
35
|
+
|
|
36
|
+
MANIFEST_VERSION = 1
|
|
37
|
+
|
|
38
|
+
__all__ = ["__version__", "Rowsmith", "RowsmithError", "RowsmithResult", "request_body", "assert_golden", "manifest_entry",
|
|
39
|
+
"DEFAULT_URL", "USER_AGENT", "MANIFEST_VERSION"]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class RowsmithError(Exception):
|
|
43
|
+
"""A generate call that did not return data: an HTTP error from the service or a network failure.
|
|
44
|
+
|
|
45
|
+
Attributes: ``status`` (int, or None when no HTTP answer arrived), ``error`` and ``next`` (the service's
|
|
46
|
+
text, when it sent them) and ``reproduce`` (the X-Reproduce line when the service sent one, otherwise the
|
|
47
|
+
request that was sent, in the same form).
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
def __init__(self, status, error, next_step, reproduce, url):
|
|
51
|
+
self.status = status
|
|
52
|
+
self.error = error
|
|
53
|
+
self.next = next_step
|
|
54
|
+
self.reproduce = reproduce
|
|
55
|
+
self.url = url
|
|
56
|
+
if status is None:
|
|
57
|
+
head = "Rowsmith API call failed with no HTTP answer: %s" % (error or "network error")
|
|
58
|
+
else:
|
|
59
|
+
head = "Rowsmith API answered HTTP %d: %s" % (status, error or "(no error text)")
|
|
60
|
+
parts = [head]
|
|
61
|
+
if next_step:
|
|
62
|
+
parts.append("Next: %s" % next_step)
|
|
63
|
+
parts.append("Reproduce: %s" % reproduce)
|
|
64
|
+
parts.append("URL: %s" % url)
|
|
65
|
+
super().__init__("\n".join(parts))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class RowsmithResult:
|
|
69
|
+
"""The bytes and kept headers of one generate answer, from the network or from the cache."""
|
|
70
|
+
|
|
71
|
+
def __init__(self, body, headers, request_body, from_cache, fmt):
|
|
72
|
+
self.body = body # bytes, exactly as the service sent them
|
|
73
|
+
self.headers = dict(headers) # lower-case names; only KEPT_HEADERS
|
|
74
|
+
self.request_body = request_body # the exact bytes sent (and hashed for the cache key)
|
|
75
|
+
self.from_cache = from_cache
|
|
76
|
+
self.format = fmt
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def text(self):
|
|
80
|
+
return self.body.decode("utf-8")
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def sha256(self):
|
|
84
|
+
return hashlib.sha256(self.body).hexdigest()
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def output_sha256(self):
|
|
88
|
+
"""The service's X-Output-SHA256 (sha256 of the body bytes); without that header, the same hash computed
|
|
89
|
+
here from the body bytes."""
|
|
90
|
+
return self.headers.get("x-output-sha256") or self.sha256
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def request_sha256(self):
|
|
94
|
+
"""The service's X-Request-SHA256: the sha256 of the canonical request that X-Reproduce encodes (defaults
|
|
95
|
+
filled in, version pinned; for a schema request, the mapped columns). Without that header, the sha256 of the
|
|
96
|
+
body in a body-b64url X-Reproduce line (the exact bytes); otherwise None (run once with --rowsmith-refresh)."""
|
|
97
|
+
value = self.headers.get("x-request-sha256")
|
|
98
|
+
if value:
|
|
99
|
+
return value
|
|
100
|
+
line = self.headers.get("x-reproduce") or ""
|
|
101
|
+
if line.startswith("POST /v1/generate body-b64url="):
|
|
102
|
+
b64 = line.split("=", 1)[1]
|
|
103
|
+
return hashlib.sha256(base64.urlsafe_b64decode(b64 + "=" * (-len(b64) % 4))).hexdigest()
|
|
104
|
+
return None
|
|
105
|
+
|
|
106
|
+
@property
|
|
107
|
+
def schema_version(self):
|
|
108
|
+
"""X-Schema-Version for a schema request; None for a columns request."""
|
|
109
|
+
return self.headers.get("x-schema-version")
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def generator_version(self):
|
|
113
|
+
return self.headers.get("x-generator-version")
|
|
114
|
+
|
|
115
|
+
@property
|
|
116
|
+
def seed(self):
|
|
117
|
+
value = self.headers.get("x-seed")
|
|
118
|
+
return int(value) if value is not None else None
|
|
119
|
+
|
|
120
|
+
@property
|
|
121
|
+
def schema_unmapped(self):
|
|
122
|
+
"""For a schema request: the paths of the properties the service could not map (X-Schema-Unmapped),
|
|
123
|
+
e.g. ["tags", "address.zip"]; an empty list when every property mapped or for a columns request."""
|
|
124
|
+
value = self.headers.get("x-schema-unmapped")
|
|
125
|
+
return value.split(",") if value else []
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def reproduce(self):
|
|
129
|
+
"""The service's X-Reproduce line (CSV and SQL answers carry one; JSON answers do not, so for JSON
|
|
130
|
+
this is the request that was sent, in the same form)."""
|
|
131
|
+
return self.headers.get("x-reproduce") or _request_line(self.request_body)
|
|
132
|
+
|
|
133
|
+
def parsed(self):
|
|
134
|
+
"""List of dicts for csv and json; the text itself for sql. CSV values are strings."""
|
|
135
|
+
if self.format == "json":
|
|
136
|
+
return json.loads(self.text)
|
|
137
|
+
if self.format == "csv":
|
|
138
|
+
return list(csv.DictReader(io.StringIO(self.text, newline="")))
|
|
139
|
+
return self.text
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _self_contained(result):
|
|
143
|
+
"""A reproduce line that regenerates the bytes by itself: X-Reproduce when it carries the body, otherwise the
|
|
144
|
+
request this client sent (version pinned too, so it regenerates the same bytes or is refused with 409)."""
|
|
145
|
+
line = result.headers.get("x-reproduce") or ""
|
|
146
|
+
return line if line.startswith("POST /v1/generate body-b64url=") else _request_line(result.request_body)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def manifest_entry(result, name):
|
|
150
|
+
"""One golden-manifest entry for a RowsmithResult: a dict with name, request_sha256, output_sha256,
|
|
151
|
+
output_version, schema_version (None for a columns request) and reproduce. Collect entries as
|
|
152
|
+
{"rowsmith_manifest": 1, "entries": [...]} and keep the file beside your eval."""
|
|
153
|
+
if not isinstance(name, str) or not name:
|
|
154
|
+
raise TypeError("name must be a non-empty string, e.g. \"orders-v1\"")
|
|
155
|
+
request_sha256 = result.request_sha256
|
|
156
|
+
if request_sha256 is None:
|
|
157
|
+
raise ValueError("this answer has no request sha256 (no X-Request-SHA256 header and no body-b64url X-Reproduce line); run once with --rowsmith-refresh")
|
|
158
|
+
return {
|
|
159
|
+
"name": name,
|
|
160
|
+
"request_sha256": request_sha256,
|
|
161
|
+
"output_sha256": result.sha256,
|
|
162
|
+
"output_version": result.generator_version,
|
|
163
|
+
"schema_version": result.schema_version,
|
|
164
|
+
"reproduce": _self_contained(result),
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def assert_golden(result, expected):
|
|
169
|
+
"""Fails (AssertionError) unless the sha256 of result's body bytes is ``expected``: a hex string, or a manifest
|
|
170
|
+
entry dict (whose request_sha256 must then match too). The message names both hashes and the reproduce line."""
|
|
171
|
+
__tracebackhide__ = True
|
|
172
|
+
entry = expected if isinstance(expected, dict) else {"output_sha256": expected}
|
|
173
|
+
want = entry.get("output_sha256")
|
|
174
|
+
if not isinstance(want, str) or len(want) != 64:
|
|
175
|
+
raise TypeError("expected must be a 64-character sha256 hex string or a manifest entry")
|
|
176
|
+
label = " (%s)" % entry["name"] if entry.get("name") else ""
|
|
177
|
+
lines = []
|
|
178
|
+
if entry.get("request_sha256") and entry["request_sha256"] != result.request_sha256:
|
|
179
|
+
lines.append("Rowsmith request differs from the golden entry%s: expected request sha256 %s, got %s."
|
|
180
|
+
% (label, entry["request_sha256"], result.request_sha256))
|
|
181
|
+
if result.sha256 != want.lower():
|
|
182
|
+
lines.append("Rowsmith output differs from the golden entry%s: expected output sha256 %s, got %s "
|
|
183
|
+
"(output version %s, schema version %s, %s)."
|
|
184
|
+
% (label, want, result.sha256, result.generator_version, result.schema_version or "none",
|
|
185
|
+
"from the cache" if result.from_cache else "from the service"))
|
|
186
|
+
if lines:
|
|
187
|
+
lines.append("Reproduce: %s" % _self_contained(result))
|
|
188
|
+
raise AssertionError("\n".join(lines))
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def strip_examples(value, names=False):
|
|
192
|
+
"""A copy of a JSON Schema without example, examples, default, const and x-* keys (their values never leave
|
|
193
|
+
this machine). Keys directly inside properties, $defs, definitions and components.schemas are names, and kept."""
|
|
194
|
+
if isinstance(value, list):
|
|
195
|
+
return [strip_examples(v) for v in value]
|
|
196
|
+
if not isinstance(value, dict):
|
|
197
|
+
return value
|
|
198
|
+
out = {}
|
|
199
|
+
for k, v in value.items():
|
|
200
|
+
if not names and (k in EXAMPLE_KEYS or (isinstance(k, str) and k.startswith("x-"))):
|
|
201
|
+
continue
|
|
202
|
+
out[k] = strip_examples(v, not names and k in NAME_MAPS)
|
|
203
|
+
return out
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _ref_target(ref):
|
|
207
|
+
"""Mirrors Doc.lookup in the service (src/schemain.ts): (container, name) for a local $ref, else None.
|
|
208
|
+
#/$defs/X and #/definitions/X are read from the document root, #/components/schemas/X from
|
|
209
|
+
components.schemas; parts are unescaped (~1 is /, ~0 is ~) after splitting on /; refs over 512 UTF-16 units
|
|
210
|
+
are not followed."""
|
|
211
|
+
if not isinstance(ref, str) or not ref.startswith("#/") or len(ref.encode("utf-16-le")) // 2 > 512:
|
|
212
|
+
return None
|
|
213
|
+
parts = [p.replace("~1", "/").replace("~0", "~") for p in ref[2:].split("/")]
|
|
214
|
+
if len(parts) == 2 and parts[0] in ("$defs", "definitions"):
|
|
215
|
+
return parts[0], parts[1]
|
|
216
|
+
if len(parts) == 3 and parts[0] == "components" and parts[1] == "schemas":
|
|
217
|
+
return "components", parts[2]
|
|
218
|
+
return None
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def minimal_openapi(schema):
|
|
222
|
+
"""For {"openapi": <document>, "component": <name>}: the same wrapper holding only the named component and the
|
|
223
|
+
schemas it reaches through local $refs (the forms the service follows), transitively and each once, so cycles
|
|
224
|
+
end. Kept schemas keep their names and paths and the document's order; paths, servers, security, info and
|
|
225
|
+
unreached schemas are not sent. Applied after strip_examples, so a $ref inside an example, default, const or
|
|
226
|
+
x-* value is never followed. A missing or non-local ref is not followed (the service reports it unmapped, as
|
|
227
|
+
with the full document). Any other input is returned as is."""
|
|
228
|
+
if not isinstance(schema, dict) or not isinstance(schema.get("component"), str) or not isinstance(schema.get("openapi"), dict):
|
|
229
|
+
return schema
|
|
230
|
+
if any(k not in ("openapi", "component") for k in schema):
|
|
231
|
+
return schema # the service refuses it
|
|
232
|
+
doc = schema["openapi"]
|
|
233
|
+
comps = doc.get("components")
|
|
234
|
+
containers = {
|
|
235
|
+
"components": comps["schemas"] if isinstance(comps, dict) and isinstance(comps.get("schemas"), dict) else None,
|
|
236
|
+
"$defs": doc["$defs"] if isinstance(doc.get("$defs"), dict) else None,
|
|
237
|
+
"definitions": doc["definitions"] if isinstance(doc.get("definitions"), dict) else None,
|
|
238
|
+
}
|
|
239
|
+
reached = {"components": set(), "$defs": set(), "definitions": set()}
|
|
240
|
+
todo = [("components", schema["component"])]
|
|
241
|
+
while todo:
|
|
242
|
+
where, name = todo.pop()
|
|
243
|
+
container = containers[where]
|
|
244
|
+
if container is None or name in reached[where]:
|
|
245
|
+
continue
|
|
246
|
+
target = container.get(name)
|
|
247
|
+
if not isinstance(target, dict):
|
|
248
|
+
continue # missing: left for the service to report
|
|
249
|
+
reached[where].add(name)
|
|
250
|
+
stack = [target]
|
|
251
|
+
while stack:
|
|
252
|
+
v = stack.pop()
|
|
253
|
+
if isinstance(v, list):
|
|
254
|
+
stack.extend(v)
|
|
255
|
+
elif isinstance(v, dict):
|
|
256
|
+
for k, x in v.items():
|
|
257
|
+
t = _ref_target(x) if k == "$ref" else None
|
|
258
|
+
if t:
|
|
259
|
+
todo.append(t)
|
|
260
|
+
else:
|
|
261
|
+
stack.append(x)
|
|
262
|
+
|
|
263
|
+
def keep(where):
|
|
264
|
+
return {n: s for n, s in (containers[where] or {}).items() if n in reached[where]}
|
|
265
|
+
|
|
266
|
+
out = {}
|
|
267
|
+
if "openapi" in doc:
|
|
268
|
+
out["openapi"] = doc["openapi"]
|
|
269
|
+
out["components"] = {"schemas": keep("components")}
|
|
270
|
+
if reached["$defs"]:
|
|
271
|
+
out["$defs"] = keep("$defs")
|
|
272
|
+
if reached["definitions"]:
|
|
273
|
+
out["definitions"] = keep("definitions")
|
|
274
|
+
return {"openapi": out, "component": schema["component"]}
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def request_body(columns=None, rows=10, seed=1, format="json", table=None, version="1", schema=None):
|
|
278
|
+
"""The exact JSON bytes sent to POST /v1/generate. Key order is fixed, so the bytes (and the cache key,
|
|
279
|
+
their sha256) depend only on the arguments. Pass exactly one of ``columns`` (the column grammar) and
|
|
280
|
+
``schema`` (a JSON Schema dict, e.g. a Pydantic ``model_json_schema()`` or Zod export, or
|
|
281
|
+
``{"openapi": <document>, "component": "<name>"}``)."""
|
|
282
|
+
if (columns is None) == (schema is None):
|
|
283
|
+
raise ValueError("pass exactly one of columns and schema")
|
|
284
|
+
if schema is not None and not isinstance(schema, dict):
|
|
285
|
+
raise TypeError("schema must be a dict (a JSON Schema object)")
|
|
286
|
+
if format not in FORMATS:
|
|
287
|
+
raise ValueError("format must be one of %s, not %r" % (", ".join(FORMATS), format))
|
|
288
|
+
if isinstance(seed, bool) or not isinstance(seed, int):
|
|
289
|
+
raise TypeError("seed must be an int")
|
|
290
|
+
if isinstance(rows, bool) or not isinstance(rows, int):
|
|
291
|
+
raise TypeError("rows must be an int")
|
|
292
|
+
if not isinstance(version, str) or not version:
|
|
293
|
+
raise TypeError("version must be a non-empty string, e.g. \"1\"")
|
|
294
|
+
body = {}
|
|
295
|
+
if table is not None:
|
|
296
|
+
body["table"] = table
|
|
297
|
+
body["rows"] = rows
|
|
298
|
+
body["seed"] = seed
|
|
299
|
+
body["format"] = format
|
|
300
|
+
body["version"] = version
|
|
301
|
+
if schema is not None:
|
|
302
|
+
body["schema"] = minimal_openapi(strip_examples(schema))
|
|
303
|
+
else:
|
|
304
|
+
body["columns"] = list(columns)
|
|
305
|
+
return json.dumps(body, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _request_line(body):
|
|
309
|
+
b64 = base64.urlsafe_b64encode(body).decode("ascii").rstrip("=")
|
|
310
|
+
return "POST /v1/generate body-b64url=" + b64
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
class Rowsmith:
|
|
314
|
+
"""Client behind the ``rowsmith`` fixture.
|
|
315
|
+
|
|
316
|
+
``cache`` is any object with ``get(key, default)`` and ``set(key, value)`` taking JSON values, such as
|
|
317
|
+
pytest's ``config.cache``; None keeps answers in memory for the life of the client only.
|
|
318
|
+
"""
|
|
319
|
+
|
|
320
|
+
def __init__(self, base_url=DEFAULT_URL, cache=None, refresh=False, timeout=TIMEOUT_S, retry_delay=1.0):
|
|
321
|
+
self.base_url = base_url.rstrip("/")
|
|
322
|
+
self.cache = cache
|
|
323
|
+
self.refresh = refresh
|
|
324
|
+
self.timeout = timeout
|
|
325
|
+
self.retry_delay = retry_delay
|
|
326
|
+
self.network_calls = 0
|
|
327
|
+
self._memory = {}
|
|
328
|
+
|
|
329
|
+
# The golden-manifest helpers, on the fixture too: rowsmith.assert_golden(...), rowsmith.manifest_entry(...).
|
|
330
|
+
assert_golden = staticmethod(assert_golden)
|
|
331
|
+
manifest_entry = staticmethod(manifest_entry)
|
|
332
|
+
|
|
333
|
+
def generate(self, columns=None, rows=10, seed=1, format="json", table=None, version="1", schema=None):
|
|
334
|
+
"""Rows for csv and json (a list of dicts), the SQL text for sql. Pass columns or schema."""
|
|
335
|
+
return self.fetch(columns, rows=rows, seed=seed, format=format, table=table, version=version, schema=schema).parsed()
|
|
336
|
+
|
|
337
|
+
def fetch(self, columns=None, rows=10, seed=1, format="json", table=None, version="1", schema=None):
|
|
338
|
+
"""The full answer (bytes, headers, reproduce line) as a RowsmithResult. Pass columns or schema."""
|
|
339
|
+
body = request_body(columns, rows=rows, seed=seed, format=format, table=table, version=version, schema=schema)
|
|
340
|
+
key = CACHE_PREFIX + hashlib.sha256(body).hexdigest()
|
|
341
|
+
if key in self._memory:
|
|
342
|
+
return self._memory[key]
|
|
343
|
+
result = None if self.refresh else self._from_cache(key, body, format)
|
|
344
|
+
if result is None:
|
|
345
|
+
status, headers, data = self._post(body)
|
|
346
|
+
kept = {h: headers[h] for h in KEPT_HEADERS if h in headers}
|
|
347
|
+
result = RowsmithResult(data, kept, body, False, format)
|
|
348
|
+
if self.cache is not None:
|
|
349
|
+
self.cache.set(key, {
|
|
350
|
+
"body_b64": base64.b64encode(data).decode("ascii"),
|
|
351
|
+
"body_sha256": hashlib.sha256(data).hexdigest(),
|
|
352
|
+
"headers": kept,
|
|
353
|
+
})
|
|
354
|
+
self._memory[key] = result
|
|
355
|
+
return result
|
|
356
|
+
|
|
357
|
+
def _from_cache(self, key, body, fmt):
|
|
358
|
+
if self.cache is None:
|
|
359
|
+
return None
|
|
360
|
+
entry = self.cache.get(key, None)
|
|
361
|
+
if not isinstance(entry, dict):
|
|
362
|
+
return None
|
|
363
|
+
try:
|
|
364
|
+
data = base64.b64decode(entry["body_b64"], validate=True)
|
|
365
|
+
headers = dict(entry["headers"])
|
|
366
|
+
except (KeyError, TypeError, ValueError):
|
|
367
|
+
return None
|
|
368
|
+
if hashlib.sha256(data).hexdigest() != entry.get("body_sha256"):
|
|
369
|
+
return None
|
|
370
|
+
return RowsmithResult(data, headers, body, True, fmt)
|
|
371
|
+
|
|
372
|
+
def _post(self, body):
|
|
373
|
+
url = self.base_url + "/v1/generate"
|
|
374
|
+
attempts = 2 # no retry on 4xx; exactly one retry on a 5xx or a network error
|
|
375
|
+
for attempt in range(attempts):
|
|
376
|
+
last = attempt == attempts - 1
|
|
377
|
+
req = urllib.request.Request(url, data=body, method="POST", headers={
|
|
378
|
+
"Content-Type": "application/json",
|
|
379
|
+
"Accept": "*/*",
|
|
380
|
+
"User-Agent": USER_AGENT,
|
|
381
|
+
"x-rowsmith-tool": TOOL_HEADER,
|
|
382
|
+
})
|
|
383
|
+
self.network_calls += 1
|
|
384
|
+
try:
|
|
385
|
+
with urllib.request.urlopen(req, timeout=self.timeout) as resp:
|
|
386
|
+
headers = {k.lower(): v for k, v in resp.headers.items()}
|
|
387
|
+
return resp.status, headers, resp.read()
|
|
388
|
+
except urllib.error.HTTPError as e:
|
|
389
|
+
headers = {k.lower(): v for k, v in (e.headers or {}).items()}
|
|
390
|
+
try:
|
|
391
|
+
raw = e.read()
|
|
392
|
+
except (OSError, HTTPException):
|
|
393
|
+
raw = b""
|
|
394
|
+
if e.code >= 500 and not last:
|
|
395
|
+
time.sleep(self.retry_delay)
|
|
396
|
+
continue
|
|
397
|
+
raise self._http_error(e.code, headers, raw, body, url) from None
|
|
398
|
+
except (urllib.error.URLError, socket.timeout, OSError, HTTPException) as e:
|
|
399
|
+
if not last:
|
|
400
|
+
time.sleep(self.retry_delay)
|
|
401
|
+
continue
|
|
402
|
+
reason = getattr(e, "reason", None) or e
|
|
403
|
+
raise RowsmithError(None, "%s: %s" % (type(e).__name__, reason), None, _request_line(body), url) from None
|
|
404
|
+
raise AssertionError("unreachable")
|
|
405
|
+
|
|
406
|
+
@staticmethod
|
|
407
|
+
def _http_error(status, headers, raw, body, url):
|
|
408
|
+
error = nxt = None
|
|
409
|
+
try:
|
|
410
|
+
doc = json.loads(raw.decode("utf-8"))
|
|
411
|
+
if isinstance(doc, dict):
|
|
412
|
+
error = doc.get("error")
|
|
413
|
+
nxt = doc.get("next")
|
|
414
|
+
except (UnicodeDecodeError, ValueError):
|
|
415
|
+
text = raw.decode("utf-8", "replace").strip()
|
|
416
|
+
error = text[:300] if text else None
|
|
417
|
+
if status == 429 and headers.get("retry-after"):
|
|
418
|
+
nxt = (nxt + " " if nxt else "") + "(Retry-After: %s s)" % headers["retry-after"]
|
|
419
|
+
reproduce = headers.get("x-reproduce") or _request_line(body)
|
|
420
|
+
return RowsmithError(status, error, nxt, reproduce, url)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""The pytest plugin: the ``rowsmith`` fixture, the ``--rowsmith-refresh`` option and the ``rowsmith_url`` ini
|
|
2
|
+
option. Loading it defines hooks only; it opens no connection."""
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from . import DEFAULT_URL, Rowsmith
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def pytest_addoption(parser):
|
|
11
|
+
group = parser.getgroup("rowsmith", "Rowsmith fake test data")
|
|
12
|
+
group.addoption(
|
|
13
|
+
"--rowsmith-refresh",
|
|
14
|
+
action="store_true",
|
|
15
|
+
default=False,
|
|
16
|
+
help="Ignore cached Rowsmith answers: call the API once per schema in this run and replace the cache.",
|
|
17
|
+
)
|
|
18
|
+
parser.addini(
|
|
19
|
+
"rowsmith_url",
|
|
20
|
+
help="Base URL of the Rowsmith API (default %s). The ROWSMITH_URL environment variable wins." % DEFAULT_URL,
|
|
21
|
+
default="",
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def base_url(config):
|
|
26
|
+
return os.environ.get("ROWSMITH_URL") or config.getini("rowsmith_url") or DEFAULT_URL
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@pytest.fixture(scope="session")
|
|
30
|
+
def rowsmith(pytestconfig):
|
|
31
|
+
"""A Rowsmith client whose answers are cached in pytest's cache (.pytest_cache), keyed by the request."""
|
|
32
|
+
return Rowsmith(
|
|
33
|
+
base_url=base_url(pytestconfig),
|
|
34
|
+
cache=getattr(pytestconfig, "cache", None),
|
|
35
|
+
refresh=pytestconfig.getoption("rowsmith_refresh"),
|
|
36
|
+
)
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pytest-rowsmith
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: pytest fixture for reproducible fake test data (CSV, JSON, SQL) from the free Rowsmith API, cached for offline runs.
|
|
5
|
+
Author: meiorz
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/meiorz/rowsmith-mcp/tree/main/pytest-rowsmith
|
|
8
|
+
Project-URL: Source, https://github.com/meiorz/rowsmith-mcp
|
|
9
|
+
Project-URL: Issues, https://github.com/meiorz/rowsmith-mcp/issues
|
|
10
|
+
Keywords: pytest,fixtures,test data,fake data,csv,sql
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Framework :: Pytest
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Software Development :: Testing
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Requires-Dist: pytest>=7
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
|
|
29
|
+
# pytest-rowsmith
|
|
30
|
+
|
|
31
|
+
A pytest plugin with one fixture, `rowsmith`, that gives your tests reproducible fake test data (CSV, JSON or
|
|
32
|
+
SQL INSERTs) generated from a table schema by the Rowsmith API.
|
|
33
|
+
|
|
34
|
+
**Network use.** The plugin calls the Rowsmith API over the network (POST
|
|
35
|
+
https://testdata-api.re-jam620.workers.dev/v1/generate) only when a test calls `rowsmith.generate()` or
|
|
36
|
+
`rowsmith.fetch()` and that request is not in the cache yet: the first time a schema (with its rows, seed,
|
|
37
|
+
format, table and version) is used, or every schema once per run with `--rowsmith-refresh`. Importing the package,
|
|
38
|
+
loading the plugin and creating the fixture make no network call. Calls go to the free Rowsmith API under its terms
|
|
39
|
+
of use (https://testdata-api.re-jam620.workers.dev/terms) and count against your daily limit.
|
|
40
|
+
|
|
41
|
+
- Free, rate-limited, no account, no API key.
|
|
42
|
+
- The same schema and seed give byte-identical output for the same generator version. The plugin pins the
|
|
43
|
+
version in every request (`"version": "1"` by default), so cached answers stay valid.
|
|
44
|
+
- Send schemas only. Never send real data.
|
|
45
|
+
- Identifiers come only from reserved or invalid ranges: emails use example.com, example.net, example.org and
|
|
46
|
+
mail.test; phones, card numbers and SSN- and IBAN-shaped values come from reserved or invalid ranges too.
|
|
47
|
+
- No runtime dependency besides pytest; HTTP goes through the standard library's urllib.
|
|
48
|
+
|
|
49
|
+
## Install
|
|
50
|
+
```sh
|
|
51
|
+
pip install pytest-rowsmith
|
|
52
|
+
```
|
|
53
|
+
Or from the public repository at the plugin's release tag pytest-rowsmith-v0.1.0:
|
|
54
|
+
```sh
|
|
55
|
+
pip install "git+https://github.com/meiorz/rowsmith-mcp@pytest-rowsmith-v0.1.0#subdirectory=pytest-rowsmith"
|
|
56
|
+
```
|
|
57
|
+
The install from the repository needs git and network access to PyPI for the build tool (setuptools); nothing is
|
|
58
|
+
compiled. pytest loads the plugin automatically. Python 3.9 or later.
|
|
59
|
+
|
|
60
|
+
## Example
|
|
61
|
+
```python
|
|
62
|
+
ORDERS = [
|
|
63
|
+
{"name": "order_id", "type": "sequence", "prefix": "ord_", "pad": 4},
|
|
64
|
+
{"name": "email", "type": "email"},
|
|
65
|
+
{"name": "total_usd", "type": "decimal", "min": 1, "max": 500},
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
def test_orders(rowsmith):
|
|
69
|
+
orders = rowsmith.generate(ORDERS, rows=20, seed=42)
|
|
70
|
+
assert len(orders) == 20 and orders[0]["order_id"] == "ord_0001"
|
|
71
|
+
assert all(o["email"].rsplit("@", 1)[1] in ("example.com", "example.net", "example.org", "mail.test") for o in orders)
|
|
72
|
+
```
|
|
73
|
+
The column types and their options are listed at https://testdata-api.re-jam620.workers.dev/v1/types; the full
|
|
74
|
+
API is described at https://testdata-api.re-jam620.workers.dev/openapi.json.
|
|
75
|
+
|
|
76
|
+
### From your own schema
|
|
77
|
+
Instead of `columns`, pass `schema=`: a JSON Schema dict (type object with properties), such as a Pydantic
|
|
78
|
+
model's `model_json_schema()` or a Zod export, or an OpenAPI component as
|
|
79
|
+
`{"openapi": <document dict>, "component": "Order"}`. The service maps it to columns; properties it cannot map
|
|
80
|
+
(arrays, patterns and other unsupported forms) are left out, never guessed, and listed in
|
|
81
|
+
`fetch(...).schema_unmapped`. Before sending, the plugin removes `example`, `examples`, `default`, `const` and `x-*`
|
|
82
|
+
keys from the schema, so their values do not leave your machine; the service ignores them too.
|
|
83
|
+
For an OpenAPI component, only the named component and the schemas it references are sent.
|
|
84
|
+
```python
|
|
85
|
+
from myapp.models import User # a Pydantic model
|
|
86
|
+
|
|
87
|
+
def test_users(rowsmith):
|
|
88
|
+
users = rowsmith.generate(schema=User.model_json_schema(), rows=20, seed=42)
|
|
89
|
+
assert len(users) == 20
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## The fixture
|
|
93
|
+
`rowsmith` is session-scoped. Its methods take the same arguments:
|
|
94
|
+
|
|
95
|
+
| Argument | Default | Meaning |
|
|
96
|
+
|---|---|---|
|
|
97
|
+
| `columns` | | list of column definitions, as in the API; pass `columns` or `schema` |
|
|
98
|
+
| `schema` | | a JSON Schema dict, or `{"openapi": ..., "component": ...}`; pass `columns` or `schema` |
|
|
99
|
+
| `rows` | `10` | 1 to 500 |
|
|
100
|
+
| `seed` | `1` | always sent, so every call is reproducible |
|
|
101
|
+
| `format` | `"json"` | `"csv"`, `"json"` or `"sql"` |
|
|
102
|
+
| `table` | not sent | table name (used in SQL output) |
|
|
103
|
+
| `version` | `"1"` | generator version, always sent |
|
|
104
|
+
|
|
105
|
+
- `rowsmith.generate(columns, ...)` (or `generate(schema=..., ...)`) returns a list of dicts for `json` (values typed as in the JSON) and `csv`
|
|
106
|
+
(values are strings), and the SQL text for `sql`.
|
|
107
|
+
- `rowsmith.fetch(columns, ...)` returns a `RowsmithResult` with `body` (the exact bytes), `text`, `sha256`,
|
|
108
|
+
`headers`, `generator_version`, `seed`, `reproduce`, `schema_unmapped`, `schema_version`, `output_sha256`,
|
|
109
|
+
`request_sha256`, `from_cache` and `parsed()`. `reproduce` is the service's
|
|
110
|
+
X-Reproduce line (sent with CSV and SQL answers, and kept in the cache); JSON answers carry none, so for them it
|
|
111
|
+
is the request that was sent, in the same form.
|
|
112
|
+
- On anything other than a successful answer the call raises `pytest_rowsmith.RowsmithError`. Its message names
|
|
113
|
+
the HTTP status, the service's error and `next` text, and the reproduce line. A 4xx answer is not retried; a 5xx
|
|
114
|
+
answer or a network error is retried exactly once, 1 second later. There is no fallback to random or local data.
|
|
115
|
+
- Each request times out after 30 s.
|
|
116
|
+
|
|
117
|
+
`pytest_rowsmith.Rowsmith` is the same client outside pytest (`Rowsmith(base_url=..., cache=None)`).
|
|
118
|
+
|
|
119
|
+
## Golden manifests
|
|
120
|
+
Pin a dataset for an eval and detect drift. The service sends `X-Output-SHA256` (the sha256 of the body bytes) and
|
|
121
|
+
`X-Request-SHA256` (the sha256 of the canonical request, the same for a schema and the columns it maps to) with
|
|
122
|
+
every answer; the result exposes them as `output_sha256` and `request_sha256`, and the cache keeps them.
|
|
123
|
+
`request_sha256` is the X-Request-SHA256 header; without it, the sha256 of the body in a body-b64url X-Reproduce
|
|
124
|
+
line; otherwise None, and a run with `--rowsmith-refresh` fetches the answer again with its headers.
|
|
125
|
+
`output_sha256` without its header is computed from the body bytes.
|
|
126
|
+
```python
|
|
127
|
+
ORDERS = [{"name": "order_id", "type": "sequence", "prefix": "ord_", "pad": 4}, {"name": "email", "type": "email"}]
|
|
128
|
+
|
|
129
|
+
def test_orders_are_golden(rowsmith):
|
|
130
|
+
result = rowsmith.fetch(ORDERS, rows=20, seed=42, format="csv")
|
|
131
|
+
entry = rowsmith.manifest_entry(result, "orders-v1") # save it once, e.g. into tests/golden.json
|
|
132
|
+
rowsmith.assert_golden(result, entry["output_sha256"]) # later: compare against the saved entry or its hash
|
|
133
|
+
```
|
|
134
|
+
- `rowsmith.manifest_entry(result, name)` returns `{"name", "request_sha256", "output_sha256", "output_version",
|
|
135
|
+
"schema_version", "reproduce"}`, one entry of `{"rowsmith_manifest": 1, "entries": [...]}` (the format is in the
|
|
136
|
+
service's llms.txt). `reproduce` regenerates the bytes by itself: the service's X-Reproduce line when it carries
|
|
137
|
+
the body, otherwise the request the plugin sent.
|
|
138
|
+
- `rowsmith.assert_golden(result, expected)` takes a sha256 hex string or a manifest entry (then the request hash
|
|
139
|
+
must match too) and fails with both hashes, the versions, whether the answer came from the cache, and the
|
|
140
|
+
reproduce line. Both are also importable from `pytest_rowsmith`.
|
|
141
|
+
- The plugin pins the version in every request. A pinned version is served as long as the service runs; a retired version answers 409. A version's output is not changed in place: a golden test checks its bytes on every build.
|
|
142
|
+
|
|
143
|
+
## Caching and offline runs
|
|
144
|
+
Answers are stored in pytest's own cache (`.pytest_cache/v/rowsmith/v1/` in your project), one entry per request,
|
|
145
|
+
keyed by the sha256 of the exact request-body bytes. An entry holds the response bytes and the headers
|
|
146
|
+
content-type, x-generator-version, x-seed, x-reproduce, x-request-sha256 and x-output-sha256, plus
|
|
147
|
+
x-schema-version and x-schema-unmapped for a schema request. A suite therefore calls the API once per distinct
|
|
148
|
+
request and then runs offline, as long as `.pytest_cache` is kept. A damaged entry is fetched again.
|
|
149
|
+
|
|
150
|
+
- `pytest --rowsmith-refresh` ignores the cache: each distinct request is sent once in that run and its entry is
|
|
151
|
+
replaced.
|
|
152
|
+
- `pytest --cache-clear` removes the cache, Rowsmith entries included.
|
|
153
|
+
- With pytest's cache disabled (`-p no:cacheprovider`), answers are kept in memory for the run only.
|
|
154
|
+
- The cache key does not include the base URL.
|
|
155
|
+
|
|
156
|
+
## Configuration
|
|
157
|
+
| Setting | Where | Default |
|
|
158
|
+
|---|---|---|
|
|
159
|
+
| API base URL | `ROWSMITH_URL` environment variable, or `rowsmith_url` in your pytest ini file; the variable wins | `https://testdata-api.re-jam620.workers.dev` |
|
|
160
|
+
| Bypass the cache | `--rowsmith-refresh` | off |
|
|
161
|
+
|
|
162
|
+
Every request sends `User-Agent: pytest-rowsmith/<version>` and `x-rowsmith-tool: pytest`. The service counts these calls under tool pytest in its aggregate usage totals (counts only; nothing per caller).
|
|
163
|
+
|
|
164
|
+
## Limits and terms
|
|
165
|
+
Up to 500 rows and 50 columns per call, with daily limits per caller; over a limit the call fails with the
|
|
166
|
+
service's 429 message. Use of the Rowsmith service is under its terms of use:
|
|
167
|
+
https://testdata-api.re-jam620.workers.dev/terms. Data notice: https://testdata-api.re-jam620.workers.dev/privacy.
|
|
168
|
+
|
|
169
|
+
The MIT licence covers this plugin's code only.
|
|
170
|
+
|
|
171
|
+
## Development
|
|
172
|
+
```sh
|
|
173
|
+
PYTHONPATH=src python -m pytest # offline: a local stub server stands in for the API
|
|
174
|
+
ROWSMITH_LIVE=1 PYTHONPATH=src python -m pytest # adds the live golden test (one real API call)
|
|
175
|
+
```
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
pytest_rowsmith/__init__.py,sha256=Qgpyu0ruAMgYhAlmOrXkpQe-ROK9QSnnthjpH7cXGTk,19832
|
|
2
|
+
pytest_rowsmith/plugin.py,sha256=kvKOuOgOHScMWAGgr70m66nkvwyFx5rK3IB77K-08UA,1212
|
|
3
|
+
pytest_rowsmith-0.1.0.dist-info/licenses/LICENSE,sha256=IpjKGfsXI0G31Oyqm-bIWq75BRVeioKRMmIK3zEGzZY,1063
|
|
4
|
+
pytest_rowsmith-0.1.0.dist-info/METADATA,sha256=5bJSSV3d7mEY5Zw-MeBp69d3kpK2SF-JrQDsNZRJSLg,10330
|
|
5
|
+
pytest_rowsmith-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
6
|
+
pytest_rowsmith-0.1.0.dist-info/entry_points.txt,sha256=rclv29oEkAASsj4ddFJnQIMnz_c024GgZaIL4uv8RsQ,45
|
|
7
|
+
pytest_rowsmith-0.1.0.dist-info/top_level.txt,sha256=zozwhHtWM7TANGcvnAMexQ8pMS2ePDguWm-ctJ-ItaQ,16
|
|
8
|
+
pytest_rowsmith-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 meiorz
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
pytest_rowsmith
|