meterline 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meterline/__init__.py +59 -0
- meterline/conformance.py +672 -0
- meterline/convert.py +120 -0
- meterline/http.py +186 -0
- meterline/published.py +49 -0
- meterline/records.py +586 -0
- meterline/run.py +189 -0
- meterline/schema.py +123 -0
- meterline/state.py +81 -0
- meterline/writers.py +101 -0
- meterline-0.2.0.dist-info/METADATA +126 -0
- meterline-0.2.0.dist-info/RECORD +15 -0
- meterline-0.2.0.dist-info/WHEEL +5 -0
- meterline-0.2.0.dist-info/licenses/LICENSE +202 -0
- meterline-0.2.0.dist-info/top_level.txt +1 -0
meterline/conformance.py
ADDED
|
@@ -0,0 +1,672 @@
|
|
|
1
|
+
"""The conformance kit: what every connector's tests call to prove it keeps the connector contract.
|
|
2
|
+
|
|
3
|
+
``docs/reference/connector-contract.md`` lists the rules; each rule here has the same identifier (``R01`` ...) and a
|
|
4
|
+
check that runs the connector's own ``run`` against its public fake API. A connector describes itself once, as a
|
|
5
|
+
``Connector``, and runs the kit in one test file::
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
from meterline.conformance import RULES, Connector, check
|
|
9
|
+
|
|
10
|
+
CONNECTOR = Connector(
|
|
11
|
+
name="meterline-example", run=run, make_fake=FakeAPI,
|
|
12
|
+
credentials=lambda fake: {"EXAMPLE_KEY": fake.api_key},
|
|
13
|
+
backfill_options={"since": date(2026, 9, 1)},
|
|
14
|
+
fail=lambda fake, status, times: fake.fail("/usage", status, times=times),
|
|
15
|
+
bad_options={"rate_per_minute": 0},
|
|
16
|
+
duplicate=lambda fake: fake.days["2026-09-02"].append(dict(fake.days["2026-09-02"][0])),
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
@pytest.mark.parametrize("rule", RULES, ids=lambda rule: rule.id)
|
|
20
|
+
def test_connector_contract(rule, tmp_path):
|
|
21
|
+
check(rule, CONNECTOR, tmp_path)
|
|
22
|
+
|
|
23
|
+
A check that fails raises ``ConformanceError`` (an ``AssertionError``) whose message starts with the rule, its title and
|
|
24
|
+
its anchor in the contract page, then says what was seen. The kit needs no network and no key, and imports no test
|
|
25
|
+
framework: ``RULES`` is a tuple of ``Rule`` objects so any runner can parametrize over it.
|
|
26
|
+
|
|
27
|
+
One more helper is not a rule. ``compare_shape(answer, shape_file, *, data_keys=(), optional=())`` lists the differences
|
|
28
|
+
in keys and nesting between a parsed answer of the fake API and a shape file of ``docs/api-shapes/``, never by value; a
|
|
29
|
+
connector's shape test calls it in place of a comparison of its own::
|
|
30
|
+
|
|
31
|
+
assert compare_shape(body, SHAPES / "usage.json", optional=["note"]) == []
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
import contextlib
|
|
35
|
+
import gzip
|
|
36
|
+
import hashlib
|
|
37
|
+
import io
|
|
38
|
+
import json
|
|
39
|
+
import os
|
|
40
|
+
import stat
|
|
41
|
+
from dataclasses import dataclass, field
|
|
42
|
+
from decimal import Decimal
|
|
43
|
+
from datetime import datetime, timedelta, timezone
|
|
44
|
+
from pathlib import Path
|
|
45
|
+
from typing import Any, Callable, Dict, Iterable, Iterator, List, Mapping, Optional, Tuple, Union
|
|
46
|
+
|
|
47
|
+
from meterline.http import FakeClock
|
|
48
|
+
from meterline.records import KINDS, validate_batch
|
|
49
|
+
from meterline.run import RunResult
|
|
50
|
+
from meterline.writers import read_jsonl
|
|
51
|
+
|
|
52
|
+
CONTRACT_PAGE = "docs/reference/connector-contract.md"
|
|
53
|
+
VOLATILE_FIELDS = ("pulled_at", "data_refreshed_at", "batch_id") # the revision rule ignores these
|
|
54
|
+
SECRET_WORDS = ("KEY", "TOKEN", "SECRET", "PASSWORD") # a credential with one of these in its name is a secret
|
|
55
|
+
MANIFEST_KEYS = ("batch_id", "command", "status", "windows", "published_counts", "pulled_at", "raw_files", "failures",
|
|
56
|
+
"failure_kind")
|
|
57
|
+
START_NOW = datetime(2026, 9, 15, 6, 0, tzinfo=timezone.utc)
|
|
58
|
+
|
|
59
|
+
Credentials = Union[Mapping[str, str], Callable[[Any], Mapping[str, str]]]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class ConformanceError(AssertionError):
|
|
63
|
+
"""A connector broke a rule of the contract; the message names the rule."""
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class Connector:
|
|
68
|
+
"""What the kit needs to know about one connector.
|
|
69
|
+
|
|
70
|
+
``run`` is the connector's ``run(command, *, data_dir, credentials, transport, ...)``. ``make_fake`` builds a fresh
|
|
71
|
+
public fake API with no faults. ``credentials`` maps the key variable names to the values the fake accepts, or is a
|
|
72
|
+
function of the fake returning that mapping. ``backfill_options`` and ``daily_options`` are the options of those
|
|
73
|
+
commands (``since`` and the like). ``fail(fake, status, times)`` scripts the fake to answer ``status`` to the next
|
|
74
|
+
``times`` requests, whatever they ask for. ``bad_options`` are options the connector rejects as input (exit 2).
|
|
75
|
+
``duplicate(fake)`` scripts the fake so that one answer holds a row, a page or a day twice, so that two records of
|
|
76
|
+
the run would share a key (R16); the run must refuse it with exit 2 and publish nothing. Leave it out and R16 fails,
|
|
77
|
+
telling you to set it. ``now`` is the clock of the first run; a later run
|
|
78
|
+
reads ``now + daily_after``.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
name: str
|
|
82
|
+
run: Callable[..., RunResult]
|
|
83
|
+
make_fake: Callable[[], Any]
|
|
84
|
+
credentials: Credentials
|
|
85
|
+
backfill_options: Mapping[str, Any]
|
|
86
|
+
fail: Callable[[Any, int, int], None]
|
|
87
|
+
bad_options: Mapping[str, Any]
|
|
88
|
+
daily_options: Mapping[str, Any] = field(default_factory=dict)
|
|
89
|
+
now: datetime = START_NOW
|
|
90
|
+
daily_after: timedelta = timedelta(days=1)
|
|
91
|
+
duplicate: Optional[Callable[[Any], None]] = None
|
|
92
|
+
|
|
93
|
+
def creds(self, fake: Any) -> Mapping[str, str]:
|
|
94
|
+
return self.credentials(fake) if callable(self.credentials) else self.credentials
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@dataclass(frozen=True)
|
|
98
|
+
class Rule:
|
|
99
|
+
id: str
|
|
100
|
+
title: str
|
|
101
|
+
check: Callable[["Connector", Path], None]
|
|
102
|
+
|
|
103
|
+
@property
|
|
104
|
+
def anchor(self) -> str:
|
|
105
|
+
return "%s#%s" % (CONTRACT_PAGE, self.id.lower())
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _fail(rule_id: str, detail: str) -> None:
|
|
109
|
+
rule = _BY_ID[rule_id]
|
|
110
|
+
raise ConformanceError("contract rule %s (%s): %s [see %s]" % (rule.id, rule.title, detail, rule.anchor))
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _expect(ok: bool, rule_id: str, detail: str) -> None:
|
|
114
|
+
if not ok:
|
|
115
|
+
_fail(rule_id, detail)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# ---- driving a connector -------------------------------------------------------------------------------------------
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _go(c: Connector, data_dir: Path, command: str, *, fake: Any = None, transport: Any = None, clock: Optional[FakeClock] = None,
|
|
122
|
+
now: Optional[datetime] = None, credentials: Optional[Mapping[str, str]] = None, options: Optional[Mapping[str, Any]] = None,
|
|
123
|
+
capture: Optional[io.StringIO] = None) -> RunResult:
|
|
124
|
+
fake = fake if fake is not None else c.make_fake()
|
|
125
|
+
clk = clk_of(clock)
|
|
126
|
+
given = dict(c.backfill_options if command == "backfill" else c.daily_options)
|
|
127
|
+
given.update(options or {})
|
|
128
|
+
if command == "rebuild":
|
|
129
|
+
given = {}
|
|
130
|
+
stderr = capture if capture is not None else io.StringIO()
|
|
131
|
+
with contextlib.redirect_stderr(stderr):
|
|
132
|
+
return c.run(
|
|
133
|
+
command, data_dir=data_dir, credentials=c.creds(fake) if credentials is None else credentials,
|
|
134
|
+
transport=transport if transport is not None else fake.transport, clock=clk, sleep=clk.sleep, wall=clk,
|
|
135
|
+
now=lambda: now or c.now, jitter=lambda: 0, **given)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def clk_of(clock: Optional[FakeClock]) -> FakeClock:
|
|
139
|
+
return clock if clock is not None else FakeClock()
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _backfilled(c: Connector, tmp: Path, name: str = "data") -> Tuple[Path, RunResult, Any]:
|
|
143
|
+
data = tmp / name
|
|
144
|
+
fake = c.make_fake()
|
|
145
|
+
result = _go(c, data, "backfill", fake=fake)
|
|
146
|
+
return data, result, fake
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _files(root: Path) -> List[Path]:
|
|
150
|
+
return sorted(p for p in root.rglob("*") if p.is_file())
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _manifest(path: Path) -> Dict[str, Any]:
|
|
154
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _manifests(data: Path) -> Dict[str, Dict[str, Any]]:
|
|
158
|
+
return {p.name: _manifest(p) for p in sorted((data / "runs").glob("*.json"))}
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _records(data: Path) -> Dict[str, List[Dict[str, Any]]]:
|
|
162
|
+
out: Dict[str, List[Dict[str, Any]]] = {}
|
|
163
|
+
for kind in KINDS:
|
|
164
|
+
rows: List[Dict[str, Any]] = []
|
|
165
|
+
for path in sorted((data / "records" / kind).glob("*.jsonl")):
|
|
166
|
+
rows.extend(json.loads(line) for line in path.read_text(encoding="utf-8").splitlines())
|
|
167
|
+
out[kind] = rows
|
|
168
|
+
return out
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _stable(record: Mapping[str, Any]) -> str:
|
|
172
|
+
return json.dumps({k: v for k, v in record.items() if k not in VOLATILE_FIELDS}, sort_keys=True, default=str)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _digest(paths: List[Path]) -> Dict[str, str]:
|
|
176
|
+
return {str(p): hashlib.sha256(p.read_bytes()).hexdigest() for p in paths}
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _timeout_transport() -> Any:
|
|
180
|
+
import httpx # every connector depends on it; the core package does not, so it is imported where it is used
|
|
181
|
+
|
|
182
|
+
def refuse(request: "httpx.Request") -> "httpx.Response":
|
|
183
|
+
raise httpx.ReadTimeout("synthetic timeout", request=request)
|
|
184
|
+
|
|
185
|
+
return httpx.MockTransport(refuse)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _cut_off(inner: Any, after: int) -> Any:
|
|
189
|
+
"""A transport that passes the first ``after`` requests to ``inner``, then answers 503 to every one."""
|
|
190
|
+
import httpx
|
|
191
|
+
|
|
192
|
+
seen = [0]
|
|
193
|
+
|
|
194
|
+
def handle(request: "httpx.Request") -> "httpx.Response":
|
|
195
|
+
seen[0] += 1
|
|
196
|
+
if seen[0] > after:
|
|
197
|
+
return httpx.Response(503, json={"error": "synthetic outage"})
|
|
198
|
+
return inner.handle_request(request)
|
|
199
|
+
|
|
200
|
+
return httpx.MockTransport(handle)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
# ---- the rules -----------------------------------------------------------------------------------------------------
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _r01_layout(c: Connector, tmp: Path) -> None:
|
|
207
|
+
data, result, _ = _backfilled(c, tmp)
|
|
208
|
+
_expect(result.exit_code == 0, "R01", "a first backfill on the fake API exited %d, not 0" % result.exit_code)
|
|
209
|
+
top = sorted(p.name for p in data.iterdir())
|
|
210
|
+
_expect(top == ["raw", "records", "runs", "state.json"], "R01", "the data directory holds %s, not raw, records, runs and state.json" % top)
|
|
211
|
+
_expect(bool(_files(data / "raw")), "R01", "raw/ holds no file: every response must be kept as received")
|
|
212
|
+
kinds = sorted(p.name for p in (data / "records").iterdir())
|
|
213
|
+
_expect(all(k in KINDS for k in kinds), "R01", "records/ holds %s; only the record kinds %s are allowed" % (kinds, list(KINDS)))
|
|
214
|
+
manifests = _manifests(data)
|
|
215
|
+
_expect(len(manifests) == 1, "R01", "one backfill wrote %d run manifests, not 1" % len(manifests))
|
|
216
|
+
(name, manifest), = manifests.items()
|
|
217
|
+
missing = [k for k in MANIFEST_KEYS if k not in manifest]
|
|
218
|
+
_expect(not missing, "R01", "the run manifest lacks the key(s) %s that RunResult reads" % missing)
|
|
219
|
+
_expect(manifest["status"] == "published" and manifest["failure_kind"] == "none", "R01",
|
|
220
|
+
"a successful run's manifest says status=%r failure_kind=%r" % (manifest["status"], manifest["failure_kind"]))
|
|
221
|
+
_expect(name == manifest["batch_id"] + ".json", "R01", "the manifest file %s is not named after batch_id %s" % (name, manifest["batch_id"]))
|
|
222
|
+
for kind in kinds:
|
|
223
|
+
names = sorted(p.name for p in (data / "records" / kind).glob("*"))
|
|
224
|
+
_expect(names == [manifest["batch_id"] + ".jsonl"], "R01", "records/%s holds %s, not <batch_id>.jsonl" % (kind, names))
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _no_floats(line: str, where: str) -> None:
|
|
228
|
+
def refuse(text: str) -> Any:
|
|
229
|
+
_fail("R02", "%s holds the number %s written with a fraction or exponent: money and quantities are decimal "
|
|
230
|
+
"strings, never floats" % (where, text))
|
|
231
|
+
|
|
232
|
+
json.loads(line, parse_float=refuse)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _r02_records(c: Connector, tmp: Path) -> None:
|
|
236
|
+
data, _, _ = _backfilled(c, tmp)
|
|
237
|
+
seen = 0
|
|
238
|
+
for path in sorted((data / "records").rglob("*.jsonl")):
|
|
239
|
+
for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
|
240
|
+
_no_floats(line, "%s line %d" % (path.name, number))
|
|
241
|
+
seen += 1
|
|
242
|
+
_expect(seen > 0, "R02", "the backfill published no record")
|
|
243
|
+
for path in sorted((data / "records").rglob("*.jsonl")):
|
|
244
|
+
try:
|
|
245
|
+
validate_batch(read_jsonl(path, KINDS[path.parent.name]))
|
|
246
|
+
except Exception as exc: # RecordError, or whatever the reader or the validator raises
|
|
247
|
+
_fail("R02", "%s/%s does not validate: %s" % (path.parent.name, path.name, exc))
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _r03_result(c: Connector, tmp: Path) -> None:
|
|
251
|
+
data, result, _ = _backfilled(c, tmp)
|
|
252
|
+
(name, manifest), = _manifests(data).items()
|
|
253
|
+
_expect(result.batch_id == manifest["batch_id"], "R03",
|
|
254
|
+
"RunResult.batch_id %r is not the manifest's %r" % (result.batch_id, manifest["batch_id"]))
|
|
255
|
+
_expect(result.manifest_path is not None and Path(result.manifest_path).name == name, "R03",
|
|
256
|
+
"RunResult.manifest_path %r does not name the manifest %s" % (result.manifest_path, name))
|
|
257
|
+
_expect(result.records == manifest["published_counts"], "R03",
|
|
258
|
+
"RunResult.records %s differ from the manifest's published_counts %s" % (result.records, manifest["published_counts"]))
|
|
259
|
+
_expect(list(result.windows) == list(manifest["windows"]), "R03", "RunResult.windows differ from the manifest's windows")
|
|
260
|
+
for kind, count in manifest["published_counts"].items():
|
|
261
|
+
lines = sum(1 for _ in (data / "records" / kind / (manifest["batch_id"] + ".jsonl")).read_text(encoding="utf-8").splitlines())
|
|
262
|
+
_expect(lines == count, "R03", "published_counts says %d %s record(s), the file holds %d" % (count, kind, lines))
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _r04_state(c: Connector, tmp: Path) -> None:
|
|
266
|
+
data, _, _ = _backfilled(c, tmp)
|
|
267
|
+
state = json.loads((data / "state.json").read_text(encoding="utf-8"))
|
|
268
|
+
_expect(isinstance(state, dict) and "pending" in state, "R04",
|
|
269
|
+
"state.json must be a JSON object with a `pending` key (null or the run in progress)")
|
|
270
|
+
_expect(state["pending"] is None, "R04", "after a finished run `pending` is %r, not null" % (state["pending"],))
|
|
271
|
+
damaged = (data / "state.json")
|
|
272
|
+
damaged.write_text('{"days_loaded": ["2026-09', encoding="utf-8")
|
|
273
|
+
fake = c.make_fake()
|
|
274
|
+
before = damaged.read_bytes()
|
|
275
|
+
err = io.StringIO()
|
|
276
|
+
result = _go(c, data, "daily", fake=fake, now=c.now + c.daily_after, capture=err)
|
|
277
|
+
_expect(result.exit_code == 2 and result.failure_kind == "input", "R04",
|
|
278
|
+
"a damaged state.json gave exit %d (%s), not 2 (input)" % (result.exit_code, result.failure_kind))
|
|
279
|
+
_expect(not fake.log, "R04", "a damaged state.json must stop the run before any request; %d request(s) were made" % len(fake.log))
|
|
280
|
+
_expect(damaged.read_bytes() == before, "R04", "a damaged state.json must be left as it was for the owner to inspect")
|
|
281
|
+
_expect("state" in err.getvalue().lower(), "R04", "the message does not name the state file")
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _r05_permissions(c: Connector, tmp: Path) -> None:
|
|
285
|
+
old = os.umask(0o022)
|
|
286
|
+
try:
|
|
287
|
+
data, _, _ = _backfilled(c, tmp)
|
|
288
|
+
_expect(os.umask(0o022) == 0o022, "R05", "the process umask was left changed after the run")
|
|
289
|
+
finally:
|
|
290
|
+
os.umask(old)
|
|
291
|
+
bad = []
|
|
292
|
+
for path in [data, *sorted(data.rglob("*"))]:
|
|
293
|
+
mode = stat.S_IMODE(path.stat().st_mode)
|
|
294
|
+
want = 0o700 if path.is_dir() else 0o600
|
|
295
|
+
if mode != want:
|
|
296
|
+
bad.append("%s is %o, not %o" % (path.relative_to(data.parent), mode, want))
|
|
297
|
+
_expect(not bad, "R05", "under a 022 umask: " + "; ".join(bad[:5]))
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _r06_raw_kept(c: Connector, tmp: Path) -> None:
|
|
301
|
+
data, _, _ = _backfilled(c, tmp)
|
|
302
|
+
before = _digest(_files(data / "raw"))
|
|
303
|
+
_go(c, data, "daily", fake=c.make_fake(), now=c.now + c.daily_after)
|
|
304
|
+
after = _digest(_files(data / "raw"))
|
|
305
|
+
changed = sorted(p for p in before if after.get(p) != before[p])
|
|
306
|
+
_expect(not changed, "R06", "raw file(s) were changed or removed by a later run: %s" % [Path(p).name for p in changed[:3]])
|
|
307
|
+
_expect(len(after) > len(before), "R06", "a later run kept no raw file of its own (%d before, %d after)" % (len(before), len(after)))
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _r07_same_second(c: Connector, tmp: Path) -> None:
|
|
311
|
+
data, first, _ = _backfilled(c, tmp)
|
|
312
|
+
fake = c.make_fake()
|
|
313
|
+
before = _digest(_files(data))
|
|
314
|
+
err = io.StringIO()
|
|
315
|
+
again = _go(c, data, "backfill", fake=fake, capture=err) # the same `now`: the same batch id
|
|
316
|
+
_expect(again.exit_code == 2 and again.failure_kind == "input", "R07",
|
|
317
|
+
"a second run in the second of a finished run exited %d (%s), not 2 (input)" % (again.exit_code, again.failure_kind))
|
|
318
|
+
_expect(_digest(_files(data)) == before, "R07", "a refused run in the same second still changed the data directory")
|
|
319
|
+
_expect(not fake.log, "R07", "a refused run in the same second made %d request(s)" % len(fake.log))
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _r08_revision(c: Connector, tmp: Path) -> None:
|
|
323
|
+
data, _, _ = _backfilled(c, tmp)
|
|
324
|
+
first = _records(data)
|
|
325
|
+
for step in (1, 2):
|
|
326
|
+
result = _go(c, data, "daily", fake=c.make_fake(), now=c.now + c.daily_after * step)
|
|
327
|
+
_expect(result.exit_code == 0, "R08", "daily number %d exited %d" % (step, result.exit_code))
|
|
328
|
+
everything = _records(data)
|
|
329
|
+
for kind, rows in everything.items():
|
|
330
|
+
seen: Dict[str, int] = {}
|
|
331
|
+
for record in rows:
|
|
332
|
+
key = _stable(record)
|
|
333
|
+
seen[key] = seen.get(key, 0) + 1
|
|
334
|
+
repeated = [k for k, n in seen.items() if n > 1]
|
|
335
|
+
_expect(not repeated, "R08", "%d %s record(s) were published again with the same content (only pulled_at, "
|
|
336
|
+
"data_refreshed_at or batch_id differ): %s" % (len(repeated), kind, repeated[0][:160] if repeated else ""))
|
|
337
|
+
_expect(sum(map(len, everything.values())) >= sum(map(len, first.values())), "R08", "a later run removed published records")
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _r09_rebuild(c: Connector, tmp: Path) -> None:
|
|
341
|
+
data, _, _ = _backfilled(c, tmp)
|
|
342
|
+
before = _digest(_files(data / "records"))
|
|
343
|
+
import shutil
|
|
344
|
+
|
|
345
|
+
shutil.rmtree(data / "records")
|
|
346
|
+
fake = c.make_fake()
|
|
347
|
+
result = _go(c, data, "rebuild", fake=fake, credentials={}, transport=_timeout_transport())
|
|
348
|
+
_expect(result.exit_code == 0, "R09", "rebuild exited %d; it must need no key and make no request" % result.exit_code)
|
|
349
|
+
_expect(not fake.log, "R09", "rebuild made %d request(s)" % len(fake.log))
|
|
350
|
+
after = _digest(_files(data / "records"))
|
|
351
|
+
_expect({Path(k).relative_to(data) for k in after} == {Path(k).relative_to(data) for k in before} and
|
|
352
|
+
sorted(after.values()) == sorted(before.values()), "R09",
|
|
353
|
+
"the records rebuilt from the raw files are not byte-identical to the published ones")
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _r10_resume(c: Connector, tmp: Path) -> None:
|
|
357
|
+
clean, clean_result, probe = _backfilled(c, tmp, "clean")
|
|
358
|
+
requests = len(probe.log)
|
|
359
|
+
_expect(requests >= 2, "R10", "the fake API saw only %d request(s) in a backfill; it must log every request" % requests)
|
|
360
|
+
data = tmp / "interrupted"
|
|
361
|
+
fake = c.make_fake()
|
|
362
|
+
cut = _cut_off(fake.transport, max(1, requests // 2))
|
|
363
|
+
broken = _go(c, data, "backfill", fake=fake, transport=cut)
|
|
364
|
+
_expect(broken.exit_code == 1 and broken.failure_kind == "transient", "R10",
|
|
365
|
+
"an outage half way gave exit %d (%s), not 1 (transient)" % (broken.exit_code, broken.failure_kind))
|
|
366
|
+
state = json.loads((data / "state.json").read_text(encoding="utf-8"))
|
|
367
|
+
_expect(isinstance(state, dict) and "pending" in state, "R10", "after a failed run state.json is not a valid state object")
|
|
368
|
+
_expect(not any((data / "records").rglob("*.jsonl")) if (data / "records").exists() else True, "R10",
|
|
369
|
+
"a run that failed before normalising published records")
|
|
370
|
+
claimed = sorted(k for k, v in state.items() if k.endswith("loaded") and v)
|
|
371
|
+
_expect(not claimed, "R10", "state.json lists loaded days under %s although the run published nothing: state is written "
|
|
372
|
+
"after the records, never before" % claimed)
|
|
373
|
+
resumed = _go(c, data, "backfill", fake=c.make_fake(), now=c.now + timedelta(minutes=1))
|
|
374
|
+
_expect(resumed.exit_code == 0, "R10", "running the same command again after the outage exited %d, not 0" % resumed.exit_code)
|
|
375
|
+
want = sorted(_stable(r) for rows in _records(clean).values() for r in rows)
|
|
376
|
+
got = sorted(_stable(r) for rows in _records(data).values() for r in rows)
|
|
377
|
+
_expect(got == want, "R10", "the records after fail-and-resume differ from an uninterrupted run (%d vs %d record(s))" % (len(got), len(want)))
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _r11_exit_codes(c: Connector, tmp: Path) -> None:
|
|
381
|
+
n = 0
|
|
382
|
+
|
|
383
|
+
def one(label: str, **go: Any) -> Tuple[RunResult, str]:
|
|
384
|
+
nonlocal n
|
|
385
|
+
n += 1
|
|
386
|
+
err = io.StringIO()
|
|
387
|
+
result = _go(c, tmp / ("exit-%d" % n), "backfill", capture=err, **go)
|
|
388
|
+
return result, err.getvalue()
|
|
389
|
+
|
|
390
|
+
for status in (400, 401, 403, 404, 422):
|
|
391
|
+
fake = c.make_fake()
|
|
392
|
+
c.fail(fake, status, 50)
|
|
393
|
+
result, text = one("status", fake=fake)
|
|
394
|
+
_expect((result.exit_code, result.failure_kind, result.http_status) == (3, "permanent", status), "R11",
|
|
395
|
+
"an API answer of %d gave exit %d (%s, http_status %s), not 3 (permanent)"
|
|
396
|
+
% (status, result.exit_code, result.failure_kind, result.http_status))
|
|
397
|
+
_expect("permanent:" in text, "R11", "the message for a %d does not start with `permanent:`" % status)
|
|
398
|
+
_expect(len(fake.log) == 1, "R11", "a %d was retried (%d requests); a refusal is never retried" % (status, len(fake.log)))
|
|
399
|
+
for status in (500, 503):
|
|
400
|
+
fake = c.make_fake()
|
|
401
|
+
c.fail(fake, status, 50)
|
|
402
|
+
result, text = one("status", fake=fake)
|
|
403
|
+
_expect((result.exit_code, result.failure_kind, result.http_status) == (1, "transient", status), "R11",
|
|
404
|
+
"an API answer of %d gave exit %d (%s, http_status %s), not 1 (transient)"
|
|
405
|
+
% (status, result.exit_code, result.failure_kind, result.http_status))
|
|
406
|
+
_expect("transient:" in text, "R11", "the message for a %d does not start with `transient:`" % status)
|
|
407
|
+
result, _ = one("timeout", transport=_timeout_transport())
|
|
408
|
+
_expect((result.exit_code, result.failure_kind) == (1, "transient"), "R11",
|
|
409
|
+
"a timeout gave exit %d (%s), not 1 (transient)" % (result.exit_code, result.failure_kind))
|
|
410
|
+
result, _ = one("bad options", options=c.bad_options)
|
|
411
|
+
_expect((result.exit_code, result.failure_kind) == (2, "input"), "R11",
|
|
412
|
+
"bad options gave exit %d (%s), not 2 (input)" % (result.exit_code, result.failure_kind))
|
|
413
|
+
result, _ = one("no key", credentials={})
|
|
414
|
+
_expect((result.exit_code, result.failure_kind) == (2, "input"), "R11",
|
|
415
|
+
"no key gave exit %d (%s), not 2 (input)" % (result.exit_code, result.failure_kind))
|
|
416
|
+
result, _ = one("ok")
|
|
417
|
+
_expect((result.exit_code, result.failure_kind, result.http_status) == (0, "none", None), "R11",
|
|
418
|
+
"a good run gave exit %d (%s, http_status %s)" % (result.exit_code, result.failure_kind, result.http_status))
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _r12_clock(c: Connector, tmp: Path) -> None:
|
|
422
|
+
fake = c.make_fake()
|
|
423
|
+
c.fail(fake, 429, 1)
|
|
424
|
+
clock = FakeClock()
|
|
425
|
+
started = clock.now
|
|
426
|
+
result = _go(c, tmp / "limited", "backfill", fake=fake, clock=clock)
|
|
427
|
+
_expect(result.exit_code == 0, "R12", "one 429 followed by good answers exited %d; it must be retried" % result.exit_code)
|
|
428
|
+
_expect(bool(clock.sleeps), "R12", "the retry after a 429 did not wait through the injected `sleep`")
|
|
429
|
+
_expect(all(isinstance(s, (int, Decimal)) and s >= 0 for s in clock.sleeps), "R12",
|
|
430
|
+
"the injected sleep got a value that is not a non-negative number: %s" % clock.sleeps[:3])
|
|
431
|
+
_expect(clock.now - started == sum(clock.sleeps), "R12", "time passed that the injected sleep did not account for")
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def _r13_secrets(c: Connector, tmp: Path) -> None:
|
|
435
|
+
fake = c.make_fake()
|
|
436
|
+
secrets = [v for k, v in c.creds(fake).items() if v and any(w in k.upper() for w in SECRET_WORDS)]
|
|
437
|
+
_expect(bool(secrets), "R13", "no credential name contains %s, so the kit has no key to look for" % "/".join(SECRET_WORDS))
|
|
438
|
+
err = io.StringIO()
|
|
439
|
+
data = tmp / "data"
|
|
440
|
+
_go(c, data, "backfill", fake=fake, capture=err)
|
|
441
|
+
blobs = [err.getvalue().encode("utf-8")]
|
|
442
|
+
for path in _files(data):
|
|
443
|
+
raw = path.read_bytes()
|
|
444
|
+
blobs.append(raw)
|
|
445
|
+
if path.suffix == ".gz":
|
|
446
|
+
blobs.append(gzip.decompress(raw))
|
|
447
|
+
for secret in secrets:
|
|
448
|
+
_expect(not any(secret.encode("utf-8") in b for b in blobs), "R13", "the key appears in a file or message of the run")
|
|
449
|
+
fake = c.make_fake()
|
|
450
|
+
c.fail(fake, 401, 50)
|
|
451
|
+
err = io.StringIO()
|
|
452
|
+
_go(c, tmp / "refused", "backfill", fake=fake, capture=err)
|
|
453
|
+
_expect(all(s not in err.getvalue() for s in secrets), "R13", "the key appears in the message of a refused run")
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _r14_fake(c: Connector, tmp: Path) -> None:
|
|
457
|
+
fake = c.make_fake()
|
|
458
|
+
_expect(hasattr(fake, "transport"), "R14", "the fake API has no `transport` to hand to run()")
|
|
459
|
+
_expect(isinstance(getattr(fake, "log", None), list), "R14", "the fake API keeps no request `log` list")
|
|
460
|
+
_go(c, tmp / "data", "backfill", fake=fake)
|
|
461
|
+
_expect(len(fake.log) > 0, "R14", "the fake API logged no request of a backfill")
|
|
462
|
+
for entry in fake.log[:1]:
|
|
463
|
+
absent = [name for name in ("time", "status", "params") if not hasattr(entry, name)]
|
|
464
|
+
_expect(not absent and (hasattr(entry, "path") or hasattr(entry, "endpoint")), "R14",
|
|
465
|
+
"a request log entry lacks %s (it needs time, status, params and path or endpoint)" % (absent or ["path or endpoint"]))
|
|
466
|
+
failing = c.make_fake()
|
|
467
|
+
c.fail(failing, 503, 1)
|
|
468
|
+
_go(c, tmp / "faulted", "backfill", fake=failing)
|
|
469
|
+
_expect(any(entry.status == 503 for entry in failing.log), "R14",
|
|
470
|
+
"a fault scripted with `fail` was never answered: no logged request has status 503")
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
@dataclass
|
|
474
|
+
class _FloatWatch:
|
|
475
|
+
"""Finds a Python float in what ``json.loads`` or ``json.load`` returns while it is installed."""
|
|
476
|
+
|
|
477
|
+
phase: str = ""
|
|
478
|
+
found: List[str] = field(default_factory=list) # the phases in which a float was read
|
|
479
|
+
|
|
480
|
+
def holds_float(self, value: Any) -> bool:
|
|
481
|
+
if type(value).__name__ == "float": # by name: the source scans of the connectors' tests refuse the word float as code
|
|
482
|
+
return True
|
|
483
|
+
if isinstance(value, dict):
|
|
484
|
+
return any(self.holds_float(v) for v in value.values())
|
|
485
|
+
if isinstance(value, (list, tuple)):
|
|
486
|
+
return any(self.holds_float(v) for v in value)
|
|
487
|
+
return False
|
|
488
|
+
|
|
489
|
+
def wrap(self, original: Callable[..., Any]) -> Callable[..., Any]:
|
|
490
|
+
def watched(*args: Any, **kwargs: Any) -> Any:
|
|
491
|
+
result = original(*args, **kwargs)
|
|
492
|
+
if self.holds_float(result) and self.phase not in self.found:
|
|
493
|
+
self.found.append(self.phase)
|
|
494
|
+
return result
|
|
495
|
+
|
|
496
|
+
return watched
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
@contextlib.contextmanager
|
|
500
|
+
def _floats_watched() -> Iterator[_FloatWatch]:
|
|
501
|
+
"""For the length of the block ``json.loads`` and ``json.load`` call the original and walk the result."""
|
|
502
|
+
watch = _FloatWatch()
|
|
503
|
+
original = (json.loads, json.load)
|
|
504
|
+
json.loads, json.load = watch.wrap(original[0]), watch.wrap(original[1])
|
|
505
|
+
try:
|
|
506
|
+
yield watch
|
|
507
|
+
finally:
|
|
508
|
+
json.loads, json.load = original
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def _r15_rebuild_keeps_decimal(c: Connector, tmp: Path) -> None:
|
|
512
|
+
import shutil
|
|
513
|
+
|
|
514
|
+
data = tmp / "data"
|
|
515
|
+
|
|
516
|
+
def no_float(watch: _FloatWatch, phase: str) -> None:
|
|
517
|
+
_expect(phase not in watch.found, "R15", "a JSON number with a fraction was read as a float (in the %s): read raw and "
|
|
518
|
+
"API JSON with json.loads(..., parse_float=Decimal)" % phase)
|
|
519
|
+
|
|
520
|
+
with _floats_watched() as watch:
|
|
521
|
+
watch.phase = "backfill"
|
|
522
|
+
result = _go(c, data, "backfill", fake=c.make_fake())
|
|
523
|
+
no_float(watch, "backfill") # before the exit code: a float makes the figures too long and the run fail
|
|
524
|
+
_expect(result.exit_code == 0, "R15", "a first backfill on the fake API exited %d, not 0" % result.exit_code)
|
|
525
|
+
live = {p.relative_to(data): p.read_bytes() for p in _files(data / "records")}
|
|
526
|
+
shutil.rmtree(data / "records")
|
|
527
|
+
watch.phase = "rebuild"
|
|
528
|
+
rebuilt = _go(c, data, "rebuild", fake=c.make_fake(), credentials={}, transport=_timeout_transport())
|
|
529
|
+
no_float(watch, "rebuild")
|
|
530
|
+
_expect(rebuilt.exit_code == 0, "R15", "rebuild exited %d; it must need no key and make no request" % rebuilt.exit_code)
|
|
531
|
+
after = {p.relative_to(data): p.read_bytes() for p in _files(data / "records")}
|
|
532
|
+
_expect(bool(live), "R15", "the backfill published no record")
|
|
533
|
+
_expect(sorted(after) == sorted(live), "R15", "rebuild wrote the record files %s, the live run wrote %s"
|
|
534
|
+
% (sorted(str(p) for p in after), sorted(str(p) for p in live)))
|
|
535
|
+
different = sorted(str(p) for p in live if after[p] != live[p])
|
|
536
|
+
_expect(not different, "R15", "the records rebuilt from the raw files differ in their bytes from the live run's: %s" % different[:3])
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _r16_batch_checks(c: Connector, tmp: Path) -> None:
|
|
540
|
+
_expect(c.duplicate is not None, "R16", "the connector must set Connector.duplicate, a function that scripts the fake so "
|
|
541
|
+
"that one answer holds a row twice (two records of the run then share a key)")
|
|
542
|
+
fake = c.make_fake()
|
|
543
|
+
c.duplicate(fake) # type: ignore[misc]
|
|
544
|
+
data = tmp / "data"
|
|
545
|
+
err = io.StringIO()
|
|
546
|
+
result = _go(c, data, "backfill", fake=fake, capture=err)
|
|
547
|
+
_expect((result.exit_code, result.failure_kind) == (2, "input"), "R16",
|
|
548
|
+
"a run whose answer holds a row twice exited %d (%s), not 2 (input): the batch checks must run before anything "
|
|
549
|
+
"is published (stderr: %s)" % (result.exit_code, result.failure_kind, err.getvalue().strip()[:200] or "empty"))
|
|
550
|
+
published = sorted(data.glob("records/**/*.jsonl"))
|
|
551
|
+
_expect(not any(p.stat().st_size for p in published), "R16",
|
|
552
|
+
"a run that failed its batch checks still published %s" % [p.name for p in published[:3]])
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
RULES: Tuple[Rule, ...] = (
|
|
556
|
+
Rule("R01", "data directory layout and run manifest keys", _r01_layout),
|
|
557
|
+
Rule("R02", "records validate; money and quantities are strings, never floats", _r02_records),
|
|
558
|
+
Rule("R03", "the RunResult is the run manifest", _r03_result),
|
|
559
|
+
Rule("R04", "state.json: shape, and a damaged one is exit 2 before any request", _r04_state),
|
|
560
|
+
Rule("R05", "files 0600 and folders 0700 under any umask", _r05_permissions),
|
|
561
|
+
Rule("R06", "raw files are kept and never overwritten", _r06_raw_kept),
|
|
562
|
+
Rule("R07", "a run in the second of a finished run is refused", _r07_same_second),
|
|
563
|
+
Rule("R08", "the revision rule: publish only records whose content differs", _r08_revision),
|
|
564
|
+
Rule("R09", "rebuild needs no key and no network and reproduces the records", _r09_rebuild),
|
|
565
|
+
Rule("R10", "failure and resume", _r10_resume),
|
|
566
|
+
Rule("R11", "exit codes 0, 1, 2, 3 and failure_kind", _r11_exit_codes),
|
|
567
|
+
Rule("R12", "the injected clock and sleep are the only waiting", _r12_clock),
|
|
568
|
+
Rule("R13", "the key never reaches a file or a message", _r13_secrets),
|
|
569
|
+
Rule("R14", "the public fake API: transport, request log, fault scripting", _r14_fake),
|
|
570
|
+
Rule("R15", "rebuild keeps Decimal: no float is read, the records are the same bytes", _r15_rebuild_keeps_decimal),
|
|
571
|
+
Rule("R16", "batch checks run before publish", _r16_batch_checks),
|
|
572
|
+
)
|
|
573
|
+
_BY_ID: Dict[str, Rule] = {rule.id: rule for rule in RULES}
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def check(rule: Union[Rule, str], connector: Connector, workdir: Union[str, Path]) -> None:
|
|
577
|
+
"""Run one rule of the contract against ``connector`` in ``workdir`` (an empty folder); raises ``ConformanceError``."""
|
|
578
|
+
chosen = _BY_ID[rule] if isinstance(rule, str) else rule
|
|
579
|
+
chosen.check(connector, Path(workdir))
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def failures(connector: Connector, workdir: Union[str, Path]) -> Iterator[Tuple[Rule, str]]:
|
|
583
|
+
"""Every rule that fails, with its message, each in its own subfolder of ``workdir``; for a report, not a test."""
|
|
584
|
+
for rule in RULES:
|
|
585
|
+
folder = Path(workdir) / rule.id
|
|
586
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
587
|
+
try:
|
|
588
|
+
rule.check(connector, folder)
|
|
589
|
+
except ConformanceError as exc:
|
|
590
|
+
yield rule, str(exc)
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
# ---- compare_shape -------------------------------------------------------------------------------------------------
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _kind(value: Any) -> str:
|
|
597
|
+
if isinstance(value, dict):
|
|
598
|
+
return "object"
|
|
599
|
+
if isinstance(value, list):
|
|
600
|
+
return "array"
|
|
601
|
+
return "scalar"
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
def _load_shape(shape_file: Any) -> Any:
|
|
605
|
+
"""A shape: a ``str`` or ``Path`` is a ``.json`` file or an ``.ndjson`` file (its first line is one row); anything
|
|
606
|
+
else is a shape already loaded."""
|
|
607
|
+
if not isinstance(shape_file, (str, Path)):
|
|
608
|
+
return shape_file
|
|
609
|
+
path = Path(shape_file)
|
|
610
|
+
if path.suffix == ".json":
|
|
611
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
612
|
+
if path.suffix == ".ndjson":
|
|
613
|
+
for line in path.read_text(encoding="utf-8").splitlines():
|
|
614
|
+
if line.strip():
|
|
615
|
+
return json.loads(line)
|
|
616
|
+
raise ValueError("the shape file %s holds no row" % path)
|
|
617
|
+
raise ValueError("a shape file is .json or .ndjson, not %s" % (path.suffix or path.name))
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def _is_optional(keys: Tuple[str, ...], optional: Tuple[Tuple[str, ...], ...]) -> bool:
|
|
621
|
+
"""True when the key path ends with one of the optional entries (``("metrics", "tokens")`` or ``("note",)``)."""
|
|
622
|
+
return any(len(entry) <= len(keys) and keys[len(keys) - len(entry):] == entry for entry in optional)
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def _shape_differences(shape: Any, actual: Any, path: str, keys: Tuple[str, ...], data_keys: frozenset,
|
|
626
|
+
optional: Tuple[Tuple[str, ...], ...]) -> List[str]:
|
|
627
|
+
if _kind(shape) != _kind(actual):
|
|
628
|
+
return ["%s: expected %s, got %s" % (path, _kind(shape), _kind(actual))]
|
|
629
|
+
found: List[str] = []
|
|
630
|
+
if isinstance(shape, dict):
|
|
631
|
+
if not shape: # an empty map in the shape file is free-form and not compared
|
|
632
|
+
return found
|
|
633
|
+
if keys and keys[-1] in data_keys: # the keys are data: compare the values with the shape of the first value
|
|
634
|
+
sample = next(iter(shape.values()))
|
|
635
|
+
for key, value in actual.items():
|
|
636
|
+
found.extend(_shape_differences(sample, value, "%s.%s" % (path, key), keys + ("*",), data_keys, optional))
|
|
637
|
+
return found
|
|
638
|
+
for key in sorted(set(shape) | set(actual), key=str):
|
|
639
|
+
here = keys + (str(key),)
|
|
640
|
+
if key not in actual:
|
|
641
|
+
if not _is_optional(here, optional):
|
|
642
|
+
found.append("%s.%s: missing" % (path, key))
|
|
643
|
+
elif key not in shape:
|
|
644
|
+
found.append("%s.%s: not in the documented shape" % (path, key))
|
|
645
|
+
else:
|
|
646
|
+
found.extend(_shape_differences(shape[key], actual[key], "%s.%s" % (path, key), here, data_keys, optional))
|
|
647
|
+
elif isinstance(shape, list) and shape: # an empty list in the shape file is not compared
|
|
648
|
+
for index, item in enumerate(actual):
|
|
649
|
+
found.extend(_shape_differences(shape[0], item, "%s[%d]" % (path, index), keys, data_keys, optional))
|
|
650
|
+
return found
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
def compare_shape(answer: Any, shape_file: Any, *, data_keys: Iterable[str] = (), optional: Iterable[str] = ()) -> List[str]:
|
|
654
|
+
"""The differences in keys and nesting between ``answer`` and a shape, never by value; ``[]`` when there are none.
|
|
655
|
+
|
|
656
|
+
``answer`` is a parsed JSON value. ``shape_file`` is the path of a ``.json`` file, or of an ``.ndjson`` file whose
|
|
657
|
+
first line is one row, or a shape already loaded. Each difference is one string, the path and what differs:
|
|
658
|
+
``$.data[0].usage: missing``, ``$.data[0].extra: not in the documented shape`` or ``$.data: expected array, got
|
|
659
|
+
object``. Every item of a list is compared with the first item of the shape's list.
|
|
660
|
+
|
|
661
|
+
``data_keys`` names maps whose keys are data (model names, ids): their values are compared with the first value of
|
|
662
|
+
the shape's map, whatever the keys are. ``optional`` lists keys that may be absent: ``"note"`` is a key of that name
|
|
663
|
+
at any depth, ``"metrics.tokens"`` is that key directly inside a ``metrics`` map (a path may start anywhere; no
|
|
664
|
+
key name may hold a dot). An optional key that is present is still compared. An empty map or list in the shape file
|
|
665
|
+
is not compared, and an answer with no item in a list has nothing to compare there: a test that must see a row
|
|
666
|
+
asks for one itself.
|
|
667
|
+
"""
|
|
668
|
+
entries = tuple(tuple(str(entry).split(".")) for entry in optional)
|
|
669
|
+
return _shape_differences(_load_shape(shape_file), answer, "$", (), frozenset(data_keys), entries)
|
|
670
|
+
|
|
671
|
+
|
|
672
|
+
__all__ = ["CONTRACT_PAGE", "RULES", "ConformanceError", "Connector", "Rule", "check", "compare_shape", "failures"]
|