setrya 0.12.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- setrya/__init__.py +3 -0
- setrya/__main__.py +3 -0
- setrya/adapter.py +84 -0
- setrya/cache_metrics.py +137 -0
- setrya/checker.py +202 -0
- setrya/cli.py +348 -0
- setrya/comparison.py +126 -0
- setrya/conformance.py +141 -0
- setrya/coverage.py +311 -0
- setrya/dependencies.py +105 -0
- setrya/drill.py +188 -0
- setrya/evidence.py +367 -0
- setrya/examples/__init__.py +1 -0
- setrya/examples/answer_cache/__init__.py +1 -0
- setrya/examples/answer_cache/__main__.py +3 -0
- setrya/examples/answer_cache/app.py +167 -0
- setrya/examples/answer_cache/contract.py +19 -0
- setrya/examples/answer_cache/drill.py +150 -0
- setrya/examples/answer_cache/evidence.py +274 -0
- setrya/examples/answer_cache/metrics.py +114 -0
- setrya/examples/answer_cache/selective.py +325 -0
- setrya/examples/answer_cache/store.py +238 -0
- setrya/examples/answer_cache/worker.py +53 -0
- setrya/examples/elasticsearch_search.py +106 -0
- setrya/examples/offboarding/__init__.py +1 -0
- setrya/examples/offboarding/demo.py +105 -0
- setrya/examples/offboarding/fixture.py +376 -0
- setrya/examples/offboarding/sqlite/__init__.py +1 -0
- setrya/examples/offboarding/sqlite/observe.py +344 -0
- setrya/examples/offboarding/sqlite/policy.json +8 -0
- setrya/examples/opendesk/Dockerfile.trial +26 -0
- setrya/examples/opendesk/__init__.py +1 -0
- setrya/examples/opendesk/cli.py +73 -0
- setrya/examples/opendesk/compose.yaml +70 -0
- setrya/examples/opendesk/observed.json +66 -0
- setrya/examples/opendesk/policy-v2.json +26 -0
- setrya/examples/opendesk/policy.json +26 -0
- setrya/examples/opendesk/provider.py +53 -0
- setrya/examples/opendesk/run.py +441 -0
- setrya/examples/opendesk/to_gate.py +129 -0
- setrya/examples/opendesk/transport.py +31 -0
- setrya/examples/opendesk/verify_report.py +254 -0
- setrya/examples/pgvector-search/compose.yml +14 -0
- setrya/examples/pgvector-search/drill.toml +40 -0
- setrya/examples/pgvector-search/reupload.toml +51 -0
- setrya/examples/pgvector_search/__init__.py +8 -0
- setrya/examples/pgvector_search/__main__.py +26 -0
- setrya/examples/pgvector_search/adapter.py +16 -0
- setrya/examples/pgvector_search/app.py +242 -0
- setrya/examples/pgvector_search/benchmark.py +118 -0
- setrya/examples/pgvector_search/pytest_reference.py +89 -0
- setrya/examples/recorded-search/drill.toml +34 -0
- setrya/examples/recorded_search/__init__.py +1 -0
- setrya/examples/recorded_search/existing_test.py +62 -0
- setrya/examples/recorded_search/verify.py +93 -0
- setrya/examples/support-search/drill.toml +40 -0
- setrya/examples/support-search/python.toml +40 -0
- setrya/examples/support-search/retired.toml +51 -0
- setrya/examples/support_search/__init__.py +1 -0
- setrya/examples/support_search/__main__.py +104 -0
- setrya/examples/support_search/adapter.py +61 -0
- setrya/examples/support_search/app.py +225 -0
- setrya/examples/support_search/baseline.py +101 -0
- setrya/examples/support_search/client.py +78 -0
- setrya/git_refs.py +23 -0
- setrya/integrations/__init__.py +1 -0
- setrya/integrations/celery.py +97 -0
- setrya/integrations/langchain.py +83 -0
- setrya/local_cache.py +143 -0
- setrya/offboarding/__init__.py +1 -0
- setrya/offboarding/cli.py +74 -0
- setrya/offboarding/gate.py +202 -0
- setrya/offboarding/inputs.py +86 -0
- setrya/offboarding/observer.py +47 -0
- setrya/offboarding/receipt-v1.schema.json +300 -0
- setrya/offboarding/receipt.py +264 -0
- setrya/offboarding/receipt_cli.py +197 -0
- setrya/offboarding/receipt_schema.py +65 -0
- setrya/offboarding/reference.py +243 -0
- setrya/offboarding/signature.py +68 -0
- setrya/pytest_plugin.py +82 -0
- setrya/qualification.py +412 -0
- setrya/recorder.py +163 -0
- setrya/review.py +119 -0
- setrya/review_scenarios.py +157 -0
- setrya/run_pr.py +168 -0
- setrya/runner.py +184 -0
- setrya/scenario_fixture.py +127 -0
- setrya/scenario_order.py +268 -0
- setrya/scenarios.py +509 -0
- setrya/status.py +22 -0
- setrya/store_signals.py +56 -0
- setrya-0.12.1.dist-info/METADATA +221 -0
- setrya-0.12.1.dist-info/RECORD +98 -0
- setrya-0.12.1.dist-info/WHEEL +5 -0
- setrya-0.12.1.dist-info/entry_points.txt +2 -0
- setrya-0.12.1.dist-info/licenses/LICENSE +202 -0
- setrya-0.12.1.dist-info/top_level.txt +1 -0
setrya/cli.py
ADDED
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
"""Command-line entry point for check, replay, demo and qualification."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
from contextlib import closing
|
|
5
|
+
from dataclasses import replace
|
|
6
|
+
import os
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import re
|
|
9
|
+
import shlex
|
|
10
|
+
import sys
|
|
11
|
+
from tempfile import NamedTemporaryFile
|
|
12
|
+
|
|
13
|
+
from . import __version__
|
|
14
|
+
from .checker import Status, Verdict, evaluate
|
|
15
|
+
from .drill import Drill, InputError
|
|
16
|
+
from .evidence import Report, bundle_manifest, matches_search
|
|
17
|
+
from .runner import RunResult, run
|
|
18
|
+
|
|
19
|
+
ROOT = Path(__file__).resolve().parent.parent
|
|
20
|
+
EXAMPLE = ROOT / "setrya" / "examples" / "support-search"
|
|
21
|
+
PGVECTOR = ROOT / "setrya" / "examples" / "pgvector-search"
|
|
22
|
+
DEMO_CASES = {"broken": Status.FAIL, "fixed": Status.PASS, "opaque": Status.NOT_TESTED}
|
|
23
|
+
DEMO_PASSAGES = {"refund@A": "30 days", "refund@B": "14 days"}
|
|
24
|
+
|
|
25
|
+
def preflight() -> None:
|
|
26
|
+
try:
|
|
27
|
+
import sqlite3
|
|
28
|
+
except ImportError:
|
|
29
|
+
raise InputError("The SQLite fixture needs a Python build with SQLite FTS5 enabled.") from None
|
|
30
|
+
try:
|
|
31
|
+
with closing(sqlite3.connect(":memory:")) as connection:
|
|
32
|
+
connection.execute("CREATE VIRTUAL TABLE preflight USING fts5(body)")
|
|
33
|
+
except sqlite3.Error:
|
|
34
|
+
raise InputError("SQLite FTS5 is unavailable. Use a Python build with FTS5 enabled.") from None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def format_verdict(
|
|
38
|
+
verdict: Verdict, report: Report, rerun: str | None = None, replay: str | None = None
|
|
39
|
+
) -> str:
|
|
40
|
+
lines = [f"{verdict.status.label} {verdict.detail}"]
|
|
41
|
+
if verdict.probe:
|
|
42
|
+
rule = next(probe for probe in report.rules.probes if probe.id == verdict.probe)
|
|
43
|
+
lines.append(f" Probe {rule.id} · route {rule.route} · scope {rule.scope} · top {rule.top_k}")
|
|
44
|
+
if verdict.event:
|
|
45
|
+
event = report.events[verdict.event - 1]
|
|
46
|
+
lines.append(f" Event {verdict.event} at {event['ns'] / 1_000_000:.1f} ms")
|
|
47
|
+
if event["kind"] == "probe":
|
|
48
|
+
lines.append(f" Observed {passage_list(event)}")
|
|
49
|
+
publication = next((event.get("publication") for event in report.events if event["kind"] == "published"), None)
|
|
50
|
+
if publication:
|
|
51
|
+
lines.append(f" Published {publication['document']} revision {publication['revision']}")
|
|
52
|
+
if verdict.status is Status.NOT_TESTED:
|
|
53
|
+
latest = {event["probe_id"]: event for event in report.events if event["kind"] == "probe"}
|
|
54
|
+
for event in latest.values():
|
|
55
|
+
where = f"{event.get('route') or '?'} / {event.get('scope') or '?'}"
|
|
56
|
+
lines.append(f" Last {event['stage']} {event['probe_id']} ({where}) → {passage_list(event)}")
|
|
57
|
+
if verdict.attribution:
|
|
58
|
+
lines.append(f" {verdict.attribution}")
|
|
59
|
+
for event in report.events:
|
|
60
|
+
if event["kind"] == "adapter_error":
|
|
61
|
+
operation = event["operation"]
|
|
62
|
+
detail = "Test error" if operation == "test" else f"Adapter error during {operation}"
|
|
63
|
+
lines.append(f" {detail} ({event['error']})")
|
|
64
|
+
lines.append(f" Run {report.run_id}")
|
|
65
|
+
if replay:
|
|
66
|
+
lines.append(f" Replay {replay}")
|
|
67
|
+
if rerun:
|
|
68
|
+
lines.append(f" Rerun {rerun}")
|
|
69
|
+
return "\n".join(lines)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def passage_list(event: dict) -> str:
|
|
73
|
+
return ", ".join(marker if marker is not None else "?" for marker in event["passages"]) or "(empty)"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def history(report: Report) -> str:
|
|
77
|
+
lines = [" History"]
|
|
78
|
+
for event in report.events:
|
|
79
|
+
if event["kind"] == "probe":
|
|
80
|
+
where = f"{event.get('route') or '?'} / {event.get('scope') or '?'}"
|
|
81
|
+
lines.append(f" {event['seq']:>3} {event['stage']} {event['probe_id']} ({where}) → {passage_list(event)}")
|
|
82
|
+
elif event["kind"] == "receipt":
|
|
83
|
+
lines.append(f" {event['seq']:>3} old write {event['outcome']}")
|
|
84
|
+
elif event["kind"] == "published" and not event.get("publication"):
|
|
85
|
+
lines.append(f" {event['seq']:>3} publication attempted")
|
|
86
|
+
elif event["kind"] == "adapter_error":
|
|
87
|
+
lines.append(f" {event['seq']:>3} adapter error during {event['operation']}")
|
|
88
|
+
else:
|
|
89
|
+
lines.append(f" {event['seq']:>3} {event['kind']}")
|
|
90
|
+
return "\n".join(lines)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def demo_case(mode: str) -> RunResult:
|
|
94
|
+
drill = Drill.load(EXAMPLE / "python.toml")
|
|
95
|
+
return run(replace(drill, options={"mode": mode}))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def demo(selected: str | None) -> int:
|
|
99
|
+
ok = True
|
|
100
|
+
for mode in (selected,) if selected else DEMO_CASES:
|
|
101
|
+
result = demo_case(mode)
|
|
102
|
+
verdict = result.verdict
|
|
103
|
+
ok &= verdict.status is DEMO_CASES[mode]
|
|
104
|
+
print(f"{verdict.status.label:<11} {verdict.detail}")
|
|
105
|
+
primary = next(probe for probe in result.report.rules.probes
|
|
106
|
+
if probe.id == result.report.rules.freshness_probe)
|
|
107
|
+
observed = []
|
|
108
|
+
for event in result.report.events:
|
|
109
|
+
if event.get("probe_id") == primary.id and matches_search(event, primary):
|
|
110
|
+
result_text = ", ".join(DEMO_PASSAGES.get(marker, marker or "?") for marker in event["passages"])
|
|
111
|
+
result_text = result_text or "(empty)"
|
|
112
|
+
if not observed or observed[-1] != result_text:
|
|
113
|
+
observed.append(result_text)
|
|
114
|
+
if observed:
|
|
115
|
+
print(f" Refund policy in search · {' → '.join(observed)}")
|
|
116
|
+
for event in result.report.events:
|
|
117
|
+
if event["kind"] == "held":
|
|
118
|
+
print(f" Checkpoint held revision {event['revision']}")
|
|
119
|
+
elif event["kind"] == "released":
|
|
120
|
+
print(" Old worker released")
|
|
121
|
+
elif event["kind"] == "receipt":
|
|
122
|
+
print(f" Old write {event['outcome']}")
|
|
123
|
+
elif event["kind"] == "adapter_error":
|
|
124
|
+
print(f" Adapter error during {event['operation']} ({event['error']})")
|
|
125
|
+
print(f" Rerun python3 -m setrya demo --case {mode}")
|
|
126
|
+
return 0 if ok else 1
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def rerun_command(path: str) -> str:
|
|
130
|
+
# Reports and summaries avoid embedding private absolute paths or file names.
|
|
131
|
+
if re.fullmatch(r"setrya/examples/(?:support-search/(?:drill|python|retired)|pgvector-search/(?:drill|reupload))\.toml", path):
|
|
132
|
+
return "python3 -m setrya check " + shlex.quote(path)
|
|
133
|
+
return 'python3 -m setrya check "$DRILL" --report run.json'
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def replay_command(report_path: str | None, *, artifact: str = "run.json", variable: str = "REPORT") -> str:
|
|
137
|
+
if report_path is None:
|
|
138
|
+
return "Save with --report run.json, then run setrya replay run.json"
|
|
139
|
+
target = artifact if Path(report_path).name == artifact else f'"${variable}"'
|
|
140
|
+
return f"setrya replay {target}"
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def console_main() -> int:
|
|
144
|
+
args = sys.argv[1:]
|
|
145
|
+
live_scenario = args[:1] == ["scenario"] and args[1:2] in (["update"], ["delete"], ["revoke"])
|
|
146
|
+
if args[:1] != ["check"] and args[:2] != ["offboarding", "drill"] and not live_scenario:
|
|
147
|
+
return main()
|
|
148
|
+
# Only live checks import adapters from the working checkout.
|
|
149
|
+
original_path = sys.path
|
|
150
|
+
try:
|
|
151
|
+
sys.path = [str(Path.cwd()), *original_path]
|
|
152
|
+
return main()
|
|
153
|
+
finally:
|
|
154
|
+
sys.path = original_path
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _write_summary(path: str | Path, title: str, text: str) -> None:
|
|
158
|
+
target = Path(path).absolute()
|
|
159
|
+
temporary = None
|
|
160
|
+
try:
|
|
161
|
+
if target.is_symlink() or (target.exists() and not target.is_file()):
|
|
162
|
+
raise InputError("summary destination must be a regular file")
|
|
163
|
+
with NamedTemporaryFile("w", dir=target.parent, prefix=".setrya-", encoding="utf-8", delete=False) as stream:
|
|
164
|
+
temporary = Path(stream.name)
|
|
165
|
+
stream.write(f"### Setrya · {title}\n\n```text\n{text}\n```\n")
|
|
166
|
+
temporary.replace(target)
|
|
167
|
+
except OSError:
|
|
168
|
+
raise InputError("could not write summary. Check the destination directory.") from None
|
|
169
|
+
finally:
|
|
170
|
+
if temporary is not None:
|
|
171
|
+
temporary.unlink(missing_ok=True)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def main(argv=None) -> int:
|
|
175
|
+
argv = sys.argv[1:] if argv is None else list(argv)
|
|
176
|
+
if argv[:1] == ["cache-metrics"]:
|
|
177
|
+
from .examples.answer_cache.metrics import main as metrics_main
|
|
178
|
+
|
|
179
|
+
return metrics_main(argv[1:])
|
|
180
|
+
if argv[:1] == ["answer-cache"]:
|
|
181
|
+
from .examples.answer_cache.drill import main as answer_cache_main
|
|
182
|
+
|
|
183
|
+
return answer_cache_main(argv[1:])
|
|
184
|
+
if argv[:1] == ["coverage"]:
|
|
185
|
+
from .coverage import main as coverage_main
|
|
186
|
+
|
|
187
|
+
return coverage_main(argv[1:])
|
|
188
|
+
if argv[:1] == ["run-pr"]:
|
|
189
|
+
from .run_pr import main as run_pr
|
|
190
|
+
|
|
191
|
+
return run_pr(argv[1:])
|
|
192
|
+
if argv[:1] == ["scenario"]:
|
|
193
|
+
from .scenarios import main as scenario_main
|
|
194
|
+
|
|
195
|
+
return scenario_main(argv[1:])
|
|
196
|
+
if argv[:1] == ["offboarding"]:
|
|
197
|
+
from .offboarding.cli import main as offboarding_main
|
|
198
|
+
|
|
199
|
+
return offboarding_main(argv[1:])
|
|
200
|
+
parser = argparse.ArgumentParser(prog="setrya", description="Test what an application can retrieve after a document update or tenant offboarding.")
|
|
201
|
+
parser.add_argument("--version", action="version", version=__version__)
|
|
202
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
203
|
+
commands.add_parser("scenario", help="drive update, deletion or access-revocation races and replay a schedule")
|
|
204
|
+
commands.add_parser("offboarding", help="check tenant offboarding in CI or review one deletion request")
|
|
205
|
+
commands.add_parser("cache-metrics", help="measure checked answers and cache outcomes from local evidence")
|
|
206
|
+
commands.add_parser("answer-cache", help="test a late Celery cache fill against a real answer endpoint")
|
|
207
|
+
commands.add_parser("coverage", help="find changed routes and jobs without matching candidate evidence")
|
|
208
|
+
commands.add_parser("run-pr", help="run an existing test on two Git revisions and review its evidence")
|
|
209
|
+
demo_parser = commands.add_parser("demo", help="run three synthetic outcomes")
|
|
210
|
+
demo_parser.add_argument("--case", choices=DEMO_CASES)
|
|
211
|
+
check = commands.add_parser("check", help="run a drill")
|
|
212
|
+
check.add_argument("drill")
|
|
213
|
+
check.add_argument("--report", help="write a portable JSON report")
|
|
214
|
+
check.add_argument("--summary", help="write a Markdown summary")
|
|
215
|
+
check.add_argument("--revision", help="record the full Git commit ID supplying the application")
|
|
216
|
+
replay = commands.add_parser("replay", help="recompute a verdict from saved observations")
|
|
217
|
+
replay.add_argument("report")
|
|
218
|
+
diagnose = commands.add_parser("diagnose", help="show the verdict and the first missing observation")
|
|
219
|
+
diagnose.add_argument("report")
|
|
220
|
+
comparison = commands.add_parser("compare", help="tell whether a before-and-after pair establishes a fix")
|
|
221
|
+
comparison.add_argument("before")
|
|
222
|
+
comparison.add_argument("after")
|
|
223
|
+
comparison.add_argument("--summary", help="write a Markdown summary")
|
|
224
|
+
review_parser = commands.add_parser("review", help="review two CI reports or two scenario directories")
|
|
225
|
+
review_parser.add_argument("before")
|
|
226
|
+
review_parser.add_argument("after")
|
|
227
|
+
review_parser.add_argument("--base-commit", help="expected full base Git commit ID (required)")
|
|
228
|
+
review_parser.add_argument("--candidate-commit", help="expected full candidate Git commit ID (required)")
|
|
229
|
+
review_parser.add_argument("--base-exit", help="native base test exit (required)")
|
|
230
|
+
review_parser.add_argument("--candidate-exit", help="native candidate test exit (required)")
|
|
231
|
+
review_parser.add_argument("--exit-contract", choices=("pytest", "setrya"),
|
|
232
|
+
help="native exits (default: pytest for files, setrya for scenario directories)")
|
|
233
|
+
review_parser.add_argument("--summary", help="write a Markdown summary")
|
|
234
|
+
qualify = commands.add_parser("qualify", help="run the shipped qualification cases")
|
|
235
|
+
qualify.add_argument("--case")
|
|
236
|
+
qualify.add_argument("--quick", action="store_true", help="skip the 90-run repeatability case")
|
|
237
|
+
args = parser.parse_args(argv)
|
|
238
|
+
error_summary = None
|
|
239
|
+
|
|
240
|
+
try:
|
|
241
|
+
if args.command in {"replay", "diagnose"}:
|
|
242
|
+
directory = Path(args.report)
|
|
243
|
+
manifest = bundle_manifest(directory) if directory.is_dir() else None
|
|
244
|
+
if manifest == "answer-cache.json":
|
|
245
|
+
from .examples.answer_cache.drill import main as answer_cache_main
|
|
246
|
+
|
|
247
|
+
return answer_cache_main(["--replay", args.report])
|
|
248
|
+
if manifest == "scenario.json" or (directory.is_dir() and args.command == "replay"):
|
|
249
|
+
from .scenarios import main as scenario_main
|
|
250
|
+
|
|
251
|
+
return scenario_main(["replay", args.report])
|
|
252
|
+
report = Report.load(args.report)
|
|
253
|
+
verdict = evaluate(report)
|
|
254
|
+
print(format_verdict(verdict, report))
|
|
255
|
+
if args.command == "replay":
|
|
256
|
+
print(history(report))
|
|
257
|
+
return verdict.status.exit_code
|
|
258
|
+
if args.command in {"compare", "review"}:
|
|
259
|
+
from .comparison import compare
|
|
260
|
+
|
|
261
|
+
if args.summary:
|
|
262
|
+
destination = Path(args.summary).resolve()
|
|
263
|
+
lexical = Path(os.path.abspath(args.summary))
|
|
264
|
+
for path in (Path(args.before), Path(args.after)):
|
|
265
|
+
source = path.resolve()
|
|
266
|
+
if destination == source or (path.is_dir() and (
|
|
267
|
+
destination.is_relative_to(source) or lexical.is_relative_to(os.path.abspath(path))
|
|
268
|
+
)):
|
|
269
|
+
raise InputError("comparison summary must use a distinct file outside the evidence directories")
|
|
270
|
+
error_summary = args.summary
|
|
271
|
+
if args.command == "review":
|
|
272
|
+
from .review import ReviewInput, review_inputs
|
|
273
|
+
|
|
274
|
+
base = ReviewInput.load(args.before, args.base_commit, args.base_exit)
|
|
275
|
+
candidate = ReviewInput.load(args.after, args.candidate_commit, args.candidate_exit)
|
|
276
|
+
lanes = (("base", base, args.before), ("candidate", candidate, args.after))
|
|
277
|
+
named_directories = frozenset(name for name, _, path in lanes if Path(path) == Path(name))
|
|
278
|
+
code, summary = review_inputs(base, candidate, named_directories=named_directories,
|
|
279
|
+
exit_contract=args.exit_contract)
|
|
280
|
+
for name, lane, path in lanes:
|
|
281
|
+
if lane.report is not None:
|
|
282
|
+
variable = f"{name.upper()}_REPORT"
|
|
283
|
+
if isinstance(lane.report, Report):
|
|
284
|
+
command = replay_command(path, artifact=f"{name}.json", variable=variable)
|
|
285
|
+
else:
|
|
286
|
+
from .review_scenarios import replay_scenario_command
|
|
287
|
+
|
|
288
|
+
command = replay_scenario_command(name, named_directories)
|
|
289
|
+
if name not in named_directories:
|
|
290
|
+
summary += f"\nSet {variable} to the path of the downloaded {name} scenario directory."
|
|
291
|
+
summary += f"\nReplay {name} {command}"
|
|
292
|
+
title = f"{base.report.rules.id} release review" if isinstance(base.report, Report) else "release review"
|
|
293
|
+
else:
|
|
294
|
+
before, after = Report.load(args.before), Report.load(args.after)
|
|
295
|
+
code, summary = compare(before, after)
|
|
296
|
+
title = f"{before.rules.id} comparison"
|
|
297
|
+
print(summary)
|
|
298
|
+
error_summary = None
|
|
299
|
+
if args.summary:
|
|
300
|
+
_write_summary(args.summary, title, summary)
|
|
301
|
+
return code
|
|
302
|
+
if args.command == "check":
|
|
303
|
+
paths = [Path(path).resolve() for path in (args.drill, args.report, args.summary) if path is not None]
|
|
304
|
+
if len(set(paths)) != len(paths):
|
|
305
|
+
raise InputError("drill, report and summary must use distinct files")
|
|
306
|
+
report_path = Path(args.report).absolute() if args.report else None
|
|
307
|
+
summary_path = Path(args.summary).absolute() if args.summary else None
|
|
308
|
+
error_summary = summary_path
|
|
309
|
+
drill = Drill.load(args.drill)
|
|
310
|
+
if drill.factory in {"setrya.examples.support_search.adapter:http_fixture",
|
|
311
|
+
"setrya.examples.support_search.adapter:python_fixture"}:
|
|
312
|
+
preflight()
|
|
313
|
+
result = run(drill, revision=args.revision)
|
|
314
|
+
if report_path:
|
|
315
|
+
result.report.save(report_path)
|
|
316
|
+
replay_hint = replay_command(args.report)
|
|
317
|
+
summary = format_verdict(result.verdict, result.report, rerun_command(args.drill), replay_hint)
|
|
318
|
+
print(summary)
|
|
319
|
+
interrupted = next((e for e in result.report.events if e["kind"] == "adapter_error"
|
|
320
|
+
and e["error"] == "KeyboardInterrupt"), None)
|
|
321
|
+
if interrupted is not None:
|
|
322
|
+
print(f"Interrupted during {interrupted['operation']}. {replay_hint}", file=sys.stderr)
|
|
323
|
+
error_summary = None
|
|
324
|
+
if summary_path:
|
|
325
|
+
_write_summary(summary_path, result.report.rules.id, summary)
|
|
326
|
+
return result.verdict.status.exit_code
|
|
327
|
+
if args.command == "demo":
|
|
328
|
+
preflight()
|
|
329
|
+
return demo(args.case)
|
|
330
|
+
if args.case not in {"pgvector", "offboarding", "offboarding-compatibility"}:
|
|
331
|
+
preflight()
|
|
332
|
+
from .qualification import qualify as run_qualification
|
|
333
|
+
|
|
334
|
+
return run_qualification(args.case, args.quick)
|
|
335
|
+
except InputError as error:
|
|
336
|
+
diagnostic = f"ERROR {error}"
|
|
337
|
+
except KeyboardInterrupt:
|
|
338
|
+
diagnostic = "ERROR Interrupted. Fixture teardown requested."
|
|
339
|
+
except Exception as error:
|
|
340
|
+
# Third-party errors may contain queries, snippets and local URLs.
|
|
341
|
+
diagnostic = f"ERROR {type(error).__name__} during {args.command}. Check adapter health and local permissions."
|
|
342
|
+
print(diagnostic, file=sys.stderr)
|
|
343
|
+
if error_summary:
|
|
344
|
+
try:
|
|
345
|
+
_write_summary(error_summary, f"{args.command} error", diagnostic)
|
|
346
|
+
except InputError as error:
|
|
347
|
+
print(f"ERROR {error}", file=sys.stderr)
|
|
348
|
+
return 2
|
setrya/comparison.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Compare two saved drills without contacting either application."""
|
|
2
|
+
|
|
3
|
+
from .checker import Status, Verdict, evaluate
|
|
4
|
+
from .drill import ProbeRules
|
|
5
|
+
from .evidence import Report, matches_search, observed_change
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _difference(before: Report, after: Report) -> str | None:
|
|
9
|
+
if before.inputs is None or after.inputs is None:
|
|
10
|
+
return "Both runs need schema 3 drill input fingerprints. Run both checks again."
|
|
11
|
+
if before.run_id == after.run_id:
|
|
12
|
+
return "Supply reports from two distinct runs."
|
|
13
|
+
if before.inputs["drill_version"] != after.inputs["drill_version"]:
|
|
14
|
+
return "Drill input versions differ."
|
|
15
|
+
if before.inputs["adapter"] != after.inputs["adapter"]:
|
|
16
|
+
return "The adapter factory differs."
|
|
17
|
+
earlier = {probe.id: probe for probe in before.rules.probes}
|
|
18
|
+
later = {probe.id: probe for probe in after.rules.probes}
|
|
19
|
+
if earlier.keys() != later.keys():
|
|
20
|
+
return "The reports cover different probes."
|
|
21
|
+
for probe_id, probe in earlier.items():
|
|
22
|
+
changed = later[probe_id]
|
|
23
|
+
if before.inputs["questions"][probe_id] != after.inputs["questions"][probe_id]:
|
|
24
|
+
return f"Question changed for {probe_id}."
|
|
25
|
+
if probe.top_k != changed.top_k:
|
|
26
|
+
return f"Search depth changed for {probe_id} from {probe.top_k} to {changed.top_k}."
|
|
27
|
+
if (probe.route, probe.scope) != (changed.route, changed.scope):
|
|
28
|
+
return f"Route or caller changed for {probe_id}."
|
|
29
|
+
if probe != changed:
|
|
30
|
+
return f"Passage expectations changed for {probe_id}."
|
|
31
|
+
if before.rules != after.rules:
|
|
32
|
+
return "Revision, schedule, drill identity, or probe order changed."
|
|
33
|
+
return None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _event(report: Report, verdict: Verdict) -> dict | None:
|
|
37
|
+
if verdict.event is not None:
|
|
38
|
+
return report.events[verdict.event - 1]
|
|
39
|
+
return None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _first_change(report: Report, primary: ProbeRules) -> dict | None:
|
|
43
|
+
return next((event for event in report.events if event.get("stage") == "freshness"
|
|
44
|
+
and event["probe_id"] == primary.id and matches_search(event, primary)
|
|
45
|
+
and observed_change(event, primary)), None)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _matching_proof(report: Report, verdict: Verdict, probe: ProbeRules,
|
|
49
|
+
failed_report: Report, failed: dict) -> dict | None:
|
|
50
|
+
if verdict.status is not Status.PASS:
|
|
51
|
+
return None
|
|
52
|
+
if failed["stage"] == "final":
|
|
53
|
+
return next((report.events[seq - 1] for seq in verdict.confirmed
|
|
54
|
+
if report.events[seq - 1]["probe_id"] == probe.id), None)
|
|
55
|
+
|
|
56
|
+
primary = next(item for item in report.rules.probes if item.id == report.rules.freshness_probe)
|
|
57
|
+
change = _first_change(report, primary)
|
|
58
|
+
if change is None:
|
|
59
|
+
return None
|
|
60
|
+
failed_change = _first_change(failed_report, primary)
|
|
61
|
+
# A recurrence on a second query needs a second query on the passing lane.
|
|
62
|
+
# A failure on the transition itself can be compared to the transition.
|
|
63
|
+
minimum = change["seq"]
|
|
64
|
+
if failed_change and failed["seq"] > failed_change["seq"]:
|
|
65
|
+
minimum += 1
|
|
66
|
+
return next((event for event in report.events
|
|
67
|
+
if event.get("stage") == "freshness" and event["probe_id"] == probe.id
|
|
68
|
+
and event["seq"] >= minimum and matches_search(event, probe)
|
|
69
|
+
and None not in event["passages"] and set(probe.require) <= set(event["passages"])
|
|
70
|
+
and not set(probe.forbid).intersection(event["passages"])), None)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _describe(label: str, verdict: Verdict, event: dict | None) -> str:
|
|
74
|
+
if event is None or event["kind"] != "probe":
|
|
75
|
+
return f"{label} {verdict.status.label} · {verdict.detail}"
|
|
76
|
+
passages = ", ".join(marker or "?" for marker in event["passages"]) or "(empty)"
|
|
77
|
+
return (f"{label} {verdict.status.label} · event {event['seq']} · {event['probe_id']} · {event['stage']} · {passages}\n"
|
|
78
|
+
f" {verdict.detail}")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def compare(before: Report, after: Report, *, allow_passing_pair: bool = False) -> tuple[int, str]:
|
|
82
|
+
old, new = evaluate(before), evaluate(after)
|
|
83
|
+
difference = _difference(before, after)
|
|
84
|
+
if difference:
|
|
85
|
+
return 3, f"INCOMPARABLE {difference}"
|
|
86
|
+
old_event, new_event = _event(before, old), _event(after, new)
|
|
87
|
+
decisive = old if old.status is Status.FAIL and new.status is not Status.FAIL else new
|
|
88
|
+
probe = next(item for item in before.rules.probes if item.id == (decisive.probe or before.rules.freshness_probe))
|
|
89
|
+
if old.status is Status.FAIL and new.status is Status.PASS:
|
|
90
|
+
new_event = _matching_proof(after, new, probe, before, old_event)
|
|
91
|
+
headline = ("PASS The affected probe passed at the matching phase after refresh. Write visibility is unverified."
|
|
92
|
+
if new.evidence_tier == "observed-after-refresh" else
|
|
93
|
+
"FIX VERIFIED The affected probe passed at the matching phase.")
|
|
94
|
+
state = (0, headline) if new_event else (
|
|
95
|
+
3, "INCONCLUSIVE The after report lacks a confirmed observation at the affected phase."
|
|
96
|
+
)
|
|
97
|
+
elif new.status is Status.FAIL:
|
|
98
|
+
state = 1, "FAILURE OBSERVED The after report records a failure."
|
|
99
|
+
if old.status is Status.PASS:
|
|
100
|
+
old_event = _matching_proof(before, old, probe, after, new_event)
|
|
101
|
+
if old_event is None:
|
|
102
|
+
state = 1, "FAILURE OBSERVED The before report lacks a confirmed observation at the affected phase."
|
|
103
|
+
elif (old.evidence_tier == "observed-after-refresh" and old_event["stage"] == "final"
|
|
104
|
+
and old_event["view"] is None):
|
|
105
|
+
state = 1, ("FAILURE OBSERVED The before probe passed after refresh. "
|
|
106
|
+
"Its visibility of the completed write is unverified.")
|
|
107
|
+
else:
|
|
108
|
+
state = 1, "REGRESSION The affected probe passed before and failed after at the matching phase."
|
|
109
|
+
elif new.status is Status.ERROR or old.status is Status.ERROR:
|
|
110
|
+
state = 2, "ERROR An execution error leaves the comparison unresolved."
|
|
111
|
+
elif new.status is Status.NOT_TESTED or old.status is Status.NOT_TESTED:
|
|
112
|
+
state = 3, "INCONCLUSIVE An observation required to establish the fix is missing."
|
|
113
|
+
elif allow_passing_pair:
|
|
114
|
+
state = 0, "PASS Both revisions met the same recorded search expectations."
|
|
115
|
+
else:
|
|
116
|
+
state = 3, "INCONCLUSIVE Both reports passed. A failing before run is needed to establish a fix."
|
|
117
|
+
|
|
118
|
+
code, headline = state
|
|
119
|
+
query = before.inputs["questions"][probe.id][:12]
|
|
120
|
+
return code, "\n".join([
|
|
121
|
+
headline,
|
|
122
|
+
f"Probe {probe.id} · query sha256 {query} · route {probe.route} · caller {probe.scope} · top {probe.top_k}",
|
|
123
|
+
_describe("Before", old, old_event),
|
|
124
|
+
_describe("After", new, new_event),
|
|
125
|
+
f"Runs {before.run_id} → {after.run_id}",
|
|
126
|
+
])
|
setrya/conformance.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Live conformance checks for adapter authors using disposable applications."""
|
|
2
|
+
from collections.abc import Callable
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import math
|
|
5
|
+
from tempfile import TemporaryDirectory
|
|
6
|
+
from time import monotonic, sleep
|
|
7
|
+
|
|
8
|
+
from .adapter import Checkpoint, Publication, Receipt
|
|
9
|
+
from .checker import Status, evaluate
|
|
10
|
+
from .drill import Drill, label
|
|
11
|
+
from .evidence import Report, matches_search
|
|
12
|
+
from .runner import run
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ContractError(AssertionError):
|
|
16
|
+
"""An adapter's observed behavior did not meet the selected contract."""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _require(ok, message):
|
|
20
|
+
if not ok:
|
|
21
|
+
raise ContractError(message)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _terminal(report):
|
|
25
|
+
events = {e["kind"]: e for e in report.events if e["kind"] in ("started", "held", "released", "receipt")}
|
|
26
|
+
_require(len(events) == 4, "expected a started, held and released attempt with a terminal receipt")
|
|
27
|
+
start, hold, release, receipt = (events[name] for name in ("started", "held", "released", "receipt"))
|
|
28
|
+
for event in (hold, receipt):
|
|
29
|
+
_require((event["run_id"], event["attempt"], event["document"], event["revision"])
|
|
30
|
+
== (report.run_id, start["attempt"], report.rules.document, report.rules.old_revision),
|
|
31
|
+
"checkpoint and receipt must identify the actual captured input and attempt")
|
|
32
|
+
_require(hold["name"] == report.rules.checkpoint and release["attempt"] == start["attempt"]
|
|
33
|
+
and receipt["ns"] - release["ns"] <= report.rules.settle_ms * 1_000_000,
|
|
34
|
+
"expected the declared checkpoint, release and timely terminal write")
|
|
35
|
+
return receipt
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def assert_store_contract(*, broken: Drill, guarded: Drill, opaque: Drill,
|
|
39
|
+
output: str | Path | None = None, expected_tier: str = "write-confirmed"):
|
|
40
|
+
"""Run three fixtures for the same drill and replay the evidence after teardown.
|
|
41
|
+
|
|
42
|
+
The broken worker must expose a forbidden passage after committing. The
|
|
43
|
+
opaque worker must commit and return correct final content without evidence
|
|
44
|
+
of write visibility. Supply a new directory to retain the three journals.
|
|
45
|
+
"""
|
|
46
|
+
_require(expected_tier in ("write-confirmed", "observed-after-refresh"), "unknown passing evidence tier")
|
|
47
|
+
for case in (broken, guarded, opaque):
|
|
48
|
+
case.validate()
|
|
49
|
+
_require(case.factory and (case.factory, case.probes, case.rules) == (guarded.factory, guarded.probes, guarded.rules),
|
|
50
|
+
"contract cases must use one adapter, rule set, caller and query set")
|
|
51
|
+
with TemporaryDirectory(prefix="setrya-contract-") as scratch:
|
|
52
|
+
out = Path(scratch)
|
|
53
|
+
if output is not None:
|
|
54
|
+
out = Path(output)
|
|
55
|
+
out.mkdir(parents=True, exist_ok=False)
|
|
56
|
+
reports = {}
|
|
57
|
+
for name, case in (("broken", broken), ("guarded", guarded), ("opaque", opaque)):
|
|
58
|
+
live = run(case)
|
|
59
|
+
path = out / (name + ".json")
|
|
60
|
+
live.report.save(path)
|
|
61
|
+
report = Report.load(path)
|
|
62
|
+
verdict = evaluate(report)
|
|
63
|
+
_require(verdict == live.verdict, f"{name}: saved evidence must reproduce the live verdict")
|
|
64
|
+
_require(not any(e["kind"] == "adapter_error" for e in report.events), f"{name}: adapter execution error")
|
|
65
|
+
receipt = _terminal(report)
|
|
66
|
+
if name == "guarded":
|
|
67
|
+
_require((verdict.status, verdict.evidence_tier) == (Status.PASS, expected_tier),
|
|
68
|
+
f"guarded: expected PASS with {expected_tier}; observed {verdict.status.label} ({verdict.detail})")
|
|
69
|
+
elif name == "broken":
|
|
70
|
+
_require(verdict.status is Status.FAIL and verdict.event is not None,
|
|
71
|
+
f"broken: expected an observed failure; got {verdict.status.label}")
|
|
72
|
+
probe = next((p for p in case.rules.probes if p.id == verdict.probe), None)
|
|
73
|
+
decisive = report.events[verdict.event - 1]
|
|
74
|
+
_require(probe is not None and decisive.get("stage") == "final"
|
|
75
|
+
and set(probe.forbid).intersection(decisive["passages"]),
|
|
76
|
+
"broken: the decisive forbidden response must occur after worker release")
|
|
77
|
+
_require(receipt["outcome"] == "committed" and any(
|
|
78
|
+
e.get("stage") == "final" and e["probe_id"] == probe.id and matches_search(e, probe)
|
|
79
|
+
and e["started_ns"] >= receipt["ns"] and set(probe.forbid).intersection(e["passages"])
|
|
80
|
+
for e in report.events),
|
|
81
|
+
"broken: the tested caller must retrieve a forbidden passage after the committed receipt")
|
|
82
|
+
else:
|
|
83
|
+
_require(verdict.status is Status.NOT_TESTED and receipt["outcome"] == "committed"
|
|
84
|
+
and verdict.detail == "Missing search visibility evidence for the old write",
|
|
85
|
+
f"opaque: expected missing read visibility after a commit; got {verdict.status.label} ({verdict.detail})")
|
|
86
|
+
for probe in case.rules.probes:
|
|
87
|
+
reads = [e for e in report.events if e.get("stage") == "final" and e["probe_id"] == probe.id]
|
|
88
|
+
last = reads[-1] if reads else None
|
|
89
|
+
_require(last and matches_search(last, probe) and last["started_ns"] >= receipt["ns"]
|
|
90
|
+
and None not in last["passages"] and set(probe.require) <= set(last["passages"])
|
|
91
|
+
and set(probe.forbid).isdisjoint(last["passages"]),
|
|
92
|
+
f"opaque: missing a clean post-write response for {probe.id}")
|
|
93
|
+
reports[name] = report
|
|
94
|
+
return reports
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _poll(read, timeout, description):
|
|
98
|
+
deadline = monotonic() + timeout
|
|
99
|
+
while monotonic() < deadline:
|
|
100
|
+
value = read()
|
|
101
|
+
if value is not None and monotonic() <= deadline:
|
|
102
|
+
return value
|
|
103
|
+
sleep(min(0.01, max(0, deadline - monotonic())))
|
|
104
|
+
raise ContractError(f"timed out waiting for {description}")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def assert_queue_contract(*, dispatch: Callable[[], str], checkpoint: Callable, publish: Callable,
|
|
108
|
+
release: Callable, receipt: Callable, change: Publication, old_revision: str,
|
|
109
|
+
checkpoint_name: str, outcome: str, timeout: float = 10) -> Receipt:
|
|
110
|
+
"""Exercise a native task in a caller-owned disposable worker fixture.
|
|
111
|
+
|
|
112
|
+
Observation callbacks must return promptly, or None while waiting. The test
|
|
113
|
+
owns shutdown, including teardown following a failed assertion.
|
|
114
|
+
"""
|
|
115
|
+
_require(isinstance(timeout, (int, float)) and not isinstance(timeout, bool)
|
|
116
|
+
and math.isfinite(timeout) and timeout > 0, "timeout must be positive and finite")
|
|
117
|
+
_require(isinstance(change, Publication), "expected the application's declared publication")
|
|
118
|
+
for value, field in ((change.run_id, "run ID"), (change.document, "document"),
|
|
119
|
+
(change.revision, "published revision"), (old_revision, "captured revision"),
|
|
120
|
+
(checkpoint_name, "checkpoint name")):
|
|
121
|
+
label(value, field)
|
|
122
|
+
_require(outcome in ("committed", "rejected") and old_revision != change.revision,
|
|
123
|
+
"select the expected terminal outcome and distinct change revision")
|
|
124
|
+
task_id = label(dispatch(), "native task ID")
|
|
125
|
+
expected = Checkpoint(change.run_id, task_id, change.document, old_revision, checkpoint_name)
|
|
126
|
+
_require(_poll(checkpoint, timeout, "capture") == expected, "worker captured a different input or task")
|
|
127
|
+
_require(receipt() is None, "worker completed before the test published the change")
|
|
128
|
+
_require(publish() == change, "application did not acknowledge the expected change")
|
|
129
|
+
_require(checkpoint() == expected and receipt() is None, "worker did not stay held across the change")
|
|
130
|
+
release()
|
|
131
|
+
terminal = _poll(receipt, timeout, "terminal receipt")
|
|
132
|
+
_require(isinstance(terminal, Receipt) and (terminal.run_id, terminal.attempt, terminal.document,
|
|
133
|
+
terminal.revision, terminal.outcome) == (change.run_id, task_id, change.document, old_revision, outcome),
|
|
134
|
+
"expected the released task's own terminal receipt")
|
|
135
|
+
if outcome == "rejected":
|
|
136
|
+
_require(terminal.generation is None and terminal.commit is None, "a rejection cannot claim a committed write")
|
|
137
|
+
else:
|
|
138
|
+
label(terminal.generation, "native store generation")
|
|
139
|
+
_require(terminal.commit is None or type(terminal.commit) is int and 0 <= terminal.commit < 2**64,
|
|
140
|
+
"committed receipt has an invalid position")
|
|
141
|
+
return terminal
|