setrya 0.12.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. setrya/__init__.py +3 -0
  2. setrya/__main__.py +3 -0
  3. setrya/adapter.py +84 -0
  4. setrya/cache_metrics.py +137 -0
  5. setrya/checker.py +202 -0
  6. setrya/cli.py +348 -0
  7. setrya/comparison.py +126 -0
  8. setrya/conformance.py +141 -0
  9. setrya/coverage.py +311 -0
  10. setrya/dependencies.py +105 -0
  11. setrya/drill.py +188 -0
  12. setrya/evidence.py +367 -0
  13. setrya/examples/__init__.py +1 -0
  14. setrya/examples/answer_cache/__init__.py +1 -0
  15. setrya/examples/answer_cache/__main__.py +3 -0
  16. setrya/examples/answer_cache/app.py +167 -0
  17. setrya/examples/answer_cache/contract.py +19 -0
  18. setrya/examples/answer_cache/drill.py +150 -0
  19. setrya/examples/answer_cache/evidence.py +274 -0
  20. setrya/examples/answer_cache/metrics.py +114 -0
  21. setrya/examples/answer_cache/selective.py +325 -0
  22. setrya/examples/answer_cache/store.py +238 -0
  23. setrya/examples/answer_cache/worker.py +53 -0
  24. setrya/examples/elasticsearch_search.py +106 -0
  25. setrya/examples/offboarding/__init__.py +1 -0
  26. setrya/examples/offboarding/demo.py +105 -0
  27. setrya/examples/offboarding/fixture.py +376 -0
  28. setrya/examples/offboarding/sqlite/__init__.py +1 -0
  29. setrya/examples/offboarding/sqlite/observe.py +344 -0
  30. setrya/examples/offboarding/sqlite/policy.json +8 -0
  31. setrya/examples/opendesk/Dockerfile.trial +26 -0
  32. setrya/examples/opendesk/__init__.py +1 -0
  33. setrya/examples/opendesk/cli.py +73 -0
  34. setrya/examples/opendesk/compose.yaml +70 -0
  35. setrya/examples/opendesk/observed.json +66 -0
  36. setrya/examples/opendesk/policy-v2.json +26 -0
  37. setrya/examples/opendesk/policy.json +26 -0
  38. setrya/examples/opendesk/provider.py +53 -0
  39. setrya/examples/opendesk/run.py +441 -0
  40. setrya/examples/opendesk/to_gate.py +129 -0
  41. setrya/examples/opendesk/transport.py +31 -0
  42. setrya/examples/opendesk/verify_report.py +254 -0
  43. setrya/examples/pgvector-search/compose.yml +14 -0
  44. setrya/examples/pgvector-search/drill.toml +40 -0
  45. setrya/examples/pgvector-search/reupload.toml +51 -0
  46. setrya/examples/pgvector_search/__init__.py +8 -0
  47. setrya/examples/pgvector_search/__main__.py +26 -0
  48. setrya/examples/pgvector_search/adapter.py +16 -0
  49. setrya/examples/pgvector_search/app.py +242 -0
  50. setrya/examples/pgvector_search/benchmark.py +118 -0
  51. setrya/examples/pgvector_search/pytest_reference.py +89 -0
  52. setrya/examples/recorded-search/drill.toml +34 -0
  53. setrya/examples/recorded_search/__init__.py +1 -0
  54. setrya/examples/recorded_search/existing_test.py +62 -0
  55. setrya/examples/recorded_search/verify.py +93 -0
  56. setrya/examples/support-search/drill.toml +40 -0
  57. setrya/examples/support-search/python.toml +40 -0
  58. setrya/examples/support-search/retired.toml +51 -0
  59. setrya/examples/support_search/__init__.py +1 -0
  60. setrya/examples/support_search/__main__.py +104 -0
  61. setrya/examples/support_search/adapter.py +61 -0
  62. setrya/examples/support_search/app.py +225 -0
  63. setrya/examples/support_search/baseline.py +101 -0
  64. setrya/examples/support_search/client.py +78 -0
  65. setrya/git_refs.py +23 -0
  66. setrya/integrations/__init__.py +1 -0
  67. setrya/integrations/celery.py +97 -0
  68. setrya/integrations/langchain.py +83 -0
  69. setrya/local_cache.py +143 -0
  70. setrya/offboarding/__init__.py +1 -0
  71. setrya/offboarding/cli.py +74 -0
  72. setrya/offboarding/gate.py +202 -0
  73. setrya/offboarding/inputs.py +86 -0
  74. setrya/offboarding/observer.py +47 -0
  75. setrya/offboarding/receipt-v1.schema.json +300 -0
  76. setrya/offboarding/receipt.py +264 -0
  77. setrya/offboarding/receipt_cli.py +197 -0
  78. setrya/offboarding/receipt_schema.py +65 -0
  79. setrya/offboarding/reference.py +243 -0
  80. setrya/offboarding/signature.py +68 -0
  81. setrya/pytest_plugin.py +82 -0
  82. setrya/qualification.py +412 -0
  83. setrya/recorder.py +163 -0
  84. setrya/review.py +119 -0
  85. setrya/review_scenarios.py +157 -0
  86. setrya/run_pr.py +168 -0
  87. setrya/runner.py +184 -0
  88. setrya/scenario_fixture.py +127 -0
  89. setrya/scenario_order.py +268 -0
  90. setrya/scenarios.py +509 -0
  91. setrya/status.py +22 -0
  92. setrya/store_signals.py +56 -0
  93. setrya-0.12.1.dist-info/METADATA +221 -0
  94. setrya-0.12.1.dist-info/RECORD +98 -0
  95. setrya-0.12.1.dist-info/WHEEL +5 -0
  96. setrya-0.12.1.dist-info/entry_points.txt +2 -0
  97. setrya-0.12.1.dist-info/licenses/LICENSE +202 -0
  98. setrya-0.12.1.dist-info/top_level.txt +1 -0
setrya/cli.py ADDED
@@ -0,0 +1,348 @@
1
+ """Command-line entry point for check, replay, demo and qualification."""
2
+
3
+ import argparse
4
+ from contextlib import closing
5
+ from dataclasses import replace
6
+ import os
7
+ from pathlib import Path
8
+ import re
9
+ import shlex
10
+ import sys
11
+ from tempfile import NamedTemporaryFile
12
+
13
+ from . import __version__
14
+ from .checker import Status, Verdict, evaluate
15
+ from .drill import Drill, InputError
16
+ from .evidence import Report, bundle_manifest, matches_search
17
+ from .runner import RunResult, run
18
+
19
+ ROOT = Path(__file__).resolve().parent.parent
20
+ EXAMPLE = ROOT / "setrya" / "examples" / "support-search"
21
+ PGVECTOR = ROOT / "setrya" / "examples" / "pgvector-search"
22
+ DEMO_CASES = {"broken": Status.FAIL, "fixed": Status.PASS, "opaque": Status.NOT_TESTED}
23
+ DEMO_PASSAGES = {"refund@A": "30 days", "refund@B": "14 days"}
24
+
25
+ def preflight() -> None:
26
+ try:
27
+ import sqlite3
28
+ except ImportError:
29
+ raise InputError("The SQLite fixture needs a Python build with SQLite FTS5 enabled.") from None
30
+ try:
31
+ with closing(sqlite3.connect(":memory:")) as connection:
32
+ connection.execute("CREATE VIRTUAL TABLE preflight USING fts5(body)")
33
+ except sqlite3.Error:
34
+ raise InputError("SQLite FTS5 is unavailable. Use a Python build with FTS5 enabled.") from None
35
+
36
+
37
+ def format_verdict(
38
+ verdict: Verdict, report: Report, rerun: str | None = None, replay: str | None = None
39
+ ) -> str:
40
+ lines = [f"{verdict.status.label} {verdict.detail}"]
41
+ if verdict.probe:
42
+ rule = next(probe for probe in report.rules.probes if probe.id == verdict.probe)
43
+ lines.append(f" Probe {rule.id} · route {rule.route} · scope {rule.scope} · top {rule.top_k}")
44
+ if verdict.event:
45
+ event = report.events[verdict.event - 1]
46
+ lines.append(f" Event {verdict.event} at {event['ns'] / 1_000_000:.1f} ms")
47
+ if event["kind"] == "probe":
48
+ lines.append(f" Observed {passage_list(event)}")
49
+ publication = next((event.get("publication") for event in report.events if event["kind"] == "published"), None)
50
+ if publication:
51
+ lines.append(f" Published {publication['document']} revision {publication['revision']}")
52
+ if verdict.status is Status.NOT_TESTED:
53
+ latest = {event["probe_id"]: event for event in report.events if event["kind"] == "probe"}
54
+ for event in latest.values():
55
+ where = f"{event.get('route') or '?'} / {event.get('scope') or '?'}"
56
+ lines.append(f" Last {event['stage']} {event['probe_id']} ({where}) → {passage_list(event)}")
57
+ if verdict.attribution:
58
+ lines.append(f" {verdict.attribution}")
59
+ for event in report.events:
60
+ if event["kind"] == "adapter_error":
61
+ operation = event["operation"]
62
+ detail = "Test error" if operation == "test" else f"Adapter error during {operation}"
63
+ lines.append(f" {detail} ({event['error']})")
64
+ lines.append(f" Run {report.run_id}")
65
+ if replay:
66
+ lines.append(f" Replay {replay}")
67
+ if rerun:
68
+ lines.append(f" Rerun {rerun}")
69
+ return "\n".join(lines)
70
+
71
+
72
+ def passage_list(event: dict) -> str:
73
+ return ", ".join(marker if marker is not None else "?" for marker in event["passages"]) or "(empty)"
74
+
75
+
76
+ def history(report: Report) -> str:
77
+ lines = [" History"]
78
+ for event in report.events:
79
+ if event["kind"] == "probe":
80
+ where = f"{event.get('route') or '?'} / {event.get('scope') or '?'}"
81
+ lines.append(f" {event['seq']:>3} {event['stage']} {event['probe_id']} ({where}) → {passage_list(event)}")
82
+ elif event["kind"] == "receipt":
83
+ lines.append(f" {event['seq']:>3} old write {event['outcome']}")
84
+ elif event["kind"] == "published" and not event.get("publication"):
85
+ lines.append(f" {event['seq']:>3} publication attempted")
86
+ elif event["kind"] == "adapter_error":
87
+ lines.append(f" {event['seq']:>3} adapter error during {event['operation']}")
88
+ else:
89
+ lines.append(f" {event['seq']:>3} {event['kind']}")
90
+ return "\n".join(lines)
91
+
92
+
93
+ def demo_case(mode: str) -> RunResult:
94
+ drill = Drill.load(EXAMPLE / "python.toml")
95
+ return run(replace(drill, options={"mode": mode}))
96
+
97
+
98
+ def demo(selected: str | None) -> int:
99
+ ok = True
100
+ for mode in (selected,) if selected else DEMO_CASES:
101
+ result = demo_case(mode)
102
+ verdict = result.verdict
103
+ ok &= verdict.status is DEMO_CASES[mode]
104
+ print(f"{verdict.status.label:<11} {verdict.detail}")
105
+ primary = next(probe for probe in result.report.rules.probes
106
+ if probe.id == result.report.rules.freshness_probe)
107
+ observed = []
108
+ for event in result.report.events:
109
+ if event.get("probe_id") == primary.id and matches_search(event, primary):
110
+ result_text = ", ".join(DEMO_PASSAGES.get(marker, marker or "?") for marker in event["passages"])
111
+ result_text = result_text or "(empty)"
112
+ if not observed or observed[-1] != result_text:
113
+ observed.append(result_text)
114
+ if observed:
115
+ print(f" Refund policy in search · {' → '.join(observed)}")
116
+ for event in result.report.events:
117
+ if event["kind"] == "held":
118
+ print(f" Checkpoint held revision {event['revision']}")
119
+ elif event["kind"] == "released":
120
+ print(" Old worker released")
121
+ elif event["kind"] == "receipt":
122
+ print(f" Old write {event['outcome']}")
123
+ elif event["kind"] == "adapter_error":
124
+ print(f" Adapter error during {event['operation']} ({event['error']})")
125
+ print(f" Rerun python3 -m setrya demo --case {mode}")
126
+ return 0 if ok else 1
127
+
128
+
129
+ def rerun_command(path: str) -> str:
130
+ # Reports and summaries avoid embedding private absolute paths or file names.
131
+ if re.fullmatch(r"setrya/examples/(?:support-search/(?:drill|python|retired)|pgvector-search/(?:drill|reupload))\.toml", path):
132
+ return "python3 -m setrya check " + shlex.quote(path)
133
+ return 'python3 -m setrya check "$DRILL" --report run.json'
134
+
135
+
136
+ def replay_command(report_path: str | None, *, artifact: str = "run.json", variable: str = "REPORT") -> str:
137
+ if report_path is None:
138
+ return "Save with --report run.json, then run setrya replay run.json"
139
+ target = artifact if Path(report_path).name == artifact else f'"${variable}"'
140
+ return f"setrya replay {target}"
141
+
142
+
143
+ def console_main() -> int:
144
+ args = sys.argv[1:]
145
+ live_scenario = args[:1] == ["scenario"] and args[1:2] in (["update"], ["delete"], ["revoke"])
146
+ if args[:1] != ["check"] and args[:2] != ["offboarding", "drill"] and not live_scenario:
147
+ return main()
148
+ # Only live checks import adapters from the working checkout.
149
+ original_path = sys.path
150
+ try:
151
+ sys.path = [str(Path.cwd()), *original_path]
152
+ return main()
153
+ finally:
154
+ sys.path = original_path
155
+
156
+
157
+ def _write_summary(path: str | Path, title: str, text: str) -> None:
158
+ target = Path(path).absolute()
159
+ temporary = None
160
+ try:
161
+ if target.is_symlink() or (target.exists() and not target.is_file()):
162
+ raise InputError("summary destination must be a regular file")
163
+ with NamedTemporaryFile("w", dir=target.parent, prefix=".setrya-", encoding="utf-8", delete=False) as stream:
164
+ temporary = Path(stream.name)
165
+ stream.write(f"### Setrya · {title}\n\n```text\n{text}\n```\n")
166
+ temporary.replace(target)
167
+ except OSError:
168
+ raise InputError("could not write summary. Check the destination directory.") from None
169
+ finally:
170
+ if temporary is not None:
171
+ temporary.unlink(missing_ok=True)
172
+
173
+
174
+ def main(argv=None) -> int:
175
+ argv = sys.argv[1:] if argv is None else list(argv)
176
+ if argv[:1] == ["cache-metrics"]:
177
+ from .examples.answer_cache.metrics import main as metrics_main
178
+
179
+ return metrics_main(argv[1:])
180
+ if argv[:1] == ["answer-cache"]:
181
+ from .examples.answer_cache.drill import main as answer_cache_main
182
+
183
+ return answer_cache_main(argv[1:])
184
+ if argv[:1] == ["coverage"]:
185
+ from .coverage import main as coverage_main
186
+
187
+ return coverage_main(argv[1:])
188
+ if argv[:1] == ["run-pr"]:
189
+ from .run_pr import main as run_pr
190
+
191
+ return run_pr(argv[1:])
192
+ if argv[:1] == ["scenario"]:
193
+ from .scenarios import main as scenario_main
194
+
195
+ return scenario_main(argv[1:])
196
+ if argv[:1] == ["offboarding"]:
197
+ from .offboarding.cli import main as offboarding_main
198
+
199
+ return offboarding_main(argv[1:])
200
+ parser = argparse.ArgumentParser(prog="setrya", description="Test what an application can retrieve after a document update or tenant offboarding.")
201
+ parser.add_argument("--version", action="version", version=__version__)
202
+ commands = parser.add_subparsers(dest="command", required=True)
203
+ commands.add_parser("scenario", help="drive update, deletion or access-revocation races and replay a schedule")
204
+ commands.add_parser("offboarding", help="check tenant offboarding in CI or review one deletion request")
205
+ commands.add_parser("cache-metrics", help="measure checked answers and cache outcomes from local evidence")
206
+ commands.add_parser("answer-cache", help="test a late Celery cache fill against a real answer endpoint")
207
+ commands.add_parser("coverage", help="find changed routes and jobs without matching candidate evidence")
208
+ commands.add_parser("run-pr", help="run an existing test on two Git revisions and review its evidence")
209
+ demo_parser = commands.add_parser("demo", help="run three synthetic outcomes")
210
+ demo_parser.add_argument("--case", choices=DEMO_CASES)
211
+ check = commands.add_parser("check", help="run a drill")
212
+ check.add_argument("drill")
213
+ check.add_argument("--report", help="write a portable JSON report")
214
+ check.add_argument("--summary", help="write a Markdown summary")
215
+ check.add_argument("--revision", help="record the full Git commit ID supplying the application")
216
+ replay = commands.add_parser("replay", help="recompute a verdict from saved observations")
217
+ replay.add_argument("report")
218
+ diagnose = commands.add_parser("diagnose", help="show the verdict and the first missing observation")
219
+ diagnose.add_argument("report")
220
+ comparison = commands.add_parser("compare", help="tell whether a before-and-after pair establishes a fix")
221
+ comparison.add_argument("before")
222
+ comparison.add_argument("after")
223
+ comparison.add_argument("--summary", help="write a Markdown summary")
224
+ review_parser = commands.add_parser("review", help="review two CI reports or two scenario directories")
225
+ review_parser.add_argument("before")
226
+ review_parser.add_argument("after")
227
+ review_parser.add_argument("--base-commit", help="expected full base Git commit ID (required)")
228
+ review_parser.add_argument("--candidate-commit", help="expected full candidate Git commit ID (required)")
229
+ review_parser.add_argument("--base-exit", help="native base test exit (required)")
230
+ review_parser.add_argument("--candidate-exit", help="native candidate test exit (required)")
231
+ review_parser.add_argument("--exit-contract", choices=("pytest", "setrya"),
232
+ help="native exits (default: pytest for files, setrya for scenario directories)")
233
+ review_parser.add_argument("--summary", help="write a Markdown summary")
234
+ qualify = commands.add_parser("qualify", help="run the shipped qualification cases")
235
+ qualify.add_argument("--case")
236
+ qualify.add_argument("--quick", action="store_true", help="skip the 90-run repeatability case")
237
+ args = parser.parse_args(argv)
238
+ error_summary = None
239
+
240
+ try:
241
+ if args.command in {"replay", "diagnose"}:
242
+ directory = Path(args.report)
243
+ manifest = bundle_manifest(directory) if directory.is_dir() else None
244
+ if manifest == "answer-cache.json":
245
+ from .examples.answer_cache.drill import main as answer_cache_main
246
+
247
+ return answer_cache_main(["--replay", args.report])
248
+ if manifest == "scenario.json" or (directory.is_dir() and args.command == "replay"):
249
+ from .scenarios import main as scenario_main
250
+
251
+ return scenario_main(["replay", args.report])
252
+ report = Report.load(args.report)
253
+ verdict = evaluate(report)
254
+ print(format_verdict(verdict, report))
255
+ if args.command == "replay":
256
+ print(history(report))
257
+ return verdict.status.exit_code
258
+ if args.command in {"compare", "review"}:
259
+ from .comparison import compare
260
+
261
+ if args.summary:
262
+ destination = Path(args.summary).resolve()
263
+ lexical = Path(os.path.abspath(args.summary))
264
+ for path in (Path(args.before), Path(args.after)):
265
+ source = path.resolve()
266
+ if destination == source or (path.is_dir() and (
267
+ destination.is_relative_to(source) or lexical.is_relative_to(os.path.abspath(path))
268
+ )):
269
+ raise InputError("comparison summary must use a distinct file outside the evidence directories")
270
+ error_summary = args.summary
271
+ if args.command == "review":
272
+ from .review import ReviewInput, review_inputs
273
+
274
+ base = ReviewInput.load(args.before, args.base_commit, args.base_exit)
275
+ candidate = ReviewInput.load(args.after, args.candidate_commit, args.candidate_exit)
276
+ lanes = (("base", base, args.before), ("candidate", candidate, args.after))
277
+ named_directories = frozenset(name for name, _, path in lanes if Path(path) == Path(name))
278
+ code, summary = review_inputs(base, candidate, named_directories=named_directories,
279
+ exit_contract=args.exit_contract)
280
+ for name, lane, path in lanes:
281
+ if lane.report is not None:
282
+ variable = f"{name.upper()}_REPORT"
283
+ if isinstance(lane.report, Report):
284
+ command = replay_command(path, artifact=f"{name}.json", variable=variable)
285
+ else:
286
+ from .review_scenarios import replay_scenario_command
287
+
288
+ command = replay_scenario_command(name, named_directories)
289
+ if name not in named_directories:
290
+ summary += f"\nSet {variable} to the path of the downloaded {name} scenario directory."
291
+ summary += f"\nReplay {name} {command}"
292
+ title = f"{base.report.rules.id} release review" if isinstance(base.report, Report) else "release review"
293
+ else:
294
+ before, after = Report.load(args.before), Report.load(args.after)
295
+ code, summary = compare(before, after)
296
+ title = f"{before.rules.id} comparison"
297
+ print(summary)
298
+ error_summary = None
299
+ if args.summary:
300
+ _write_summary(args.summary, title, summary)
301
+ return code
302
+ if args.command == "check":
303
+ paths = [Path(path).resolve() for path in (args.drill, args.report, args.summary) if path is not None]
304
+ if len(set(paths)) != len(paths):
305
+ raise InputError("drill, report and summary must use distinct files")
306
+ report_path = Path(args.report).absolute() if args.report else None
307
+ summary_path = Path(args.summary).absolute() if args.summary else None
308
+ error_summary = summary_path
309
+ drill = Drill.load(args.drill)
310
+ if drill.factory in {"setrya.examples.support_search.adapter:http_fixture",
311
+ "setrya.examples.support_search.adapter:python_fixture"}:
312
+ preflight()
313
+ result = run(drill, revision=args.revision)
314
+ if report_path:
315
+ result.report.save(report_path)
316
+ replay_hint = replay_command(args.report)
317
+ summary = format_verdict(result.verdict, result.report, rerun_command(args.drill), replay_hint)
318
+ print(summary)
319
+ interrupted = next((e for e in result.report.events if e["kind"] == "adapter_error"
320
+ and e["error"] == "KeyboardInterrupt"), None)
321
+ if interrupted is not None:
322
+ print(f"Interrupted during {interrupted['operation']}. {replay_hint}", file=sys.stderr)
323
+ error_summary = None
324
+ if summary_path:
325
+ _write_summary(summary_path, result.report.rules.id, summary)
326
+ return result.verdict.status.exit_code
327
+ if args.command == "demo":
328
+ preflight()
329
+ return demo(args.case)
330
+ if args.case not in {"pgvector", "offboarding", "offboarding-compatibility"}:
331
+ preflight()
332
+ from .qualification import qualify as run_qualification
333
+
334
+ return run_qualification(args.case, args.quick)
335
+ except InputError as error:
336
+ diagnostic = f"ERROR {error}"
337
+ except KeyboardInterrupt:
338
+ diagnostic = "ERROR Interrupted. Fixture teardown requested."
339
+ except Exception as error:
340
+ # Third-party errors may contain queries, snippets and local URLs.
341
+ diagnostic = f"ERROR {type(error).__name__} during {args.command}. Check adapter health and local permissions."
342
+ print(diagnostic, file=sys.stderr)
343
+ if error_summary:
344
+ try:
345
+ _write_summary(error_summary, f"{args.command} error", diagnostic)
346
+ except InputError as error:
347
+ print(f"ERROR {error}", file=sys.stderr)
348
+ return 2
setrya/comparison.py ADDED
@@ -0,0 +1,126 @@
1
+ """Compare two saved drills without contacting either application."""
2
+
3
+ from .checker import Status, Verdict, evaluate
4
+ from .drill import ProbeRules
5
+ from .evidence import Report, matches_search, observed_change
6
+
7
+
8
+ def _difference(before: Report, after: Report) -> str | None:
9
+ if before.inputs is None or after.inputs is None:
10
+ return "Both runs need schema 3 drill input fingerprints. Run both checks again."
11
+ if before.run_id == after.run_id:
12
+ return "Supply reports from two distinct runs."
13
+ if before.inputs["drill_version"] != after.inputs["drill_version"]:
14
+ return "Drill input versions differ."
15
+ if before.inputs["adapter"] != after.inputs["adapter"]:
16
+ return "The adapter factory differs."
17
+ earlier = {probe.id: probe for probe in before.rules.probes}
18
+ later = {probe.id: probe for probe in after.rules.probes}
19
+ if earlier.keys() != later.keys():
20
+ return "The reports cover different probes."
21
+ for probe_id, probe in earlier.items():
22
+ changed = later[probe_id]
23
+ if before.inputs["questions"][probe_id] != after.inputs["questions"][probe_id]:
24
+ return f"Question changed for {probe_id}."
25
+ if probe.top_k != changed.top_k:
26
+ return f"Search depth changed for {probe_id} from {probe.top_k} to {changed.top_k}."
27
+ if (probe.route, probe.scope) != (changed.route, changed.scope):
28
+ return f"Route or caller changed for {probe_id}."
29
+ if probe != changed:
30
+ return f"Passage expectations changed for {probe_id}."
31
+ if before.rules != after.rules:
32
+ return "Revision, schedule, drill identity, or probe order changed."
33
+ return None
34
+
35
+
36
+ def _event(report: Report, verdict: Verdict) -> dict | None:
37
+ if verdict.event is not None:
38
+ return report.events[verdict.event - 1]
39
+ return None
40
+
41
+
42
+ def _first_change(report: Report, primary: ProbeRules) -> dict | None:
43
+ return next((event for event in report.events if event.get("stage") == "freshness"
44
+ and event["probe_id"] == primary.id and matches_search(event, primary)
45
+ and observed_change(event, primary)), None)
46
+
47
+
48
+ def _matching_proof(report: Report, verdict: Verdict, probe: ProbeRules,
49
+ failed_report: Report, failed: dict) -> dict | None:
50
+ if verdict.status is not Status.PASS:
51
+ return None
52
+ if failed["stage"] == "final":
53
+ return next((report.events[seq - 1] for seq in verdict.confirmed
54
+ if report.events[seq - 1]["probe_id"] == probe.id), None)
55
+
56
+ primary = next(item for item in report.rules.probes if item.id == report.rules.freshness_probe)
57
+ change = _first_change(report, primary)
58
+ if change is None:
59
+ return None
60
+ failed_change = _first_change(failed_report, primary)
61
+ # A recurrence on a second query needs a second query on the passing lane.
62
+ # A failure on the transition itself can be compared to the transition.
63
+ minimum = change["seq"]
64
+ if failed_change and failed["seq"] > failed_change["seq"]:
65
+ minimum += 1
66
+ return next((event for event in report.events
67
+ if event.get("stage") == "freshness" and event["probe_id"] == probe.id
68
+ and event["seq"] >= minimum and matches_search(event, probe)
69
+ and None not in event["passages"] and set(probe.require) <= set(event["passages"])
70
+ and not set(probe.forbid).intersection(event["passages"])), None)
71
+
72
+
73
+ def _describe(label: str, verdict: Verdict, event: dict | None) -> str:
74
+ if event is None or event["kind"] != "probe":
75
+ return f"{label} {verdict.status.label} · {verdict.detail}"
76
+ passages = ", ".join(marker or "?" for marker in event["passages"]) or "(empty)"
77
+ return (f"{label} {verdict.status.label} · event {event['seq']} · {event['probe_id']} · {event['stage']} · {passages}\n"
78
+ f" {verdict.detail}")
79
+
80
+
81
+ def compare(before: Report, after: Report, *, allow_passing_pair: bool = False) -> tuple[int, str]:
82
+ old, new = evaluate(before), evaluate(after)
83
+ difference = _difference(before, after)
84
+ if difference:
85
+ return 3, f"INCOMPARABLE {difference}"
86
+ old_event, new_event = _event(before, old), _event(after, new)
87
+ decisive = old if old.status is Status.FAIL and new.status is not Status.FAIL else new
88
+ probe = next(item for item in before.rules.probes if item.id == (decisive.probe or before.rules.freshness_probe))
89
+ if old.status is Status.FAIL and new.status is Status.PASS:
90
+ new_event = _matching_proof(after, new, probe, before, old_event)
91
+ headline = ("PASS The affected probe passed at the matching phase after refresh. Write visibility is unverified."
92
+ if new.evidence_tier == "observed-after-refresh" else
93
+ "FIX VERIFIED The affected probe passed at the matching phase.")
94
+ state = (0, headline) if new_event else (
95
+ 3, "INCONCLUSIVE The after report lacks a confirmed observation at the affected phase."
96
+ )
97
+ elif new.status is Status.FAIL:
98
+ state = 1, "FAILURE OBSERVED The after report records a failure."
99
+ if old.status is Status.PASS:
100
+ old_event = _matching_proof(before, old, probe, after, new_event)
101
+ if old_event is None:
102
+ state = 1, "FAILURE OBSERVED The before report lacks a confirmed observation at the affected phase."
103
+ elif (old.evidence_tier == "observed-after-refresh" and old_event["stage"] == "final"
104
+ and old_event["view"] is None):
105
+ state = 1, ("FAILURE OBSERVED The before probe passed after refresh. "
106
+ "Its visibility of the completed write is unverified.")
107
+ else:
108
+ state = 1, "REGRESSION The affected probe passed before and failed after at the matching phase."
109
+ elif new.status is Status.ERROR or old.status is Status.ERROR:
110
+ state = 2, "ERROR An execution error leaves the comparison unresolved."
111
+ elif new.status is Status.NOT_TESTED or old.status is Status.NOT_TESTED:
112
+ state = 3, "INCONCLUSIVE An observation required to establish the fix is missing."
113
+ elif allow_passing_pair:
114
+ state = 0, "PASS Both revisions met the same recorded search expectations."
115
+ else:
116
+ state = 3, "INCONCLUSIVE Both reports passed. A failing before run is needed to establish a fix."
117
+
118
+ code, headline = state
119
+ query = before.inputs["questions"][probe.id][:12]
120
+ return code, "\n".join([
121
+ headline,
122
+ f"Probe {probe.id} · query sha256 {query} · route {probe.route} · caller {probe.scope} · top {probe.top_k}",
123
+ _describe("Before", old, old_event),
124
+ _describe("After", new, new_event),
125
+ f"Runs {before.run_id} → {after.run_id}",
126
+ ])
setrya/conformance.py ADDED
@@ -0,0 +1,141 @@
1
+ """Live conformance checks for adapter authors using disposable applications."""
2
+ from collections.abc import Callable
3
+ from pathlib import Path
4
+ import math
5
+ from tempfile import TemporaryDirectory
6
+ from time import monotonic, sleep
7
+
8
+ from .adapter import Checkpoint, Publication, Receipt
9
+ from .checker import Status, evaluate
10
+ from .drill import Drill, label
11
+ from .evidence import Report, matches_search
12
+ from .runner import run
13
+
14
+
15
+ class ContractError(AssertionError):
16
+ """An adapter's observed behavior did not meet the selected contract."""
17
+
18
+
19
+ def _require(ok, message):
20
+ if not ok:
21
+ raise ContractError(message)
22
+
23
+
24
+ def _terminal(report):
25
+ events = {e["kind"]: e for e in report.events if e["kind"] in ("started", "held", "released", "receipt")}
26
+ _require(len(events) == 4, "expected a started, held and released attempt with a terminal receipt")
27
+ start, hold, release, receipt = (events[name] for name in ("started", "held", "released", "receipt"))
28
+ for event in (hold, receipt):
29
+ _require((event["run_id"], event["attempt"], event["document"], event["revision"])
30
+ == (report.run_id, start["attempt"], report.rules.document, report.rules.old_revision),
31
+ "checkpoint and receipt must identify the actual captured input and attempt")
32
+ _require(hold["name"] == report.rules.checkpoint and release["attempt"] == start["attempt"]
33
+ and receipt["ns"] - release["ns"] <= report.rules.settle_ms * 1_000_000,
34
+ "expected the declared checkpoint, release and timely terminal write")
35
+ return receipt
36
+
37
+
38
+ def assert_store_contract(*, broken: Drill, guarded: Drill, opaque: Drill,
39
+ output: str | Path | None = None, expected_tier: str = "write-confirmed"):
40
+ """Run three fixtures for the same drill and replay the evidence after teardown.
41
+
42
+ The broken worker must expose a forbidden passage after committing. The
43
+ opaque worker must commit and return correct final content without evidence
44
+ of write visibility. Supply a new directory to retain the three journals.
45
+ """
46
+ _require(expected_tier in ("write-confirmed", "observed-after-refresh"), "unknown passing evidence tier")
47
+ for case in (broken, guarded, opaque):
48
+ case.validate()
49
+ _require(case.factory and (case.factory, case.probes, case.rules) == (guarded.factory, guarded.probes, guarded.rules),
50
+ "contract cases must use one adapter, rule set, caller and query set")
51
+ with TemporaryDirectory(prefix="setrya-contract-") as scratch:
52
+ out = Path(scratch)
53
+ if output is not None:
54
+ out = Path(output)
55
+ out.mkdir(parents=True, exist_ok=False)
56
+ reports = {}
57
+ for name, case in (("broken", broken), ("guarded", guarded), ("opaque", opaque)):
58
+ live = run(case)
59
+ path = out / (name + ".json")
60
+ live.report.save(path)
61
+ report = Report.load(path)
62
+ verdict = evaluate(report)
63
+ _require(verdict == live.verdict, f"{name}: saved evidence must reproduce the live verdict")
64
+ _require(not any(e["kind"] == "adapter_error" for e in report.events), f"{name}: adapter execution error")
65
+ receipt = _terminal(report)
66
+ if name == "guarded":
67
+ _require((verdict.status, verdict.evidence_tier) == (Status.PASS, expected_tier),
68
+ f"guarded: expected PASS with {expected_tier}; observed {verdict.status.label} ({verdict.detail})")
69
+ elif name == "broken":
70
+ _require(verdict.status is Status.FAIL and verdict.event is not None,
71
+ f"broken: expected an observed failure; got {verdict.status.label}")
72
+ probe = next((p for p in case.rules.probes if p.id == verdict.probe), None)
73
+ decisive = report.events[verdict.event - 1]
74
+ _require(probe is not None and decisive.get("stage") == "final"
75
+ and set(probe.forbid).intersection(decisive["passages"]),
76
+ "broken: the decisive forbidden response must occur after worker release")
77
+ _require(receipt["outcome"] == "committed" and any(
78
+ e.get("stage") == "final" and e["probe_id"] == probe.id and matches_search(e, probe)
79
+ and e["started_ns"] >= receipt["ns"] and set(probe.forbid).intersection(e["passages"])
80
+ for e in report.events),
81
+ "broken: the tested caller must retrieve a forbidden passage after the committed receipt")
82
+ else:
83
+ _require(verdict.status is Status.NOT_TESTED and receipt["outcome"] == "committed"
84
+ and verdict.detail == "Missing search visibility evidence for the old write",
85
+ f"opaque: expected missing read visibility after a commit; got {verdict.status.label} ({verdict.detail})")
86
+ for probe in case.rules.probes:
87
+ reads = [e for e in report.events if e.get("stage") == "final" and e["probe_id"] == probe.id]
88
+ last = reads[-1] if reads else None
89
+ _require(last and matches_search(last, probe) and last["started_ns"] >= receipt["ns"]
90
+ and None not in last["passages"] and set(probe.require) <= set(last["passages"])
91
+ and set(probe.forbid).isdisjoint(last["passages"]),
92
+ f"opaque: missing a clean post-write response for {probe.id}")
93
+ reports[name] = report
94
+ return reports
95
+
96
+
97
+ def _poll(read, timeout, description):
98
+ deadline = monotonic() + timeout
99
+ while monotonic() < deadline:
100
+ value = read()
101
+ if value is not None and monotonic() <= deadline:
102
+ return value
103
+ sleep(min(0.01, max(0, deadline - monotonic())))
104
+ raise ContractError(f"timed out waiting for {description}")
105
+
106
+
107
+ def assert_queue_contract(*, dispatch: Callable[[], str], checkpoint: Callable, publish: Callable,
108
+ release: Callable, receipt: Callable, change: Publication, old_revision: str,
109
+ checkpoint_name: str, outcome: str, timeout: float = 10) -> Receipt:
110
+ """Exercise a native task in a caller-owned disposable worker fixture.
111
+
112
+ Observation callbacks must return promptly, or None while waiting. The test
113
+ owns shutdown, including teardown following a failed assertion.
114
+ """
115
+ _require(isinstance(timeout, (int, float)) and not isinstance(timeout, bool)
116
+ and math.isfinite(timeout) and timeout > 0, "timeout must be positive and finite")
117
+ _require(isinstance(change, Publication), "expected the application's declared publication")
118
+ for value, field in ((change.run_id, "run ID"), (change.document, "document"),
119
+ (change.revision, "published revision"), (old_revision, "captured revision"),
120
+ (checkpoint_name, "checkpoint name")):
121
+ label(value, field)
122
+ _require(outcome in ("committed", "rejected") and old_revision != change.revision,
123
+ "select the expected terminal outcome and distinct change revision")
124
+ task_id = label(dispatch(), "native task ID")
125
+ expected = Checkpoint(change.run_id, task_id, change.document, old_revision, checkpoint_name)
126
+ _require(_poll(checkpoint, timeout, "capture") == expected, "worker captured a different input or task")
127
+ _require(receipt() is None, "worker completed before the test published the change")
128
+ _require(publish() == change, "application did not acknowledge the expected change")
129
+ _require(checkpoint() == expected and receipt() is None, "worker did not stay held across the change")
130
+ release()
131
+ terminal = _poll(receipt, timeout, "terminal receipt")
132
+ _require(isinstance(terminal, Receipt) and (terminal.run_id, terminal.attempt, terminal.document,
133
+ terminal.revision, terminal.outcome) == (change.run_id, task_id, change.document, old_revision, outcome),
134
+ "expected the released task's own terminal receipt")
135
+ if outcome == "rejected":
136
+ _require(terminal.generation is None and terminal.commit is None, "a rejection cannot claim a committed write")
137
+ else:
138
+ label(terminal.generation, "native store generation")
139
+ _require(terminal.commit is None or type(terminal.commit) is int and 0 <= terminal.commit < 2**64,
140
+ "committed receipt has an invalid position")
141
+ return terminal