earful 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- earful/__init__.py +46 -0
- earful/__main__.py +7 -0
- earful/cli.py +504 -0
- earful/core/__init__.py +47 -0
- earful/core/backend.py +322 -0
- earful/core/bundle.py +257 -0
- earful/core/case.py +237 -0
- earful/core/checker/__init__.py +110 -0
- earful/core/checker/a1.py +80 -0
- earful/core/checker/a2.py +71 -0
- earful/core/checker/a3.py +97 -0
- earful/core/checker/l1.py +335 -0
- earful/core/checker/l2.py +126 -0
- earful/core/checker/m1.py +243 -0
- earful/core/checker/m2.py +225 -0
- earful/core/checker/memory.py +202 -0
- earful/core/checker/policy.py +106 -0
- earful/core/checker/report.py +233 -0
- earful/core/checker/view.py +592 -0
- earful/core/crosscheck.py +443 -0
- earful/core/html.py +2300 -0
- earful/core/ledger.py +183 -0
- earful/core/predicate.py +51 -0
- earful/core/schema.py +391 -0
- earful/core/schemas/earful.bundle-1.schema.json +275 -0
- earful/core/schemas/earful.case-1.schema.json +194 -0
- earful/core/schemas/earful.ledger-1.schema.json +2157 -0
- earful/core/search.py +1069 -0
- earful/core/shrink.py +497 -0
- earful/core/timebase.py +28 -0
- earful/core/timeline.py +56 -0
- earful/livekit/__init__.py +81 -0
- earful/livekit/_compat.py +10 -0
- earful/livekit/api.py +232 -0
- earful/livekit/cases.py +177 -0
- earful/livekit/clock.py +458 -0
- earful/livekit/harness.py +751 -0
- earful/livekit/playbook.py +320 -0
- earful/livekit/providers.py +723 -0
- earful/livekit/scenario.py +346 -0
- earful/livekit/sink.py +371 -0
- earful/livekit/tools.py +345 -0
- earful/py.typed +0 -0
- earful/pytest_plugin.py +185 -0
- earful-0.1.0.dist-info/METADATA +318 -0
- earful-0.1.0.dist-info/RECORD +50 -0
- earful-0.1.0.dist-info/WHEEL +4 -0
- earful-0.1.0.dist-info/entry_points.txt +5 -0
- earful-0.1.0.dist-info/licenses/LICENSE +201 -0
- earful-0.1.0.dist-info/licenses/NOTICE +15 -0
earful/__init__.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Earful: deterministic regression tests for real-time voice agents.
|
|
2
|
+
|
|
3
|
+
This package is the framework-independent part: the ledger, the contract checker, the
|
|
4
|
+
backend a test's side effects are recorded in, and the names a stand-in service or a
|
|
5
|
+
ledger file needs. It imports no voice framework.
|
|
6
|
+
|
|
7
|
+
from earful import check, read_ledger
|
|
8
|
+
|
|
9
|
+
report = check(read_ledger("run.ledger.jsonl"))
|
|
10
|
+
report.assert_contracts()
|
|
11
|
+
|
|
12
|
+
Running an agent is the adapter's part. For LiveKit Agents, installed with
|
|
13
|
+
``pip install "earful[livekit]"``:
|
|
14
|
+
|
|
15
|
+
from earful.livekit import Case, ScriptedModel, UserSpeech, explore, run
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from importlib import metadata as _metadata
|
|
19
|
+
|
|
20
|
+
from .core.backend import BackendError, BackendTimeout, Effect, effect
|
|
21
|
+
from .core.checker import ContractFailure, Report, Verdict, check
|
|
22
|
+
from .core.ledger import ledger_hash
|
|
23
|
+
from .core.ledger import read_jsonl as read_ledger
|
|
24
|
+
from .core.ledger import write_jsonl as write_ledger
|
|
25
|
+
from .core.predicate import Predicate
|
|
26
|
+
|
|
27
|
+
try:
|
|
28
|
+
__version__ = _metadata.version("earful")
|
|
29
|
+
except _metadata.PackageNotFoundError: # run from a source tree that is not installed
|
|
30
|
+
__version__ = "0+unknown"
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
"BackendError",
|
|
34
|
+
"BackendTimeout",
|
|
35
|
+
"ContractFailure",
|
|
36
|
+
"Effect",
|
|
37
|
+
"Predicate",
|
|
38
|
+
"Report",
|
|
39
|
+
"Verdict",
|
|
40
|
+
"__version__",
|
|
41
|
+
"check",
|
|
42
|
+
"effect",
|
|
43
|
+
"ledger_hash",
|
|
44
|
+
"read_ledger",
|
|
45
|
+
"write_ledger",
|
|
46
|
+
]
|
earful/__main__.py
ADDED
earful/cli.py
ADDED
|
@@ -0,0 +1,504 @@
|
|
|
1
|
+
"""The ``earful`` command.
|
|
2
|
+
|
|
3
|
+
earful check LEDGER... the verdicts for recorded ledgers
|
|
4
|
+
earful run SCENARIO run a scenario once and print its verdicts
|
|
5
|
+
earful search SCENARIO search a scenario for the timings and faults that break it
|
|
6
|
+
earful replay BUNDLE run a kept failure again
|
|
7
|
+
earful crosscheck SCENARIO run a case on the real clock too, and compare
|
|
8
|
+
earful report LEDGER|BUNDLE a run, drawn as one HTML page or written out as text
|
|
9
|
+
|
|
10
|
+
A SCENARIO is ``module:function``: a function that takes no arguments and returns the
|
|
11
|
+
case. Running one imports and calls it, and replaying a bundle imports and runs the
|
|
12
|
+
scenario the bundle names, so run only code you trust.
|
|
13
|
+
|
|
14
|
+
``check`` and ``report`` read files and need nothing else. ``run``, ``search``,
|
|
15
|
+
``replay`` and ``crosscheck`` run an agent and need the framework adapter:
|
|
16
|
+
``pip install "earful[livekit]"``.
|
|
17
|
+
|
|
18
|
+
Exit status, for every command:
|
|
19
|
+
|
|
20
|
+
0 nothing broke, or what happened is what was expected; for ``report``, the
|
|
21
|
+
report was written, whatever it says
|
|
22
|
+
1 a contract rule broke, a replay did not do what --expect names, or a real-clock
|
|
23
|
+
run did not agree with the virtual one
|
|
24
|
+
2 the command could not be carried out: bad arguments, a file that is not what it
|
|
25
|
+
should be, a scenario that does not exist
|
|
26
|
+
3 there is no verdict: a run was not valid evidence, did not repeat, or could not
|
|
27
|
+
be compared
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import argparse
|
|
33
|
+
import json
|
|
34
|
+
import os
|
|
35
|
+
import sys
|
|
36
|
+
from collections.abc import Callable, Sequence
|
|
37
|
+
from pathlib import Path
|
|
38
|
+
from typing import TYPE_CHECKING, Any
|
|
39
|
+
|
|
40
|
+
from . import __version__
|
|
41
|
+
from .core import bundle, crosscheck, html, timeline
|
|
42
|
+
from .core import case as cases
|
|
43
|
+
from .core.checker import Report, check
|
|
44
|
+
from .core.checker.report import allowed
|
|
45
|
+
from .core.ledger import read_jsonl, write_jsonl
|
|
46
|
+
from .core.predicate import Predicate
|
|
47
|
+
|
|
48
|
+
if TYPE_CHECKING:
|
|
49
|
+
from types import ModuleType
|
|
50
|
+
|
|
51
|
+
OK, BROKEN, REFUSED, NO_VERDICT = 0, 1, 2, 3
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class Refused(Exception):
|
|
55
|
+
"""The command cannot be carried out as asked. Its text is what the user is told."""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _adapter() -> ModuleType:
|
|
59
|
+
"""The LiveKit adapter, imported only by the commands that run an agent.
|
|
60
|
+
|
|
61
|
+
Those commands name a scenario as ``module:function``, so the directory the command
|
|
62
|
+
is run from is put on the import path, as ``python -m`` and pytest put it there. The
|
|
63
|
+
installed ``earful`` script would otherwise not find a module beside it.
|
|
64
|
+
"""
|
|
65
|
+
here = os.getcwd()
|
|
66
|
+
if here not in sys.path:
|
|
67
|
+
sys.path.insert(0, here)
|
|
68
|
+
try:
|
|
69
|
+
from . import livekit
|
|
70
|
+
except ImportError as exc:
|
|
71
|
+
raise Refused(str(exc)) from exc
|
|
72
|
+
return livekit
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _ledger(path: Path) -> list[dict[str, Any]]:
|
|
76
|
+
try:
|
|
77
|
+
events = read_jsonl(path)
|
|
78
|
+
except OSError as exc:
|
|
79
|
+
raise Refused(f"cannot read {path}: {exc.strerror}") from exc
|
|
80
|
+
except ValueError as exc: # not JSON, or not ASCII
|
|
81
|
+
raise Refused(f"{path} is not a ledger file: {exc}") from exc
|
|
82
|
+
if not all(isinstance(event, dict) for event in events):
|
|
83
|
+
raise Refused(f"{path} is not a ledger file: a line is not a JSON object")
|
|
84
|
+
return events
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _case(path: Path | None) -> cases.Case | None:
|
|
88
|
+
if path is None:
|
|
89
|
+
return None
|
|
90
|
+
try:
|
|
91
|
+
data = json.loads(path.read_text())
|
|
92
|
+
except OSError as exc:
|
|
93
|
+
raise Refused(f"cannot read {path}: {exc.strerror}") from exc
|
|
94
|
+
except ValueError as exc:
|
|
95
|
+
raise Refused(f"{path} is not a case file: {exc}") from exc
|
|
96
|
+
errors = cases.validate(data)
|
|
97
|
+
if errors:
|
|
98
|
+
raise Refused(f"{path} is not an earful.case/1 document: " + "; ".join(errors))
|
|
99
|
+
return dict(data)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _status(reports: Sequence[Report], allow: Sequence[str] = ()) -> int:
|
|
103
|
+
"""A rule that broke outranks a run that could not be judged."""
|
|
104
|
+
if any(rule not in allowed(allow) for report in reports for rule in report.broken()):
|
|
105
|
+
return BROKEN
|
|
106
|
+
return NO_VERDICT if any(not report.valid for report in reports) else OK
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _rules(text: list[str] | None) -> list[str]:
|
|
110
|
+
return [str(Predicate.parse(rule)) for rule in text or []]
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
# --------------------------------------------------------------------------- check
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _check(args: argparse.Namespace) -> int:
|
|
117
|
+
reports = []
|
|
118
|
+
for path in args.ledger:
|
|
119
|
+
report = check(_ledger(path))
|
|
120
|
+
reports.append(report)
|
|
121
|
+
if args.json:
|
|
122
|
+
print(json.dumps({"ledger": path.name, **report.to_json()}))
|
|
123
|
+
continue
|
|
124
|
+
print(f"== {path.name}")
|
|
125
|
+
for line in report.lines(measures=args.measures):
|
|
126
|
+
print(f" {line}")
|
|
127
|
+
return _status(reports, _rules(args.allow))
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
# --------------------------------------------------------------------------- run
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _run(args: argparse.Namespace) -> int:
|
|
134
|
+
adapter = _adapter()
|
|
135
|
+
try:
|
|
136
|
+
result = adapter.run(args.scenario, case=_case(args.case), seed=args.seed)
|
|
137
|
+
except adapter.InvalidRun as exc:
|
|
138
|
+
result = exc.result
|
|
139
|
+
if args.ledger is not None:
|
|
140
|
+
write_jsonl(result.events, args.ledger, normalized=True)
|
|
141
|
+
report = result.report
|
|
142
|
+
if args.json:
|
|
143
|
+
print(json.dumps({"case": result.case, "hash": result.hash, **report.to_json()}))
|
|
144
|
+
else:
|
|
145
|
+
print("\n".join([*timeline.heading(result.events), "", *report.lines()]))
|
|
146
|
+
if args.ledger is not None:
|
|
147
|
+
print(f"\nledger {args.ledger}")
|
|
148
|
+
return _status([report], _rules(args.allow))
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
# --------------------------------------------------------------------------- search
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _search(args: argparse.Namespace) -> int:
|
|
155
|
+
adapter = _adapter()
|
|
156
|
+
keep = [Predicate.parse(rule) for rule in args.keep or []]
|
|
157
|
+
if keep and args.out is None:
|
|
158
|
+
raise Refused("--keep needs --out, the directory to write bundles to")
|
|
159
|
+
faults = () if args.faults == "none" else tuple(args.faults.split(","))
|
|
160
|
+
case = _case(args.case)
|
|
161
|
+
|
|
162
|
+
def search(workers: int) -> Any:
|
|
163
|
+
return adapter.explore(
|
|
164
|
+
args.scenario,
|
|
165
|
+
case=case,
|
|
166
|
+
seed=args.seed,
|
|
167
|
+
workers=workers,
|
|
168
|
+
variables=() if args.variables == "none" else tuple(args.variables.split(",")),
|
|
169
|
+
faults=faults,
|
|
170
|
+
pairs=args.pairs,
|
|
171
|
+
profiles=args.profiles,
|
|
172
|
+
words=not args.no_words,
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
try:
|
|
176
|
+
found = search((os.cpu_count() or 1) if args.workers is None else args.workers)
|
|
177
|
+
except adapter.WorkersUnavailable as exc:
|
|
178
|
+
if args.workers is not None: # asked for by number: say so, do not do otherwise
|
|
179
|
+
raise Refused(str(exc)) from exc
|
|
180
|
+
print(f"earful: {exc}. Searching in this process.", file=sys.stderr)
|
|
181
|
+
found = search(0)
|
|
182
|
+
# A rule that is kept is measured whatever else is: its reduction starts from there.
|
|
183
|
+
broken = found.breaks(windows=keep if args.no_windows else True)
|
|
184
|
+
kept: dict[str, Any] = {}
|
|
185
|
+
function = args.scenario.rpartition(":")[2]
|
|
186
|
+
release = (found.runner.ledger(found.case) or [{}])[0].get("data", {}).get("framework", {})
|
|
187
|
+
for predicate in keep:
|
|
188
|
+
if not any(one.predicate == predicate for one in broken):
|
|
189
|
+
continue
|
|
190
|
+
rule = f"{predicate.contract}-{predicate.rule}".replace(" ", "-")
|
|
191
|
+
directory = args.out / f"lk-{release.get('version')}.{function}.{rule}"
|
|
192
|
+
manifest = adapter.keep(found, predicate, directory).manifest
|
|
193
|
+
kept[str(predicate)] = {
|
|
194
|
+
"directory": str(directory),
|
|
195
|
+
"speeches": [manifest["original"]["speeches"], manifest["minimal"]["speeches"]],
|
|
196
|
+
"delays": [manifest["original"]["delays"], manifest["minimal"]["delays"]],
|
|
197
|
+
"faults": [manifest["original"]["faults"], manifest["minimal"]["faults"]],
|
|
198
|
+
"reduction": manifest["reduction"],
|
|
199
|
+
"windows": manifest["windows"],
|
|
200
|
+
"replay": manifest["replay"],
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
if args.json:
|
|
204
|
+
summary = {"scenario": args.scenario, **found.summary(), "coverage": found.coverage()}
|
|
205
|
+
for one in broken:
|
|
206
|
+
if not args.no_windows or one.predicate in keep:
|
|
207
|
+
entry = summary["found"][str(one.predicate)]
|
|
208
|
+
entry["windows"] = [w.to_json() for w in one.windows]
|
|
209
|
+
entry["instants_only"] = one.instants_only
|
|
210
|
+
summary["bundles"] = kept
|
|
211
|
+
print(json.dumps(summary, indent=2))
|
|
212
|
+
else:
|
|
213
|
+
print(found.describe(broken))
|
|
214
|
+
for rule, entry in kept.items():
|
|
215
|
+
print(f"\nkept {rule} in {entry['directory']}")
|
|
216
|
+
missing = [str(p) for p in keep if str(p) not in kept]
|
|
217
|
+
if missing:
|
|
218
|
+
print(f"\nnot kept, since no case broke them: {', '.join(missing)}")
|
|
219
|
+
counted = [
|
|
220
|
+
one
|
|
221
|
+
for one in broken
|
|
222
|
+
if not one.instants_only and str(one.predicate) not in allowed(_rules(args.allow))
|
|
223
|
+
]
|
|
224
|
+
return BROKEN if counted else OK
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
# --------------------------------------------------------------------------- replay
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _replay(args: argparse.Namespace) -> int:
|
|
231
|
+
adapter = _adapter()
|
|
232
|
+
try:
|
|
233
|
+
kept = bundle.load(args.bundle)
|
|
234
|
+
except OSError as exc:
|
|
235
|
+
raise Refused(f"cannot read the bundle in {args.bundle}: {exc.strerror}") from exc
|
|
236
|
+
result = adapter.replay(args.bundle, repeats=args.repeats)
|
|
237
|
+
outcome = {
|
|
238
|
+
"predicate": str(kept.predicate),
|
|
239
|
+
"made_on": kept.manifest["framework"],
|
|
240
|
+
"replayed_on": result.framework,
|
|
241
|
+
"reproduced": result.reproduced,
|
|
242
|
+
"repeatable": result.repeatable,
|
|
243
|
+
"identical_to_recorded": result.identical,
|
|
244
|
+
"scenario_code_changed": result.scenario_changed,
|
|
245
|
+
"invalid": list(result.invalid),
|
|
246
|
+
}
|
|
247
|
+
if args.json:
|
|
248
|
+
print(json.dumps(outcome, indent=2))
|
|
249
|
+
else:
|
|
250
|
+
made, on = kept.manifest["framework"], result.framework
|
|
251
|
+
print(f"{kept.predicate}, kept on {made.get('name')} {made.get('version')}")
|
|
252
|
+
print(f"replayed {args.repeats} times on {on.get('name')} {on.get('version')}")
|
|
253
|
+
print(f" breaks the rule {'yes' if result.reproduced else 'no'}")
|
|
254
|
+
print(f" every run the same {'yes' if result.repeatable else 'no'}")
|
|
255
|
+
print(f" the ledger recorded {'yes' if result.identical else 'no'}")
|
|
256
|
+
changed = {None: "not known", True: "yes", False: "no"}[result.scenario_changed]
|
|
257
|
+
print(f" scenario code changed {changed}")
|
|
258
|
+
for reason in result.invalid:
|
|
259
|
+
print(f" invalid: {reason}")
|
|
260
|
+
if result.invalid or not result.repeatable:
|
|
261
|
+
return NO_VERDICT
|
|
262
|
+
if args.expect is None:
|
|
263
|
+
return OK
|
|
264
|
+
return OK if result.reproduced == (args.expect == "reproduced") else BROKEN
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
# --------------------------------------------------------------------------- crosscheck
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _crosscheck(args: argparse.Namespace) -> int:
|
|
271
|
+
adapter = _adapter()
|
|
272
|
+
reference: str = args.source
|
|
273
|
+
case = _case(args.case)
|
|
274
|
+
if Path(args.source).is_dir(): # a kept failure: its reduced case
|
|
275
|
+
try:
|
|
276
|
+
case = bundle.load(Path(args.source)).minimal
|
|
277
|
+
except OSError as exc:
|
|
278
|
+
raise Refused(f"cannot read the bundle in {args.source}: {exc.strerror}") from exc
|
|
279
|
+
reference = case["scenario"]
|
|
280
|
+
tolerance = crosscheck.Tolerance(time_us=args.tolerance_ms * 1000, words=args.words)
|
|
281
|
+
try:
|
|
282
|
+
checked = adapter.cross_check(
|
|
283
|
+
reference, case=case, seed=args.seed, runs=args.runs, tolerance=tolerance
|
|
284
|
+
)
|
|
285
|
+
except adapter.InvalidRun as exc:
|
|
286
|
+
print("the virtual run is not valid evidence, so there is nothing to compare with:")
|
|
287
|
+
for reason in exc.reasons:
|
|
288
|
+
print(f" {reason}")
|
|
289
|
+
return NO_VERDICT
|
|
290
|
+
if args.out is not None:
|
|
291
|
+
write_jsonl(checked.virtual, args.out / "virtual.ledger.jsonl", normalized=True)
|
|
292
|
+
for number, events in enumerate(checked.real, 1):
|
|
293
|
+
write_jsonl(events, args.out / f"real-{number}.ledger.jsonl", normalized=True)
|
|
294
|
+
if args.json:
|
|
295
|
+
print(json.dumps(checked.to_json(), indent=2))
|
|
296
|
+
else:
|
|
297
|
+
print("\n".join(checked.lines()))
|
|
298
|
+
if args.out is not None:
|
|
299
|
+
print(f"\nledgers {args.out}")
|
|
300
|
+
if checked.agrees:
|
|
301
|
+
return OK
|
|
302
|
+
return NO_VERDICT if checked.unusable else BROKEN
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
# --------------------------------------------------------------------------- report
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _report(args: argparse.Namespace) -> int:
|
|
309
|
+
kept = args.source.is_dir()
|
|
310
|
+
if kept:
|
|
311
|
+
try:
|
|
312
|
+
events = bundle.load(args.source).ledger(args.which)
|
|
313
|
+
except OSError as exc:
|
|
314
|
+
raise Refused(f"cannot read the bundle in {args.source}: {exc.strerror}") from exc
|
|
315
|
+
else:
|
|
316
|
+
events = _ledger(args.source)
|
|
317
|
+
# A page where the output is a page, text where it is a terminal or a text file.
|
|
318
|
+
as_page = args.format == "html" or (
|
|
319
|
+
args.format is None
|
|
320
|
+
and args.output is not None
|
|
321
|
+
and args.output.suffix.lower() in (".html", ".htm")
|
|
322
|
+
)
|
|
323
|
+
if not as_page:
|
|
324
|
+
written = timeline.text(events)
|
|
325
|
+
elif kept: # with what the bundle froze beside its ledger
|
|
326
|
+
written = html.render_bundle(args.source, args.which)
|
|
327
|
+
else:
|
|
328
|
+
written = html.render(events, title=args.source.name)
|
|
329
|
+
if args.output is None:
|
|
330
|
+
sys.stdout.write(written)
|
|
331
|
+
else:
|
|
332
|
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
333
|
+
args.output.write_text(written, encoding="utf-8")
|
|
334
|
+
print(f"report {args.output}")
|
|
335
|
+
return OK # the report was written; what it says is not this command's status
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
# --------------------------------------------------------------------------- the command
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _parser() -> argparse.ArgumentParser:
|
|
342
|
+
parser = argparse.ArgumentParser(
|
|
343
|
+
prog="earful",
|
|
344
|
+
description="Deterministic regression tests for real-time voice agents.",
|
|
345
|
+
epilog="Exit status: 0 nothing broke; 1 a contract rule broke; 2 the command could "
|
|
346
|
+
"not be carried out; 3 no verdict (a run was not valid evidence, or did not repeat).",
|
|
347
|
+
)
|
|
348
|
+
parser.add_argument("--version", action="version", version=f"earful {__version__}")
|
|
349
|
+
commands = parser.add_subparsers(dest="command", required=True, metavar="COMMAND")
|
|
350
|
+
|
|
351
|
+
def command(name: str, run: Callable[[argparse.Namespace], int], help: str) -> Any:
|
|
352
|
+
sub = commands.add_parser(name, help=help, description=help)
|
|
353
|
+
sub.set_defaults(run=run)
|
|
354
|
+
return sub
|
|
355
|
+
|
|
356
|
+
def allow(sub: Any) -> None:
|
|
357
|
+
sub.add_argument(
|
|
358
|
+
"--allow",
|
|
359
|
+
action="append",
|
|
360
|
+
metavar="CONTRACT/RULE",
|
|
361
|
+
help="a rule that may break without failing the command (repeatable)",
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
def scenario(sub: Any) -> None:
|
|
365
|
+
sub.add_argument("scenario", metavar="SCENARIO", help="module:function")
|
|
366
|
+
sub.add_argument("--case", type=Path, metavar="FILE", help="a saved case (earful.case/1)")
|
|
367
|
+
sub.add_argument("--seed", type=int, default=0, help="seed of the run (default 0)")
|
|
368
|
+
|
|
369
|
+
sub = command("check", _check, "Print the verdicts for recorded ledgers.")
|
|
370
|
+
sub.add_argument("ledger", nargs="+", type=Path, metavar="LEDGER")
|
|
371
|
+
sub.add_argument("--json", action="store_true", help="one JSON report per ledger")
|
|
372
|
+
sub.add_argument("--measures", action="store_true", help="what each verdict measured")
|
|
373
|
+
allow(sub)
|
|
374
|
+
|
|
375
|
+
sub = command("run", _run, "Run a scenario once and print its verdicts.")
|
|
376
|
+
scenario(sub)
|
|
377
|
+
sub.add_argument("--ledger", type=Path, metavar="FILE", help="write the ledger here")
|
|
378
|
+
sub.add_argument("--json", action="store_true", help="the report as JSON")
|
|
379
|
+
allow(sub)
|
|
380
|
+
|
|
381
|
+
sub = command(
|
|
382
|
+
"search", _search, "Search a scenario for the timings and faults that break a contract."
|
|
383
|
+
)
|
|
384
|
+
scenario(sub)
|
|
385
|
+
sub.add_argument(
|
|
386
|
+
"--variables",
|
|
387
|
+
default=",".join(("speech.position", *cases.DELAY_KINDS, cases.HANGUP)),
|
|
388
|
+
metavar="A,B",
|
|
389
|
+
help="kinds of time to move (default: speech.position, every provider delay and "
|
|
390
|
+
'user.hangup; speech.end and speech.stt_delay can be added; "none" for none)',
|
|
391
|
+
)
|
|
392
|
+
sub.add_argument(
|
|
393
|
+
"--faults",
|
|
394
|
+
default=",".join(cases.FAULT_KINDS),
|
|
395
|
+
metavar="A,B",
|
|
396
|
+
help='faults to add to each backend write (default: all three; "none" for none)',
|
|
397
|
+
)
|
|
398
|
+
sub.add_argument(
|
|
399
|
+
"--pairs",
|
|
400
|
+
action="store_true",
|
|
401
|
+
help="also move every speech and the hangup with each fault and each stretched "
|
|
402
|
+
"delay in place, and add faults to two writes at once",
|
|
403
|
+
)
|
|
404
|
+
sub.add_argument(
|
|
405
|
+
"--profiles", type=int, default=0, metavar="N", help="seeded provider-timing profiles"
|
|
406
|
+
)
|
|
407
|
+
sub.add_argument(
|
|
408
|
+
"--no-words", action="store_true", help="the end of a played word is not a boundary"
|
|
409
|
+
)
|
|
410
|
+
sub.add_argument(
|
|
411
|
+
"--no-windows",
|
|
412
|
+
action="store_true",
|
|
413
|
+
help="do not measure how wide each failure is (a kept one is still measured)",
|
|
414
|
+
)
|
|
415
|
+
sub.add_argument(
|
|
416
|
+
"--keep",
|
|
417
|
+
action="append",
|
|
418
|
+
metavar="CONTRACT/RULE",
|
|
419
|
+
help="reduce a case that breaks this rule and write it as a bundle (repeatable)",
|
|
420
|
+
)
|
|
421
|
+
sub.add_argument("--out", type=Path, metavar="DIR", help="where bundles go")
|
|
422
|
+
sub.add_argument(
|
|
423
|
+
"--workers",
|
|
424
|
+
type=int,
|
|
425
|
+
metavar="N",
|
|
426
|
+
help="worker processes (default: one per processor, or this process where worker "
|
|
427
|
+
"processes cannot be started; 0 runs in this process)",
|
|
428
|
+
)
|
|
429
|
+
sub.add_argument("--json", action="store_true", help="the whole result as JSON")
|
|
430
|
+
allow(sub)
|
|
431
|
+
|
|
432
|
+
sub = command("replay", _replay, "Run a kept failure's reduced case again.")
|
|
433
|
+
sub.add_argument("bundle", type=Path, metavar="BUNDLE_DIR")
|
|
434
|
+
sub.add_argument("--repeats", type=int, default=3, metavar="N", help="runs (default 3)")
|
|
435
|
+
sub.add_argument(
|
|
436
|
+
"--expect",
|
|
437
|
+
choices=("reproduced", "clean"),
|
|
438
|
+
help="fail unless the rule still breaks (reproduced) or no longer does (clean)",
|
|
439
|
+
)
|
|
440
|
+
sub.add_argument("--json", action="store_true", help="the outcome as JSON")
|
|
441
|
+
|
|
442
|
+
sub = command(
|
|
443
|
+
"crosscheck",
|
|
444
|
+
_crosscheck,
|
|
445
|
+
"Run a case once on the virtual clock and again on the real clock, with nothing "
|
|
446
|
+
"patched, and say whether the real-clock runs agree with the virtual one.",
|
|
447
|
+
)
|
|
448
|
+
sub.add_argument(
|
|
449
|
+
"source", metavar="SCENARIO|BUNDLE_DIR", help="module:function, or a kept failure"
|
|
450
|
+
)
|
|
451
|
+
sub.add_argument("--case", type=Path, metavar="FILE", help="a saved case (earful.case/1)")
|
|
452
|
+
sub.add_argument("--seed", type=int, default=0, help="seed of the runs (default 0)")
|
|
453
|
+
sub.add_argument("--runs", type=int, default=3, metavar="N", help="real-clock runs (default 3)")
|
|
454
|
+
sub.add_argument(
|
|
455
|
+
"--tolerance-ms",
|
|
456
|
+
type=int,
|
|
457
|
+
default=100,
|
|
458
|
+
metavar="MS",
|
|
459
|
+
help="how far an event may be from its virtual time (default 100)",
|
|
460
|
+
)
|
|
461
|
+
sub.add_argument(
|
|
462
|
+
"--words",
|
|
463
|
+
type=int,
|
|
464
|
+
default=1,
|
|
465
|
+
metavar="N",
|
|
466
|
+
help="words a segment that was cut short may differ by (default 1)",
|
|
467
|
+
)
|
|
468
|
+
sub.add_argument("--out", type=Path, metavar="DIR", help="write every run's ledger here")
|
|
469
|
+
sub.add_argument("--json", action="store_true", help="the result as JSON")
|
|
470
|
+
|
|
471
|
+
sub = command(
|
|
472
|
+
"report",
|
|
473
|
+
_report,
|
|
474
|
+
"Draw a run as one HTML page (a timeline, the verdicts and every event), or write "
|
|
475
|
+
"it out as text.",
|
|
476
|
+
)
|
|
477
|
+
sub.add_argument("source", type=Path, metavar="LEDGER|BUNDLE_DIR")
|
|
478
|
+
sub.add_argument(
|
|
479
|
+
"--which",
|
|
480
|
+
choices=tuple(bundle.FILES),
|
|
481
|
+
default="minimal",
|
|
482
|
+
help="for a bundle: the reduced case or the one first found (default minimal)",
|
|
483
|
+
)
|
|
484
|
+
sub.add_argument("-o", "--output", type=Path, metavar="FILE", help="write here, not stdout")
|
|
485
|
+
sub.add_argument(
|
|
486
|
+
"--format",
|
|
487
|
+
choices=("html", "text"),
|
|
488
|
+
help="default: html for an output file named .html, text otherwise",
|
|
489
|
+
)
|
|
490
|
+
return parser
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
494
|
+
args = _parser().parse_args(argv)
|
|
495
|
+
try:
|
|
496
|
+
return int(args.run(args))
|
|
497
|
+
except Refused as exc:
|
|
498
|
+
print(f"earful: {exc}", file=sys.stderr)
|
|
499
|
+
except ValueError as exc: # a scenario, a case or a bundle the core refuses
|
|
500
|
+
print(f"earful: {exc}", file=sys.stderr)
|
|
501
|
+
return REFUSED
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
__all__ = ["BROKEN", "NO_VERDICT", "OK", "REFUSED", "main"]
|
earful/core/__init__.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""The portable core: ledger, validator, contract checker, effect backend, case format,
|
|
2
|
+
boundary scheduler, shrinker and failure bundle.
|
|
3
|
+
|
|
4
|
+
Nothing here imports a voice framework. An adapter supplies one function, from a case to
|
|
5
|
+
the ledger of running it, and everything else works on plain data.
|
|
6
|
+
|
|
7
|
+
The names most code needs are here. Each module holds the rest:
|
|
8
|
+
|
|
9
|
+
backend the effect backend, ``effect`` and ``Effect``
|
|
10
|
+
bundle a kept failure: ``capture``, ``load``, ``replay``
|
|
11
|
+
case a case as data (earful.case/1) and its variables
|
|
12
|
+
checker ``check`` and the report it returns
|
|
13
|
+
ledger the event ledger, its normal form and its hash
|
|
14
|
+
predicate a contract rule that a run breaks
|
|
15
|
+
schema ``validate`` and ``validity`` for earful.ledger/1
|
|
16
|
+
search the boundary scheduler: ``explore``
|
|
17
|
+
shrink reduction of a failing case
|
|
18
|
+
timeline a ledger as text, for a person
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from .backend import BackendError, BackendTimeout, Effect, EffectBackend, effect
|
|
22
|
+
from .checker import ContractFailure, ContractResult, Finding, Report, Verdict, check
|
|
23
|
+
from .ledger import Ledger, ledger_hash, normalize, read_jsonl, write_jsonl
|
|
24
|
+
from .predicate import Predicate
|
|
25
|
+
from .schema import validate, validity
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"BackendError",
|
|
29
|
+
"BackendTimeout",
|
|
30
|
+
"ContractFailure",
|
|
31
|
+
"ContractResult",
|
|
32
|
+
"Effect",
|
|
33
|
+
"EffectBackend",
|
|
34
|
+
"Finding",
|
|
35
|
+
"Ledger",
|
|
36
|
+
"Predicate",
|
|
37
|
+
"Report",
|
|
38
|
+
"Verdict",
|
|
39
|
+
"check",
|
|
40
|
+
"effect",
|
|
41
|
+
"ledger_hash",
|
|
42
|
+
"normalize",
|
|
43
|
+
"read_jsonl",
|
|
44
|
+
"validate",
|
|
45
|
+
"validity",
|
|
46
|
+
"write_jsonl",
|
|
47
|
+
]
|