evarness 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. evarness/__init__.py +123 -0
  2. evarness/cli/__init__.py +557 -0
  3. evarness/core/__init__.py +5 -0
  4. evarness/core/attest.py +130 -0
  5. evarness/core/config.py +52 -0
  6. evarness/core/errors.py +47 -0
  7. evarness/core/executor.py +218 -0
  8. evarness/core/graph.py +174 -0
  9. evarness/core/interfaces.py +39 -0
  10. evarness/core/invariants.py +266 -0
  11. evarness/core/prove.py +714 -0
  12. evarness/core/registry.py +227 -0
  13. evarness/core/store.py +420 -0
  14. evarness/core/trace.py +97 -0
  15. evarness/domains/__init__.py +7 -0
  16. evarness/domains/agents/__init__.py +92 -0
  17. evarness/domains/agents/catalog.py +71 -0
  18. evarness/domains/agents/classification.py +207 -0
  19. evarness/domains/agents/classification.yaml +26 -0
  20. evarness/domains/agents/contracts.yaml +88 -0
  21. evarness/domains/agents/grounding.py +426 -0
  22. evarness/domains/agents/grounding.yaml +90 -0
  23. evarness/domains/agents/judges.py +220 -0
  24. evarness/domains/agents/judges.yaml +33 -0
  25. evarness/domains/agents/lint.py +39 -0
  26. evarness/domains/agents/nodes/__init__.py +61 -0
  27. evarness/domains/agents/nodes/base.py +222 -0
  28. evarness/domains/agents/nodes/basic.py +230 -0
  29. evarness/domains/agents/nodes/governance.py +719 -0
  30. evarness/domains/agents/nodes/loop.py +440 -0
  31. evarness/domains/agents/nodes/memory.py +283 -0
  32. evarness/domains/agents/nodes/observability.py +130 -0
  33. evarness/domains/agents/nodes/rag.py +138 -0
  34. evarness/domains/agents/nodes/tools.py +188 -0
  35. evarness/domains/agents/patterns/__init__.py +272 -0
  36. evarness/domains/agents/patterns/approval_gated_send/fixtures/send.yaml +16 -0
  37. evarness/domains/agents/patterns/approval_gated_send/graph.json +41 -0
  38. evarness/domains/agents/patterns/approval_gated_send/invariants.yaml +17 -0
  39. evarness/domains/agents/patterns/approval_gated_send/lesson.md +83 -0
  40. evarness/domains/agents/patterns/governed_email_assistant/fixtures/failure.yaml +20 -0
  41. evarness/domains/agents/patterns/governed_email_assistant/fixtures/happy.yaml +17 -0
  42. evarness/domains/agents/patterns/governed_email_assistant/graph.json +50 -0
  43. evarness/domains/agents/patterns/governed_email_assistant/lesson.md +12 -0
  44. evarness/domains/agents/patterns/single_shot_qa/fixtures/happy.yaml +7 -0
  45. evarness/domains/agents/patterns/single_shot_qa/graph.json +25 -0
  46. evarness/domains/agents/patterns/single_shot_qa/lesson.md +3 -0
  47. evarness/domains/agents/prompts.py +33 -0
  48. evarness/domains/agents/prompts.yaml +94 -0
  49. evarness/domains/agents/providers.py +55 -0
  50. evarness/domains/agents/sim.py +214 -0
  51. evarness/domains/agents/state.py +42 -0
  52. evarness/domains/agents/tiers.py +130 -0
  53. evarness/domains/agents/tiers.yaml +26 -0
  54. evarness/domains/agents/tools.yaml +138 -0
  55. evarness/domains/agents/toolspec.py +328 -0
  56. evarness/io/__init__.py +0 -0
  57. evarness/io/exporters.py +469 -0
  58. evarness/io/exporters.yaml +11 -0
  59. evarness/io/render.py +841 -0
  60. evarness/py.typed +0 -0
  61. evarness-0.1.0a1.dist-info/METADATA +165 -0
  62. evarness-0.1.0a1.dist-info/RECORD +66 -0
  63. evarness-0.1.0a1.dist-info/WHEEL +5 -0
  64. evarness-0.1.0a1.dist-info/entry_points.txt +2 -0
  65. evarness-0.1.0a1.dist-info/licenses/LICENSE +201 -0
  66. evarness-0.1.0a1.dist-info/top_level.txt +1 -0
evarness/__init__.py ADDED
@@ -0,0 +1,123 @@
1
+ """evarness — prove an AI agent harness before it touches real data.
2
+
3
+ ``import evarness`` gives you the batteries-included surface: the kernel plus
4
+ the agents domain, with plugins discovered — loaded lazily, on first
5
+ attribute access. For the bare kernel (e.g. when building your own domain),
6
+ import from ``evarness.core`` directly — it never loads a domain behind your
7
+ back, and ``tests/test_architecture_boundaries.py`` holds that promise.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from typing import TYPE_CHECKING, Any
13
+
14
+ if TYPE_CHECKING: # static types for the lazy surface
15
+ from evarness.core.errors import (
16
+ EvarnessError as EvarnessError,
17
+ GraphValidationError as GraphValidationError,
18
+ NodeBlocked as NodeBlocked,
19
+ RegistryError as RegistryError,
20
+ RunPaused as RunPaused,
21
+ )
22
+ from evarness.core.executor import (
23
+ Emitter as Emitter,
24
+ RunContext as RunContext,
25
+ RunResult as RunResult,
26
+ execute as execute,
27
+ )
28
+ from evarness.core.graph import (
29
+ GraphModel as GraphModel,
30
+ lint as lint,
31
+ migrate as migrate,
32
+ topological_order as topological_order,
33
+ )
34
+ from evarness.core.invariants import (
35
+ check_invariants as check_invariants,
36
+ load_invariant_defs as load_invariant_defs,
37
+ )
38
+ from evarness.core.prove import prove as prove, verify_proof as verify_proof
39
+ from evarness.core.registry import (
40
+ NODE_TYPES as NODE_TYPES,
41
+ load_entry_point_plugins as load_entry_point_plugins,
42
+ )
43
+ from evarness.core.trace import canonical_trace as canonical_trace, trace_digest as trace_digest
44
+ from evarness.domains import agents as agents
45
+ from evarness.domains.agents.sim import Fixture as Fixture, load_fixture as load_fixture
46
+
47
+ __version__ = "0.1.0a1"
48
+
49
+ __all__ = [
50
+ "EvarnessError",
51
+ "GraphValidationError",
52
+ "NodeBlocked",
53
+ "RegistryError",
54
+ "RunPaused",
55
+ "Emitter",
56
+ "RunContext",
57
+ "RunResult",
58
+ "execute",
59
+ "GraphModel",
60
+ "lint",
61
+ "migrate",
62
+ "topological_order",
63
+ "check_invariants",
64
+ "load_invariant_defs",
65
+ "prove",
66
+ "verify_proof",
67
+ "NODE_TYPES",
68
+ "load_entry_point_plugins",
69
+ "canonical_trace",
70
+ "trace_digest",
71
+ "Fixture",
72
+ "load_fixture",
73
+ "agents",
74
+ "__version__",
75
+ ]
76
+
77
+ _surface: dict[str, Any] = {}
78
+
79
+
80
+ def _load_surface() -> dict[str, Any]:
81
+ from evarness.core import errors, executor, graph, invariants, prove, registry, trace
82
+ from evarness.domains import agents
83
+ from evarness.domains.agents import sim
84
+
85
+ registry.load_entry_point_plugins()
86
+ return {
87
+ "EvarnessError": errors.EvarnessError,
88
+ "GraphValidationError": errors.GraphValidationError,
89
+ "NodeBlocked": errors.NodeBlocked,
90
+ "RegistryError": errors.RegistryError,
91
+ "RunPaused": errors.RunPaused,
92
+ "Emitter": executor.Emitter,
93
+ "RunContext": executor.RunContext,
94
+ "RunResult": executor.RunResult,
95
+ "execute": executor.execute,
96
+ "GraphModel": graph.GraphModel,
97
+ "lint": graph.lint,
98
+ "migrate": graph.migrate,
99
+ "topological_order": graph.topological_order,
100
+ "check_invariants": invariants.check_invariants,
101
+ "load_invariant_defs": invariants.load_invariant_defs,
102
+ "prove": prove.prove,
103
+ "verify_proof": prove.verify_proof,
104
+ "NODE_TYPES": registry.NODE_TYPES,
105
+ "load_entry_point_plugins": registry.load_entry_point_plugins,
106
+ "canonical_trace": trace.canonical_trace,
107
+ "trace_digest": trace.trace_digest,
108
+ "Fixture": sim.Fixture,
109
+ "load_fixture": sim.load_fixture,
110
+ "agents": agents,
111
+ }
112
+
113
+
114
+ def __getattr__(name: str) -> Any:
115
+ if name in __all__:
116
+ if not _surface:
117
+ _surface.update(_load_surface())
118
+ return _surface[name]
119
+ raise AttributeError(f"module 'evarness' has no attribute {name!r}")
120
+
121
+
122
+ def __dir__() -> list[str]:
123
+ return sorted(set(globals()) | set(__all__))
@@ -0,0 +1,557 @@
1
+ """evarness CLI — everything the library can do, headless.
2
+
3
+ Commands: validate | run | render | prove | verify | export | patterns.
4
+ Every command is recorded in the activity log (actor=cli) — full
5
+ traceability.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import argparse
11
+ import json
12
+ import sys
13
+ from pathlib import Path
14
+
15
+ import yaml
16
+
17
+ from evarness.core.errors import EvarnessError
18
+ from evarness.core.registry import load_entry_point_plugins
19
+ from evarness.core.executor import GraphValidationError, execute
20
+ from evarness.io.exporters import export_bundle, export_trace
21
+ from evarness.io.render import (
22
+ RENDER_VERSION,
23
+ RenderSubject,
24
+ render,
25
+ render_proof_browser,
26
+ )
27
+ from evarness.domains.agents.nodes import REGISTRY
28
+ from evarness.domains.agents.nodes.base import presentation
29
+ from evarness.domains.agents.patterns import (
30
+ fixture_names,
31
+ fixture_text,
32
+ invariant_defs as pattern_invariant_defs,
33
+ list_patterns,
34
+ load_pattern,
35
+ )
36
+ from evarness.core.prove import prove, render_junit, render_proof_html, render_sarif, verify_proof
37
+ from evarness.core.graph import GraphModel, lint, migrate
38
+ from evarness.domains.agents.sim import load_fixture
39
+ from evarness.core.store import init_db, log_activity
40
+ from evarness.core.trace import canonical_trace, trace_digest
41
+
42
+
43
+ def _load_graph(path: str) -> GraphModel:
44
+ doc = migrate(json.loads(Path(path).read_text()))
45
+ return GraphModel.model_validate(doc)
46
+
47
+
48
+ def _presentation_for(graph: GraphModel) -> dict:
49
+ """The domain's palette as plain data — io/render stays domain-agnostic;
50
+ the CLI (a caller) is where domain presentation gets wired in."""
51
+ return {t: presentation(t) for t in {n.type for n in graph.nodes}}
52
+
53
+
54
+ def cmd_validate(args) -> int:
55
+ graph = _load_graph(args.graph)
56
+ issues = lint(graph, REGISTRY)
57
+ log_activity("cli.validate", graph.id, actor="cli", issues=len(issues))
58
+ if not issues:
59
+ print("OK — graph valid, no lint findings")
60
+ return 0
61
+ for i in issues:
62
+ print(f"{i['level'].upper():<8} [{i['code']}] {i['message']}")
63
+ return 1 if any(i["level"] == "error" for i in issues) else 0
64
+
65
+
66
+ def cmd_run(args) -> int:
67
+ graph = _load_graph(args.graph)
68
+ fixture = load_fixture(args.fixture)
69
+ # --approve gate=decision (repeatable): supply human decisions so an
70
+ # approval_gate resumes instead of pausing. Replay is deterministic.
71
+ approvals: dict = {}
72
+ for pair in args.approve or []:
73
+ gate, _, decision = pair.partition("=")
74
+ approvals[gate.strip()] = decision.strip() or "approve"
75
+ # pattern-local contracts travel with the graph: an invariants.yaml next
76
+ # to the graph file is loaded as highest-precedence definitions
77
+ inv_file = Path(args.graph).parent / "invariants.yaml"
78
+ invariant_defs = None
79
+ if inv_file.exists():
80
+ doc = yaml.safe_load(inv_file.read_text(encoding="utf-8")) or {}
81
+ invariant_defs = doc.get("invariants") or {}
82
+ log_activity("cli.run", graph.id, actor="cli", fixture=fixture.scenario)
83
+ try:
84
+ run = execute(
85
+ graph,
86
+ fixture,
87
+ user_input=args.input,
88
+ approvals=approvals,
89
+ invariant_defs=invariant_defs,
90
+ )
91
+ except GraphValidationError as exc:
92
+ print("Graph invalid:", exc)
93
+ return 2
94
+ digest = trace_digest(run.events)
95
+ if args.json:
96
+ print(
97
+ json.dumps(
98
+ {
99
+ "id": run.id,
100
+ "status": run.status,
101
+ "output": run.output,
102
+ "totals": run.totals,
103
+ "trace_digest": digest,
104
+ "invariants": run.invariants,
105
+ "events": run.events,
106
+ },
107
+ indent=2,
108
+ )
109
+ )
110
+ else:
111
+ for ev in run.events:
112
+ print(
113
+ f"{ev['seq']:>3} {ev['type']:<22} {ev['node_id'] or '':<6} "
114
+ f"{json.dumps(ev['payload'])[:110]}"
115
+ )
116
+ print(f"\nSTATUS: {run.status}" + (f" ({run.reason})" if run.reason else ""))
117
+ print(f"TRACE: {len(run.events)} events, digest {digest}")
118
+ if run.invariants:
119
+ _print_verdicts(run.invariants)
120
+ if run.status == "paused" and run.pending:
121
+ g = run.pending["node_id"]
122
+ print(f"PAUSED at {g}: {run.pending['prompt']}")
123
+ print(f" resume with: --approve {g}=approve (or {g}=reject)")
124
+ print(f"OUTPUT: {run.output}")
125
+ if args.trace_out:
126
+ meta = {
127
+ "run_id": run.id,
128
+ "name": graph.name,
129
+ "graph_id": graph.id,
130
+ "status": run.status,
131
+ "reason": run.reason,
132
+ "seed": graph.params.seed,
133
+ "provider": graph.params.provider,
134
+ "fixture": fixture.scenario,
135
+ "trace_digest": digest,
136
+ }
137
+ doc, _ = export_trace(args.trace_format, run.events, meta)
138
+ Path(args.trace_out).write_text(doc)
139
+ if not args.json:
140
+ print(f"wrote {args.trace_out} ({args.trace_format})")
141
+ if args.html:
142
+ subject = RenderSubject(
143
+ graph=graph,
144
+ events=canonical_trace(run.events),
145
+ verdicts=run.invariants,
146
+ presentation=_presentation_for(graph),
147
+ meta={
148
+ "scenario": fixture.scenario,
149
+ "status": run.status,
150
+ "reason": run.reason,
151
+ "provider": graph.params.provider,
152
+ "seed": graph.params.seed,
153
+ "deterministic": (
154
+ run.events[0]["payload"].get("deterministic") if run.events else None
155
+ ),
156
+ },
157
+ )
158
+ Path(args.html).write_text(render(subject))
159
+ if not args.json:
160
+ print(f"wrote {args.html} (render {RENDER_VERSION})")
161
+ # CI gate: a completed run with a failed invariant is a failing exit code
162
+ inv_failed = bool(run.invariants and run.invariants["failed"])
163
+ return 0 if run.status == "completed" and not inv_failed else 1
164
+
165
+
166
+ def _bundle_graph(bundle: dict, graph_arg: str | None) -> tuple[GraphModel | None, str | None]:
167
+ """Resolve a canvas graph for a proof bundle, honestly: an explicit
168
+ ``--graph`` is hash-checked inside the renderer (mismatch raises); the
169
+ bundle's pattern id is auto-resolved only if its hash still matches the
170
+ pinned subject; otherwise the canvas is omitted with a printed note —
171
+ never silently drawn from an unproven graph."""
172
+ from evarness.core.prove import graph_hash
173
+
174
+ subject = bundle.get("subject") or {}
175
+ if graph_arg:
176
+ return _load_graph(graph_arg), None # renderer enforces the hash pin
177
+ pattern = subject.get("pattern")
178
+ if pattern:
179
+ pattern_doc = load_pattern(pattern)
180
+ if pattern_doc is None:
181
+ return None, f"note: pattern '{pattern}' not found locally — canvas omitted"
182
+ graph = GraphModel.model_validate(migrate(pattern_doc))
183
+ if graph_hash(graph) != subject.get("graph_sha256"):
184
+ return None, (
185
+ f"note: pattern '{pattern}' on disk no longer matches the bundle's "
186
+ "pinned graph — canvas omitted (the canvas is only ever drawn from "
187
+ "the proven graph)"
188
+ )
189
+ return graph, None
190
+ return None, (
191
+ "note: the bundle pins its graph by hash only — pass --graph PATH to "
192
+ "draw the canvas (it must match the pinned graph_sha256)"
193
+ )
194
+
195
+
196
+ def cmd_render(args) -> int:
197
+ """Draw a target as a self-contained HTML artifact — no execution.
198
+ A graph file or pattern id gets a static canvas with lint findings
199
+ (a run's replay comes from ``run --html``); a proof bundle gets the
200
+ browsable proof: verdict badge, one viewer per scenario, the bundle's
201
+ not_proven section, and the whole bundle embedded for offline verify."""
202
+ target = args.target
203
+ if target.endswith(".json"):
204
+ try:
205
+ doc = json.loads(Path(target).read_text())
206
+ except FileNotFoundError:
207
+ print(f"error: file not found: {target}", file=sys.stderr)
208
+ return 2
209
+ except json.JSONDecodeError as exc:
210
+ print(f"error: {target}: not valid JSON: {exc}", file=sys.stderr)
211
+ return 2
212
+ else:
213
+ doc = None
214
+
215
+ if doc is not None and "proof_version" in doc:
216
+ if args.renderer != "html":
217
+ print(
218
+ f"error: proof bundles only support --renderer html (got '{args.renderer}')",
219
+ file=sys.stderr,
220
+ )
221
+ return 2
222
+ graph, note = _bundle_graph(doc, args.graph)
223
+ html = render_proof_browser(
224
+ doc, graph=graph, presentation=_presentation_for(graph) if graph else None
225
+ )
226
+ subject = doc.get("subject") or {}
227
+ log_activity("cli.render", str(subject.get("graph_id")), actor="cli", bundle=True)
228
+ out = args.out or str(Path(target).with_suffix(".html"))
229
+ Path(out).write_text(html)
230
+ if note:
231
+ print(note)
232
+ print(f"wrote {out} (proof browser, render {RENDER_VERSION})")
233
+ return 0
234
+
235
+ if doc is None:
236
+ doc = load_pattern(target)
237
+ if doc is None:
238
+ print(f"unknown target '{target}': not a .json path or a known pattern id")
239
+ return 2
240
+ graph = GraphModel.model_validate(migrate(doc))
241
+ issues = lint(graph, REGISTRY)
242
+ log_activity("cli.render", graph.id, actor="cli", issues=len(issues))
243
+ subject_r = RenderSubject(
244
+ graph=graph, presentation=_presentation_for(graph), meta={"lint": issues}
245
+ )
246
+ out = args.out or (
247
+ str(Path(target).with_suffix(".html")) if target.endswith(".json") else f"{target}.html"
248
+ )
249
+ Path(out).write_text(render(subject_r, args.renderer))
250
+ print(f"wrote {out} ({args.renderer}, render {RENDER_VERSION})")
251
+ return 0
252
+
253
+
254
+ def cmd_export(args) -> int:
255
+ """Unpack a proof bundle into the standard interchange set: per-scenario
256
+ canonical JSONL + OTLP (and any plugin format), JUnit/SARIF verdicts,
257
+ and a manifest with sha256 receipts. The bundle is verified first —
258
+ a failing bundle is refused with nothing written."""
259
+ bundle = json.loads(Path(args.bundle).read_text())
260
+ out = args.out or f"{Path(args.bundle).stem}-export"
261
+ formats = tuple(args.format) if args.format else ("jsonl", "otlp")
262
+ manifest = export_bundle(bundle, out, trace_formats=formats)
263
+ log_activity(
264
+ "cli.export",
265
+ str((manifest["subject"] or {}).get("graph_id")),
266
+ actor="cli",
267
+ files=len(manifest["files"]),
268
+ )
269
+ for f in manifest["files"]:
270
+ print(f" {f['path']:<36} {f['kind']:<12} sha256:{f['sha256'][:12]}…")
271
+ skipped = [s["fixture"] for s in manifest["scenarios"] if not s["events_exported"]]
272
+ if skipped:
273
+ print(
274
+ f" (no events embedded for: {', '.join(skipped)} — named by digest "
275
+ "in the manifest, not exported)"
276
+ )
277
+ print(f"wrote {out}/manifest.json (export {manifest['export_version']}, verified bundle)")
278
+ return 0
279
+
280
+
281
+ def cmd_prove(args) -> int:
282
+ """Build a proof bundle: run every scenario twice, check invariants,
283
+ emit proof.json (+ optional HTML). Exit 0 only when the proof holds."""
284
+ approvals: dict = {}
285
+ for pair in args.approve or []:
286
+ gate, _, decision = pair.partition("=")
287
+ approvals[gate.strip()] = decision.strip() or "approve"
288
+
289
+ pattern_doc = load_pattern(args.target) if not args.target.endswith(".json") else None
290
+ if pattern_doc:
291
+ graph = GraphModel.model_validate(migrate(pattern_doc))
292
+ names = args.fixture or fixture_names(args.target)
293
+ scenarios = [(n, fixture_text(args.target, n) or "") for n in names]
294
+ missing = [n for n, t in scenarios if not t]
295
+ if missing:
296
+ print(f"fixtures not found for pattern '{args.target}': {', '.join(missing)}")
297
+ return 2
298
+ inv_defs = pattern_invariant_defs(args.target) or None
299
+ pattern_id = args.target
300
+ else:
301
+ graph = _load_graph(args.target)
302
+ if not args.fixture:
303
+ print("--fixture PATH is required when proving a graph file")
304
+ return 2
305
+ scenarios = [(Path(f).stem, Path(f).read_text()) for f in args.fixture]
306
+ inv_file = Path(args.target).parent / "invariants.yaml"
307
+ inv_defs = (
308
+ (yaml.safe_load(inv_file.read_text()) or {}).get("invariants")
309
+ if inv_file.exists()
310
+ else None
311
+ )
312
+ pattern_id = None
313
+
314
+ log_activity("cli.prove", graph.id, actor="cli", scenarios=[n for n, _ in scenarios])
315
+ proof = prove(
316
+ graph,
317
+ scenarios,
318
+ pattern_id=pattern_id,
319
+ invariant_defs=inv_defs,
320
+ approvals=approvals,
321
+ include_events=not args.no_events,
322
+ )
323
+ if args.sign:
324
+ from evarness.core.attest import AttestationError, sign_proof
325
+
326
+ try:
327
+ proof = sign_proof(proof, key_path=args.key)
328
+ except AttestationError as exc:
329
+ print(f"cannot sign: {exc}")
330
+ return 2
331
+ att = proof["attestation"]
332
+ if att["key_created"]:
333
+ print(
334
+ f"new signing key created at {att['key_path']} "
335
+ f"(share {att['key_path'].removesuffix('.pem')}.pub out of band)"
336
+ )
337
+
338
+ Path(args.out).write_text(json.dumps(proof, indent=2))
339
+ if args.html:
340
+ Path(args.html).write_text(render_proof_html(proof))
341
+ if args.junit:
342
+ Path(args.junit).write_text(render_junit(proof))
343
+ if args.sarif:
344
+ Path(args.sarif).write_text(render_sarif(proof))
345
+
346
+ v = proof["verdict"]
347
+ for sc in proof["scenarios"]:
348
+ inv = sc["invariants"]
349
+ inv_txt = f"{inv['passed']}\u2713/{inv['failed']}\u2717" if inv else "-"
350
+ repro = {True: "reproduced", False: "NOT REPRODUCED", None: "-"}[sc["reproduced"]]
351
+ print(
352
+ f" {sc['fixture']:<16} {sc['status']:<10} invariants {inv_txt:<8} "
353
+ f"{repro:<15} {sc['trace_digest'][:24]}\u2026"
354
+ )
355
+ state = {True: "HOLDS", False: "FAILED", None: "PENDING"}[v["ok"]]
356
+ print(
357
+ f"PROOF: {state} \u2014 "
358
+ f"{v['scenarios']} scenario(s), invariants_pass={v['invariants_pass']}, "
359
+ f"reproduced={v['reproduced']}" + (f" ({v['note']})" if v.get("note") else "")
360
+ )
361
+ extras = [p for p in (args.html, args.junit, args.sarif) if p]
362
+ print(f"wrote {args.out}" + (f" and {', '.join(extras)}" if extras else ""))
363
+ return 0 if v["ok"] else 1
364
+
365
+
366
+ def cmd_verify(args) -> int:
367
+ """Re-check a proof bundle without running anything (digests, chains,
368
+ verdict consistency, signature). The reviewer-side half of `prove`."""
369
+ doc = json.loads(Path(args.bundle).read_text())
370
+ pubkey = args.pubkey
371
+ if pubkey and Path(pubkey).is_file(): # accept a .pub file or raw b64
372
+ pubkey = Path(pubkey).read_text().strip()
373
+ result = verify_proof(doc, pubkey_b64=pubkey, require_signature=args.require_signature)
374
+ log_activity("cli.verify", str(args.bundle), actor="cli", ok=result["ok"])
375
+ for c in result["checks"]:
376
+ mark = {True: "✓", False: "✗", None: "-"}[c["ok"]]
377
+ where = f" [{c['scenario']}]" if c.get("scenario") else ""
378
+ print(f" {mark} {c['check']}{where} — {c['detail']}")
379
+ print(
380
+ f"VERIFY: {'OK' if result['ok'] else 'FAILED'} — bundle is "
381
+ + ("internally consistent" if result["ok"] else "inconsistent or tampered; see ✗ above")
382
+ )
383
+ print(
384
+ "note: verification proves consistency and (if signed) integrity — "
385
+ "not that the runs happened; that trust reduces to the signer"
386
+ )
387
+ return 0 if result["ok"] else 1
388
+
389
+
390
+ def _print_verdicts(inv: dict) -> None:
391
+ print(
392
+ f"INVARIANTS: {inv['passed']} passed, {inv['failed']} FAILED"
393
+ if inv["failed"]
394
+ else f"INVARIANTS: {inv['passed']} passed"
395
+ )
396
+ for r in inv["results"]:
397
+ mark = "✓" if r["ok"] else "✗"
398
+ where = (
399
+ f" (seq {', '.join(map(str, r['evidence_seq']))})"
400
+ if not r["ok"] and r["evidence_seq"]
401
+ else ""
402
+ )
403
+ print(f" {mark} {r['id']}" + ("" if r["ok"] else f" — {r['detail']}{where}"))
404
+
405
+
406
+ def cmd_patterns(args) -> int:
407
+ for p in list_patterns():
408
+ print(
409
+ f"{p['id']:<28} {p['name']:<34} [{p['source']:<7}] "
410
+ f"fixtures: {', '.join(p['fixtures'])}"
411
+ )
412
+ return 0
413
+
414
+
415
+ def main(argv: list[str] | None = None) -> int:
416
+ init_db()
417
+ # the CLI is the batteries-included surface: installed plugins register
418
+ # their nodes/rules before any command parses a graph (the lazy top-level
419
+ # import no longer does this as a side effect — E18)
420
+ load_entry_point_plugins()
421
+ ap = argparse.ArgumentParser(
422
+ prog="evarness", description="Evarness engine — run harness graphs headless"
423
+ )
424
+ sub = ap.add_subparsers(dest="cmd", required=True)
425
+
426
+ v = sub.add_parser("validate", help="lint a graph JSON file")
427
+ v.add_argument("graph")
428
+ v.set_defaults(fn=cmd_validate)
429
+
430
+ r = sub.add_parser("run", help="execute a graph against a fixture")
431
+ r.add_argument("graph")
432
+ r.add_argument("--fixture", help="fixture YAML path (defaults to empty fixture)")
433
+ r.add_argument("--input", help="override the user input")
434
+ r.add_argument(
435
+ "--approve",
436
+ action="append",
437
+ metavar="GATE=approve|reject",
438
+ help="supply an approval_gate decision (repeatable); resumes a paused run",
439
+ )
440
+ r.add_argument("--json", action="store_true", help="machine-readable output")
441
+ r.add_argument(
442
+ "--trace-out",
443
+ metavar="PATH",
444
+ help="also export the run's trace to PATH (see --trace-format)",
445
+ )
446
+ r.add_argument(
447
+ "--trace-format",
448
+ default="jsonl",
449
+ help="trace export format for --trace-out (jsonl, otlp, or a " "plugin-registered format)",
450
+ )
451
+ r.add_argument(
452
+ "--html",
453
+ metavar="PATH",
454
+ help="also write a self-contained replay artifact "
455
+ "(canvas + playhead over the canonical events + verdicts)",
456
+ )
457
+ r.set_defaults(fn=cmd_run)
458
+
459
+ rd = sub.add_parser(
460
+ "render", help="draw a graph, pattern, or proof bundle as a self-contained HTML artifact"
461
+ )
462
+ rd.add_argument("target", help="graph.json path, pattern id, or proof.json bundle")
463
+ rd.add_argument("-o", "--out", help="output path (default: the target path with .html)")
464
+ rd.add_argument("--renderer", default="html", help="registered renderer name (default: html)")
465
+ rd.add_argument(
466
+ "--graph",
467
+ metavar="PATH",
468
+ help="bundle browsing: draw the canvas from this graph "
469
+ "(must match the bundle's pinned graph_sha256)",
470
+ )
471
+ rd.set_defaults(fn=cmd_render)
472
+
473
+ pv = sub.add_parser("prove", help="build a proof bundle: scenarios + invariants + digests")
474
+ pv.add_argument("target", help="pattern id, or path to a graph.json")
475
+ pv.add_argument(
476
+ "--fixture",
477
+ action="append",
478
+ help="fixture name (pattern) or YAML path (graph file); repeatable; "
479
+ "default: all of the pattern's fixtures",
480
+ )
481
+ pv.add_argument(
482
+ "--approve",
483
+ action="append",
484
+ metavar="GATE=approve|reject",
485
+ help="approval decision applied to every scenario (repeatable)",
486
+ )
487
+ pv.add_argument("-o", "--out", default="proof.json", help="proof bundle path")
488
+ pv.add_argument("--html", help="also write a self-contained HTML report")
489
+ pv.add_argument(
490
+ "--no-events", action="store_true", help="omit the canonical event streams (smaller bundle)"
491
+ )
492
+ pv.add_argument(
493
+ "--junit", metavar="PATH", help="also write the verdicts as JUnit XML (CI test report)"
494
+ )
495
+ pv.add_argument(
496
+ "--sarif",
497
+ metavar="PATH",
498
+ help="also write the violations as SARIF 2.1.0 (code-scanning report)",
499
+ )
500
+ pv.add_argument(
501
+ "--sign",
502
+ action="store_true",
503
+ help="attach an Ed25519 attestation (key auto-created on first "
504
+ "use; needs the [sign] extra)",
505
+ )
506
+ pv.add_argument(
507
+ "--key",
508
+ metavar="PATH",
509
+ help="signing key path (default ~/.evarness/keys/proof_ed25519.pem)",
510
+ )
511
+ pv.set_defaults(fn=cmd_prove)
512
+
513
+ vf = sub.add_parser(
514
+ "verify", help="re-check a proof bundle: digests, chains, verdict, signature"
515
+ )
516
+ vf.add_argument("bundle", help="path to a proof.json")
517
+ vf.add_argument("--pubkey", help="pin the signer: base64 key or a .pub file path")
518
+ vf.add_argument(
519
+ "--require-signature",
520
+ action="store_true",
521
+ help="fail if the bundle is unsigned or unverifiable",
522
+ )
523
+ vf.set_defaults(fn=cmd_verify)
524
+
525
+ ex = sub.add_parser(
526
+ "export",
527
+ help="unpack a verified proof bundle into standard interchange files "
528
+ "(per-scenario JSONL + OTLP traces, JUnit/SARIF verdicts, manifest)",
529
+ )
530
+ ex.add_argument("bundle", help="path to a proof.json")
531
+ ex.add_argument("-o", "--out", help="output directory (default: <bundle>-export/)")
532
+ ex.add_argument(
533
+ "--format",
534
+ action="append",
535
+ metavar="FMT",
536
+ help="trace format to export (repeatable; default: jsonl and otlp; "
537
+ "any registered exporter format works)",
538
+ )
539
+ ex.set_defaults(fn=cmd_export)
540
+
541
+ p = sub.add_parser("patterns", help="list built-in and user patterns")
542
+ p.set_defaults(fn=cmd_patterns)
543
+
544
+ args = ap.parse_args(argv)
545
+ try:
546
+ return args.fn(args)
547
+ except EvarnessError as exc:
548
+ # Refusals and misconfigurations are messages, not tracebacks (E9):
549
+ # the family covers provider refusals, graph validation, registry
550
+ # misses, and unknown export formats. Anything OUTSIDE the family is
551
+ # a genuine bug and keeps its stack trace.
552
+ print(f"error: {exc}", file=sys.stderr)
553
+ return 1
554
+
555
+
556
+ if __name__ == "__main__":
557
+ sys.exit(main())
@@ -0,0 +1,5 @@
1
+ """The Evarness kernel: graphs, events, digests, contracts, proofs.
2
+
3
+ Domain-agnostic by rule — this package never imports ``evarness.domains`` or
4
+ ``evarness.io``. Domains extend it through ``evarness.core.registry``.
5
+ """