evarness 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evarness/__init__.py +123 -0
- evarness/cli/__init__.py +557 -0
- evarness/core/__init__.py +5 -0
- evarness/core/attest.py +130 -0
- evarness/core/config.py +52 -0
- evarness/core/errors.py +47 -0
- evarness/core/executor.py +218 -0
- evarness/core/graph.py +174 -0
- evarness/core/interfaces.py +39 -0
- evarness/core/invariants.py +266 -0
- evarness/core/prove.py +714 -0
- evarness/core/registry.py +227 -0
- evarness/core/store.py +420 -0
- evarness/core/trace.py +97 -0
- evarness/domains/__init__.py +7 -0
- evarness/domains/agents/__init__.py +92 -0
- evarness/domains/agents/catalog.py +71 -0
- evarness/domains/agents/classification.py +207 -0
- evarness/domains/agents/classification.yaml +26 -0
- evarness/domains/agents/contracts.yaml +88 -0
- evarness/domains/agents/grounding.py +426 -0
- evarness/domains/agents/grounding.yaml +90 -0
- evarness/domains/agents/judges.py +220 -0
- evarness/domains/agents/judges.yaml +33 -0
- evarness/domains/agents/lint.py +39 -0
- evarness/domains/agents/nodes/__init__.py +61 -0
- evarness/domains/agents/nodes/base.py +222 -0
- evarness/domains/agents/nodes/basic.py +230 -0
- evarness/domains/agents/nodes/governance.py +719 -0
- evarness/domains/agents/nodes/loop.py +440 -0
- evarness/domains/agents/nodes/memory.py +283 -0
- evarness/domains/agents/nodes/observability.py +130 -0
- evarness/domains/agents/nodes/rag.py +138 -0
- evarness/domains/agents/nodes/tools.py +188 -0
- evarness/domains/agents/patterns/__init__.py +272 -0
- evarness/domains/agents/patterns/approval_gated_send/fixtures/send.yaml +16 -0
- evarness/domains/agents/patterns/approval_gated_send/graph.json +41 -0
- evarness/domains/agents/patterns/approval_gated_send/invariants.yaml +17 -0
- evarness/domains/agents/patterns/approval_gated_send/lesson.md +83 -0
- evarness/domains/agents/patterns/governed_email_assistant/fixtures/failure.yaml +20 -0
- evarness/domains/agents/patterns/governed_email_assistant/fixtures/happy.yaml +17 -0
- evarness/domains/agents/patterns/governed_email_assistant/graph.json +50 -0
- evarness/domains/agents/patterns/governed_email_assistant/lesson.md +12 -0
- evarness/domains/agents/patterns/single_shot_qa/fixtures/happy.yaml +7 -0
- evarness/domains/agents/patterns/single_shot_qa/graph.json +25 -0
- evarness/domains/agents/patterns/single_shot_qa/lesson.md +3 -0
- evarness/domains/agents/prompts.py +33 -0
- evarness/domains/agents/prompts.yaml +94 -0
- evarness/domains/agents/providers.py +55 -0
- evarness/domains/agents/sim.py +214 -0
- evarness/domains/agents/state.py +42 -0
- evarness/domains/agents/tiers.py +130 -0
- evarness/domains/agents/tiers.yaml +26 -0
- evarness/domains/agents/tools.yaml +138 -0
- evarness/domains/agents/toolspec.py +328 -0
- evarness/io/__init__.py +0 -0
- evarness/io/exporters.py +469 -0
- evarness/io/exporters.yaml +11 -0
- evarness/io/render.py +841 -0
- evarness/py.typed +0 -0
- evarness-0.1.0a1.dist-info/METADATA +165 -0
- evarness-0.1.0a1.dist-info/RECORD +66 -0
- evarness-0.1.0a1.dist-info/WHEEL +5 -0
- evarness-0.1.0a1.dist-info/entry_points.txt +2 -0
- evarness-0.1.0a1.dist-info/licenses/LICENSE +201 -0
- evarness-0.1.0a1.dist-info/top_level.txt +1 -0
evarness/__init__.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""evarness — prove an AI agent harness before it touches real data.
|
|
2
|
+
|
|
3
|
+
``import evarness`` gives you the batteries-included surface: the kernel plus
|
|
4
|
+
the agents domain, with plugins discovered — loaded lazily, on first
|
|
5
|
+
attribute access. For the bare kernel (e.g. when building your own domain),
|
|
6
|
+
import from ``evarness.core`` directly — it never loads a domain behind your
|
|
7
|
+
back, and ``tests/test_architecture_boundaries.py`` holds that promise.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from typing import TYPE_CHECKING, Any
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING: # static types for the lazy surface
|
|
15
|
+
from evarness.core.errors import (
|
|
16
|
+
EvarnessError as EvarnessError,
|
|
17
|
+
GraphValidationError as GraphValidationError,
|
|
18
|
+
NodeBlocked as NodeBlocked,
|
|
19
|
+
RegistryError as RegistryError,
|
|
20
|
+
RunPaused as RunPaused,
|
|
21
|
+
)
|
|
22
|
+
from evarness.core.executor import (
|
|
23
|
+
Emitter as Emitter,
|
|
24
|
+
RunContext as RunContext,
|
|
25
|
+
RunResult as RunResult,
|
|
26
|
+
execute as execute,
|
|
27
|
+
)
|
|
28
|
+
from evarness.core.graph import (
|
|
29
|
+
GraphModel as GraphModel,
|
|
30
|
+
lint as lint,
|
|
31
|
+
migrate as migrate,
|
|
32
|
+
topological_order as topological_order,
|
|
33
|
+
)
|
|
34
|
+
from evarness.core.invariants import (
|
|
35
|
+
check_invariants as check_invariants,
|
|
36
|
+
load_invariant_defs as load_invariant_defs,
|
|
37
|
+
)
|
|
38
|
+
from evarness.core.prove import prove as prove, verify_proof as verify_proof
|
|
39
|
+
from evarness.core.registry import (
|
|
40
|
+
NODE_TYPES as NODE_TYPES,
|
|
41
|
+
load_entry_point_plugins as load_entry_point_plugins,
|
|
42
|
+
)
|
|
43
|
+
from evarness.core.trace import canonical_trace as canonical_trace, trace_digest as trace_digest
|
|
44
|
+
from evarness.domains import agents as agents
|
|
45
|
+
from evarness.domains.agents.sim import Fixture as Fixture, load_fixture as load_fixture
|
|
46
|
+
|
|
47
|
+
__version__ = "0.1.0a1"
|
|
48
|
+
|
|
49
|
+
__all__ = [
|
|
50
|
+
"EvarnessError",
|
|
51
|
+
"GraphValidationError",
|
|
52
|
+
"NodeBlocked",
|
|
53
|
+
"RegistryError",
|
|
54
|
+
"RunPaused",
|
|
55
|
+
"Emitter",
|
|
56
|
+
"RunContext",
|
|
57
|
+
"RunResult",
|
|
58
|
+
"execute",
|
|
59
|
+
"GraphModel",
|
|
60
|
+
"lint",
|
|
61
|
+
"migrate",
|
|
62
|
+
"topological_order",
|
|
63
|
+
"check_invariants",
|
|
64
|
+
"load_invariant_defs",
|
|
65
|
+
"prove",
|
|
66
|
+
"verify_proof",
|
|
67
|
+
"NODE_TYPES",
|
|
68
|
+
"load_entry_point_plugins",
|
|
69
|
+
"canonical_trace",
|
|
70
|
+
"trace_digest",
|
|
71
|
+
"Fixture",
|
|
72
|
+
"load_fixture",
|
|
73
|
+
"agents",
|
|
74
|
+
"__version__",
|
|
75
|
+
]
|
|
76
|
+
|
|
77
|
+
_surface: dict[str, Any] = {}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _load_surface() -> dict[str, Any]:
|
|
81
|
+
from evarness.core import errors, executor, graph, invariants, prove, registry, trace
|
|
82
|
+
from evarness.domains import agents
|
|
83
|
+
from evarness.domains.agents import sim
|
|
84
|
+
|
|
85
|
+
registry.load_entry_point_plugins()
|
|
86
|
+
return {
|
|
87
|
+
"EvarnessError": errors.EvarnessError,
|
|
88
|
+
"GraphValidationError": errors.GraphValidationError,
|
|
89
|
+
"NodeBlocked": errors.NodeBlocked,
|
|
90
|
+
"RegistryError": errors.RegistryError,
|
|
91
|
+
"RunPaused": errors.RunPaused,
|
|
92
|
+
"Emitter": executor.Emitter,
|
|
93
|
+
"RunContext": executor.RunContext,
|
|
94
|
+
"RunResult": executor.RunResult,
|
|
95
|
+
"execute": executor.execute,
|
|
96
|
+
"GraphModel": graph.GraphModel,
|
|
97
|
+
"lint": graph.lint,
|
|
98
|
+
"migrate": graph.migrate,
|
|
99
|
+
"topological_order": graph.topological_order,
|
|
100
|
+
"check_invariants": invariants.check_invariants,
|
|
101
|
+
"load_invariant_defs": invariants.load_invariant_defs,
|
|
102
|
+
"prove": prove.prove,
|
|
103
|
+
"verify_proof": prove.verify_proof,
|
|
104
|
+
"NODE_TYPES": registry.NODE_TYPES,
|
|
105
|
+
"load_entry_point_plugins": registry.load_entry_point_plugins,
|
|
106
|
+
"canonical_trace": trace.canonical_trace,
|
|
107
|
+
"trace_digest": trace.trace_digest,
|
|
108
|
+
"Fixture": sim.Fixture,
|
|
109
|
+
"load_fixture": sim.load_fixture,
|
|
110
|
+
"agents": agents,
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def __getattr__(name: str) -> Any:
|
|
115
|
+
if name in __all__:
|
|
116
|
+
if not _surface:
|
|
117
|
+
_surface.update(_load_surface())
|
|
118
|
+
return _surface[name]
|
|
119
|
+
raise AttributeError(f"module 'evarness' has no attribute {name!r}")
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def __dir__() -> list[str]:
|
|
123
|
+
return sorted(set(globals()) | set(__all__))
|
evarness/cli/__init__.py
ADDED
|
@@ -0,0 +1,557 @@
|
|
|
1
|
+
"""evarness CLI — everything the library can do, headless.
|
|
2
|
+
|
|
3
|
+
Commands: validate | run | render | prove | verify | export | patterns.
|
|
4
|
+
Every command is recorded in the activity log (actor=cli) — full
|
|
5
|
+
traceability.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import sys
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
import yaml
|
|
16
|
+
|
|
17
|
+
from evarness.core.errors import EvarnessError
|
|
18
|
+
from evarness.core.registry import load_entry_point_plugins
|
|
19
|
+
from evarness.core.executor import GraphValidationError, execute
|
|
20
|
+
from evarness.io.exporters import export_bundle, export_trace
|
|
21
|
+
from evarness.io.render import (
|
|
22
|
+
RENDER_VERSION,
|
|
23
|
+
RenderSubject,
|
|
24
|
+
render,
|
|
25
|
+
render_proof_browser,
|
|
26
|
+
)
|
|
27
|
+
from evarness.domains.agents.nodes import REGISTRY
|
|
28
|
+
from evarness.domains.agents.nodes.base import presentation
|
|
29
|
+
from evarness.domains.agents.patterns import (
|
|
30
|
+
fixture_names,
|
|
31
|
+
fixture_text,
|
|
32
|
+
invariant_defs as pattern_invariant_defs,
|
|
33
|
+
list_patterns,
|
|
34
|
+
load_pattern,
|
|
35
|
+
)
|
|
36
|
+
from evarness.core.prove import prove, render_junit, render_proof_html, render_sarif, verify_proof
|
|
37
|
+
from evarness.core.graph import GraphModel, lint, migrate
|
|
38
|
+
from evarness.domains.agents.sim import load_fixture
|
|
39
|
+
from evarness.core.store import init_db, log_activity
|
|
40
|
+
from evarness.core.trace import canonical_trace, trace_digest
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _load_graph(path: str) -> GraphModel:
|
|
44
|
+
doc = migrate(json.loads(Path(path).read_text()))
|
|
45
|
+
return GraphModel.model_validate(doc)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _presentation_for(graph: GraphModel) -> dict:
|
|
49
|
+
"""The domain's palette as plain data — io/render stays domain-agnostic;
|
|
50
|
+
the CLI (a caller) is where domain presentation gets wired in."""
|
|
51
|
+
return {t: presentation(t) for t in {n.type for n in graph.nodes}}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def cmd_validate(args) -> int:
|
|
55
|
+
graph = _load_graph(args.graph)
|
|
56
|
+
issues = lint(graph, REGISTRY)
|
|
57
|
+
log_activity("cli.validate", graph.id, actor="cli", issues=len(issues))
|
|
58
|
+
if not issues:
|
|
59
|
+
print("OK — graph valid, no lint findings")
|
|
60
|
+
return 0
|
|
61
|
+
for i in issues:
|
|
62
|
+
print(f"{i['level'].upper():<8} [{i['code']}] {i['message']}")
|
|
63
|
+
return 1 if any(i["level"] == "error" for i in issues) else 0
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def cmd_run(args) -> int:
|
|
67
|
+
graph = _load_graph(args.graph)
|
|
68
|
+
fixture = load_fixture(args.fixture)
|
|
69
|
+
# --approve gate=decision (repeatable): supply human decisions so an
|
|
70
|
+
# approval_gate resumes instead of pausing. Replay is deterministic.
|
|
71
|
+
approvals: dict = {}
|
|
72
|
+
for pair in args.approve or []:
|
|
73
|
+
gate, _, decision = pair.partition("=")
|
|
74
|
+
approvals[gate.strip()] = decision.strip() or "approve"
|
|
75
|
+
# pattern-local contracts travel with the graph: an invariants.yaml next
|
|
76
|
+
# to the graph file is loaded as highest-precedence definitions
|
|
77
|
+
inv_file = Path(args.graph).parent / "invariants.yaml"
|
|
78
|
+
invariant_defs = None
|
|
79
|
+
if inv_file.exists():
|
|
80
|
+
doc = yaml.safe_load(inv_file.read_text(encoding="utf-8")) or {}
|
|
81
|
+
invariant_defs = doc.get("invariants") or {}
|
|
82
|
+
log_activity("cli.run", graph.id, actor="cli", fixture=fixture.scenario)
|
|
83
|
+
try:
|
|
84
|
+
run = execute(
|
|
85
|
+
graph,
|
|
86
|
+
fixture,
|
|
87
|
+
user_input=args.input,
|
|
88
|
+
approvals=approvals,
|
|
89
|
+
invariant_defs=invariant_defs,
|
|
90
|
+
)
|
|
91
|
+
except GraphValidationError as exc:
|
|
92
|
+
print("Graph invalid:", exc)
|
|
93
|
+
return 2
|
|
94
|
+
digest = trace_digest(run.events)
|
|
95
|
+
if args.json:
|
|
96
|
+
print(
|
|
97
|
+
json.dumps(
|
|
98
|
+
{
|
|
99
|
+
"id": run.id,
|
|
100
|
+
"status": run.status,
|
|
101
|
+
"output": run.output,
|
|
102
|
+
"totals": run.totals,
|
|
103
|
+
"trace_digest": digest,
|
|
104
|
+
"invariants": run.invariants,
|
|
105
|
+
"events": run.events,
|
|
106
|
+
},
|
|
107
|
+
indent=2,
|
|
108
|
+
)
|
|
109
|
+
)
|
|
110
|
+
else:
|
|
111
|
+
for ev in run.events:
|
|
112
|
+
print(
|
|
113
|
+
f"{ev['seq']:>3} {ev['type']:<22} {ev['node_id'] or '':<6} "
|
|
114
|
+
f"{json.dumps(ev['payload'])[:110]}"
|
|
115
|
+
)
|
|
116
|
+
print(f"\nSTATUS: {run.status}" + (f" ({run.reason})" if run.reason else ""))
|
|
117
|
+
print(f"TRACE: {len(run.events)} events, digest {digest}")
|
|
118
|
+
if run.invariants:
|
|
119
|
+
_print_verdicts(run.invariants)
|
|
120
|
+
if run.status == "paused" and run.pending:
|
|
121
|
+
g = run.pending["node_id"]
|
|
122
|
+
print(f"PAUSED at {g}: {run.pending['prompt']}")
|
|
123
|
+
print(f" resume with: --approve {g}=approve (or {g}=reject)")
|
|
124
|
+
print(f"OUTPUT: {run.output}")
|
|
125
|
+
if args.trace_out:
|
|
126
|
+
meta = {
|
|
127
|
+
"run_id": run.id,
|
|
128
|
+
"name": graph.name,
|
|
129
|
+
"graph_id": graph.id,
|
|
130
|
+
"status": run.status,
|
|
131
|
+
"reason": run.reason,
|
|
132
|
+
"seed": graph.params.seed,
|
|
133
|
+
"provider": graph.params.provider,
|
|
134
|
+
"fixture": fixture.scenario,
|
|
135
|
+
"trace_digest": digest,
|
|
136
|
+
}
|
|
137
|
+
doc, _ = export_trace(args.trace_format, run.events, meta)
|
|
138
|
+
Path(args.trace_out).write_text(doc)
|
|
139
|
+
if not args.json:
|
|
140
|
+
print(f"wrote {args.trace_out} ({args.trace_format})")
|
|
141
|
+
if args.html:
|
|
142
|
+
subject = RenderSubject(
|
|
143
|
+
graph=graph,
|
|
144
|
+
events=canonical_trace(run.events),
|
|
145
|
+
verdicts=run.invariants,
|
|
146
|
+
presentation=_presentation_for(graph),
|
|
147
|
+
meta={
|
|
148
|
+
"scenario": fixture.scenario,
|
|
149
|
+
"status": run.status,
|
|
150
|
+
"reason": run.reason,
|
|
151
|
+
"provider": graph.params.provider,
|
|
152
|
+
"seed": graph.params.seed,
|
|
153
|
+
"deterministic": (
|
|
154
|
+
run.events[0]["payload"].get("deterministic") if run.events else None
|
|
155
|
+
),
|
|
156
|
+
},
|
|
157
|
+
)
|
|
158
|
+
Path(args.html).write_text(render(subject))
|
|
159
|
+
if not args.json:
|
|
160
|
+
print(f"wrote {args.html} (render {RENDER_VERSION})")
|
|
161
|
+
# CI gate: a completed run with a failed invariant is a failing exit code
|
|
162
|
+
inv_failed = bool(run.invariants and run.invariants["failed"])
|
|
163
|
+
return 0 if run.status == "completed" and not inv_failed else 1
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _bundle_graph(bundle: dict, graph_arg: str | None) -> tuple[GraphModel | None, str | None]:
|
|
167
|
+
"""Resolve a canvas graph for a proof bundle, honestly: an explicit
|
|
168
|
+
``--graph`` is hash-checked inside the renderer (mismatch raises); the
|
|
169
|
+
bundle's pattern id is auto-resolved only if its hash still matches the
|
|
170
|
+
pinned subject; otherwise the canvas is omitted with a printed note —
|
|
171
|
+
never silently drawn from an unproven graph."""
|
|
172
|
+
from evarness.core.prove import graph_hash
|
|
173
|
+
|
|
174
|
+
subject = bundle.get("subject") or {}
|
|
175
|
+
if graph_arg:
|
|
176
|
+
return _load_graph(graph_arg), None # renderer enforces the hash pin
|
|
177
|
+
pattern = subject.get("pattern")
|
|
178
|
+
if pattern:
|
|
179
|
+
pattern_doc = load_pattern(pattern)
|
|
180
|
+
if pattern_doc is None:
|
|
181
|
+
return None, f"note: pattern '{pattern}' not found locally — canvas omitted"
|
|
182
|
+
graph = GraphModel.model_validate(migrate(pattern_doc))
|
|
183
|
+
if graph_hash(graph) != subject.get("graph_sha256"):
|
|
184
|
+
return None, (
|
|
185
|
+
f"note: pattern '{pattern}' on disk no longer matches the bundle's "
|
|
186
|
+
"pinned graph — canvas omitted (the canvas is only ever drawn from "
|
|
187
|
+
"the proven graph)"
|
|
188
|
+
)
|
|
189
|
+
return graph, None
|
|
190
|
+
return None, (
|
|
191
|
+
"note: the bundle pins its graph by hash only — pass --graph PATH to "
|
|
192
|
+
"draw the canvas (it must match the pinned graph_sha256)"
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def cmd_render(args) -> int:
|
|
197
|
+
"""Draw a target as a self-contained HTML artifact — no execution.
|
|
198
|
+
A graph file or pattern id gets a static canvas with lint findings
|
|
199
|
+
(a run's replay comes from ``run --html``); a proof bundle gets the
|
|
200
|
+
browsable proof: verdict badge, one viewer per scenario, the bundle's
|
|
201
|
+
not_proven section, and the whole bundle embedded for offline verify."""
|
|
202
|
+
target = args.target
|
|
203
|
+
if target.endswith(".json"):
|
|
204
|
+
try:
|
|
205
|
+
doc = json.loads(Path(target).read_text())
|
|
206
|
+
except FileNotFoundError:
|
|
207
|
+
print(f"error: file not found: {target}", file=sys.stderr)
|
|
208
|
+
return 2
|
|
209
|
+
except json.JSONDecodeError as exc:
|
|
210
|
+
print(f"error: {target}: not valid JSON: {exc}", file=sys.stderr)
|
|
211
|
+
return 2
|
|
212
|
+
else:
|
|
213
|
+
doc = None
|
|
214
|
+
|
|
215
|
+
if doc is not None and "proof_version" in doc:
|
|
216
|
+
if args.renderer != "html":
|
|
217
|
+
print(
|
|
218
|
+
f"error: proof bundles only support --renderer html (got '{args.renderer}')",
|
|
219
|
+
file=sys.stderr,
|
|
220
|
+
)
|
|
221
|
+
return 2
|
|
222
|
+
graph, note = _bundle_graph(doc, args.graph)
|
|
223
|
+
html = render_proof_browser(
|
|
224
|
+
doc, graph=graph, presentation=_presentation_for(graph) if graph else None
|
|
225
|
+
)
|
|
226
|
+
subject = doc.get("subject") or {}
|
|
227
|
+
log_activity("cli.render", str(subject.get("graph_id")), actor="cli", bundle=True)
|
|
228
|
+
out = args.out or str(Path(target).with_suffix(".html"))
|
|
229
|
+
Path(out).write_text(html)
|
|
230
|
+
if note:
|
|
231
|
+
print(note)
|
|
232
|
+
print(f"wrote {out} (proof browser, render {RENDER_VERSION})")
|
|
233
|
+
return 0
|
|
234
|
+
|
|
235
|
+
if doc is None:
|
|
236
|
+
doc = load_pattern(target)
|
|
237
|
+
if doc is None:
|
|
238
|
+
print(f"unknown target '{target}': not a .json path or a known pattern id")
|
|
239
|
+
return 2
|
|
240
|
+
graph = GraphModel.model_validate(migrate(doc))
|
|
241
|
+
issues = lint(graph, REGISTRY)
|
|
242
|
+
log_activity("cli.render", graph.id, actor="cli", issues=len(issues))
|
|
243
|
+
subject_r = RenderSubject(
|
|
244
|
+
graph=graph, presentation=_presentation_for(graph), meta={"lint": issues}
|
|
245
|
+
)
|
|
246
|
+
out = args.out or (
|
|
247
|
+
str(Path(target).with_suffix(".html")) if target.endswith(".json") else f"{target}.html"
|
|
248
|
+
)
|
|
249
|
+
Path(out).write_text(render(subject_r, args.renderer))
|
|
250
|
+
print(f"wrote {out} ({args.renderer}, render {RENDER_VERSION})")
|
|
251
|
+
return 0
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def cmd_export(args) -> int:
|
|
255
|
+
"""Unpack a proof bundle into the standard interchange set: per-scenario
|
|
256
|
+
canonical JSONL + OTLP (and any plugin format), JUnit/SARIF verdicts,
|
|
257
|
+
and a manifest with sha256 receipts. The bundle is verified first —
|
|
258
|
+
a failing bundle is refused with nothing written."""
|
|
259
|
+
bundle = json.loads(Path(args.bundle).read_text())
|
|
260
|
+
out = args.out or f"{Path(args.bundle).stem}-export"
|
|
261
|
+
formats = tuple(args.format) if args.format else ("jsonl", "otlp")
|
|
262
|
+
manifest = export_bundle(bundle, out, trace_formats=formats)
|
|
263
|
+
log_activity(
|
|
264
|
+
"cli.export",
|
|
265
|
+
str((manifest["subject"] or {}).get("graph_id")),
|
|
266
|
+
actor="cli",
|
|
267
|
+
files=len(manifest["files"]),
|
|
268
|
+
)
|
|
269
|
+
for f in manifest["files"]:
|
|
270
|
+
print(f" {f['path']:<36} {f['kind']:<12} sha256:{f['sha256'][:12]}…")
|
|
271
|
+
skipped = [s["fixture"] for s in manifest["scenarios"] if not s["events_exported"]]
|
|
272
|
+
if skipped:
|
|
273
|
+
print(
|
|
274
|
+
f" (no events embedded for: {', '.join(skipped)} — named by digest "
|
|
275
|
+
"in the manifest, not exported)"
|
|
276
|
+
)
|
|
277
|
+
print(f"wrote {out}/manifest.json (export {manifest['export_version']}, verified bundle)")
|
|
278
|
+
return 0
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def cmd_prove(args) -> int:
|
|
282
|
+
"""Build a proof bundle: run every scenario twice, check invariants,
|
|
283
|
+
emit proof.json (+ optional HTML). Exit 0 only when the proof holds."""
|
|
284
|
+
approvals: dict = {}
|
|
285
|
+
for pair in args.approve or []:
|
|
286
|
+
gate, _, decision = pair.partition("=")
|
|
287
|
+
approvals[gate.strip()] = decision.strip() or "approve"
|
|
288
|
+
|
|
289
|
+
pattern_doc = load_pattern(args.target) if not args.target.endswith(".json") else None
|
|
290
|
+
if pattern_doc:
|
|
291
|
+
graph = GraphModel.model_validate(migrate(pattern_doc))
|
|
292
|
+
names = args.fixture or fixture_names(args.target)
|
|
293
|
+
scenarios = [(n, fixture_text(args.target, n) or "") for n in names]
|
|
294
|
+
missing = [n for n, t in scenarios if not t]
|
|
295
|
+
if missing:
|
|
296
|
+
print(f"fixtures not found for pattern '{args.target}': {', '.join(missing)}")
|
|
297
|
+
return 2
|
|
298
|
+
inv_defs = pattern_invariant_defs(args.target) or None
|
|
299
|
+
pattern_id = args.target
|
|
300
|
+
else:
|
|
301
|
+
graph = _load_graph(args.target)
|
|
302
|
+
if not args.fixture:
|
|
303
|
+
print("--fixture PATH is required when proving a graph file")
|
|
304
|
+
return 2
|
|
305
|
+
scenarios = [(Path(f).stem, Path(f).read_text()) for f in args.fixture]
|
|
306
|
+
inv_file = Path(args.target).parent / "invariants.yaml"
|
|
307
|
+
inv_defs = (
|
|
308
|
+
(yaml.safe_load(inv_file.read_text()) or {}).get("invariants")
|
|
309
|
+
if inv_file.exists()
|
|
310
|
+
else None
|
|
311
|
+
)
|
|
312
|
+
pattern_id = None
|
|
313
|
+
|
|
314
|
+
log_activity("cli.prove", graph.id, actor="cli", scenarios=[n for n, _ in scenarios])
|
|
315
|
+
proof = prove(
|
|
316
|
+
graph,
|
|
317
|
+
scenarios,
|
|
318
|
+
pattern_id=pattern_id,
|
|
319
|
+
invariant_defs=inv_defs,
|
|
320
|
+
approvals=approvals,
|
|
321
|
+
include_events=not args.no_events,
|
|
322
|
+
)
|
|
323
|
+
if args.sign:
|
|
324
|
+
from evarness.core.attest import AttestationError, sign_proof
|
|
325
|
+
|
|
326
|
+
try:
|
|
327
|
+
proof = sign_proof(proof, key_path=args.key)
|
|
328
|
+
except AttestationError as exc:
|
|
329
|
+
print(f"cannot sign: {exc}")
|
|
330
|
+
return 2
|
|
331
|
+
att = proof["attestation"]
|
|
332
|
+
if att["key_created"]:
|
|
333
|
+
print(
|
|
334
|
+
f"new signing key created at {att['key_path']} "
|
|
335
|
+
f"(share {att['key_path'].removesuffix('.pem')}.pub out of band)"
|
|
336
|
+
)
|
|
337
|
+
|
|
338
|
+
Path(args.out).write_text(json.dumps(proof, indent=2))
|
|
339
|
+
if args.html:
|
|
340
|
+
Path(args.html).write_text(render_proof_html(proof))
|
|
341
|
+
if args.junit:
|
|
342
|
+
Path(args.junit).write_text(render_junit(proof))
|
|
343
|
+
if args.sarif:
|
|
344
|
+
Path(args.sarif).write_text(render_sarif(proof))
|
|
345
|
+
|
|
346
|
+
v = proof["verdict"]
|
|
347
|
+
for sc in proof["scenarios"]:
|
|
348
|
+
inv = sc["invariants"]
|
|
349
|
+
inv_txt = f"{inv['passed']}\u2713/{inv['failed']}\u2717" if inv else "-"
|
|
350
|
+
repro = {True: "reproduced", False: "NOT REPRODUCED", None: "-"}[sc["reproduced"]]
|
|
351
|
+
print(
|
|
352
|
+
f" {sc['fixture']:<16} {sc['status']:<10} invariants {inv_txt:<8} "
|
|
353
|
+
f"{repro:<15} {sc['trace_digest'][:24]}\u2026"
|
|
354
|
+
)
|
|
355
|
+
state = {True: "HOLDS", False: "FAILED", None: "PENDING"}[v["ok"]]
|
|
356
|
+
print(
|
|
357
|
+
f"PROOF: {state} \u2014 "
|
|
358
|
+
f"{v['scenarios']} scenario(s), invariants_pass={v['invariants_pass']}, "
|
|
359
|
+
f"reproduced={v['reproduced']}" + (f" ({v['note']})" if v.get("note") else "")
|
|
360
|
+
)
|
|
361
|
+
extras = [p for p in (args.html, args.junit, args.sarif) if p]
|
|
362
|
+
print(f"wrote {args.out}" + (f" and {', '.join(extras)}" if extras else ""))
|
|
363
|
+
return 0 if v["ok"] else 1
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def cmd_verify(args) -> int:
|
|
367
|
+
"""Re-check a proof bundle without running anything (digests, chains,
|
|
368
|
+
verdict consistency, signature). The reviewer-side half of `prove`."""
|
|
369
|
+
doc = json.loads(Path(args.bundle).read_text())
|
|
370
|
+
pubkey = args.pubkey
|
|
371
|
+
if pubkey and Path(pubkey).is_file(): # accept a .pub file or raw b64
|
|
372
|
+
pubkey = Path(pubkey).read_text().strip()
|
|
373
|
+
result = verify_proof(doc, pubkey_b64=pubkey, require_signature=args.require_signature)
|
|
374
|
+
log_activity("cli.verify", str(args.bundle), actor="cli", ok=result["ok"])
|
|
375
|
+
for c in result["checks"]:
|
|
376
|
+
mark = {True: "✓", False: "✗", None: "-"}[c["ok"]]
|
|
377
|
+
where = f" [{c['scenario']}]" if c.get("scenario") else ""
|
|
378
|
+
print(f" {mark} {c['check']}{where} — {c['detail']}")
|
|
379
|
+
print(
|
|
380
|
+
f"VERIFY: {'OK' if result['ok'] else 'FAILED'} — bundle is "
|
|
381
|
+
+ ("internally consistent" if result["ok"] else "inconsistent or tampered; see ✗ above")
|
|
382
|
+
)
|
|
383
|
+
print(
|
|
384
|
+
"note: verification proves consistency and (if signed) integrity — "
|
|
385
|
+
"not that the runs happened; that trust reduces to the signer"
|
|
386
|
+
)
|
|
387
|
+
return 0 if result["ok"] else 1
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _print_verdicts(inv: dict) -> None:
|
|
391
|
+
print(
|
|
392
|
+
f"INVARIANTS: {inv['passed']} passed, {inv['failed']} FAILED"
|
|
393
|
+
if inv["failed"]
|
|
394
|
+
else f"INVARIANTS: {inv['passed']} passed"
|
|
395
|
+
)
|
|
396
|
+
for r in inv["results"]:
|
|
397
|
+
mark = "✓" if r["ok"] else "✗"
|
|
398
|
+
where = (
|
|
399
|
+
f" (seq {', '.join(map(str, r['evidence_seq']))})"
|
|
400
|
+
if not r["ok"] and r["evidence_seq"]
|
|
401
|
+
else ""
|
|
402
|
+
)
|
|
403
|
+
print(f" {mark} {r['id']}" + ("" if r["ok"] else f" — {r['detail']}{where}"))
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def cmd_patterns(args) -> int:
|
|
407
|
+
for p in list_patterns():
|
|
408
|
+
print(
|
|
409
|
+
f"{p['id']:<28} {p['name']:<34} [{p['source']:<7}] "
|
|
410
|
+
f"fixtures: {', '.join(p['fixtures'])}"
|
|
411
|
+
)
|
|
412
|
+
return 0
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def main(argv: list[str] | None = None) -> int:
|
|
416
|
+
init_db()
|
|
417
|
+
# the CLI is the batteries-included surface: installed plugins register
|
|
418
|
+
# their nodes/rules before any command parses a graph (the lazy top-level
|
|
419
|
+
# import no longer does this as a side effect — E18)
|
|
420
|
+
load_entry_point_plugins()
|
|
421
|
+
ap = argparse.ArgumentParser(
|
|
422
|
+
prog="evarness", description="Evarness engine — run harness graphs headless"
|
|
423
|
+
)
|
|
424
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
425
|
+
|
|
426
|
+
v = sub.add_parser("validate", help="lint a graph JSON file")
|
|
427
|
+
v.add_argument("graph")
|
|
428
|
+
v.set_defaults(fn=cmd_validate)
|
|
429
|
+
|
|
430
|
+
r = sub.add_parser("run", help="execute a graph against a fixture")
|
|
431
|
+
r.add_argument("graph")
|
|
432
|
+
r.add_argument("--fixture", help="fixture YAML path (defaults to empty fixture)")
|
|
433
|
+
r.add_argument("--input", help="override the user input")
|
|
434
|
+
r.add_argument(
|
|
435
|
+
"--approve",
|
|
436
|
+
action="append",
|
|
437
|
+
metavar="GATE=approve|reject",
|
|
438
|
+
help="supply an approval_gate decision (repeatable); resumes a paused run",
|
|
439
|
+
)
|
|
440
|
+
r.add_argument("--json", action="store_true", help="machine-readable output")
|
|
441
|
+
r.add_argument(
|
|
442
|
+
"--trace-out",
|
|
443
|
+
metavar="PATH",
|
|
444
|
+
help="also export the run's trace to PATH (see --trace-format)",
|
|
445
|
+
)
|
|
446
|
+
r.add_argument(
|
|
447
|
+
"--trace-format",
|
|
448
|
+
default="jsonl",
|
|
449
|
+
help="trace export format for --trace-out (jsonl, otlp, or a " "plugin-registered format)",
|
|
450
|
+
)
|
|
451
|
+
r.add_argument(
|
|
452
|
+
"--html",
|
|
453
|
+
metavar="PATH",
|
|
454
|
+
help="also write a self-contained replay artifact "
|
|
455
|
+
"(canvas + playhead over the canonical events + verdicts)",
|
|
456
|
+
)
|
|
457
|
+
r.set_defaults(fn=cmd_run)
|
|
458
|
+
|
|
459
|
+
rd = sub.add_parser(
|
|
460
|
+
"render", help="draw a graph, pattern, or proof bundle as a self-contained HTML artifact"
|
|
461
|
+
)
|
|
462
|
+
rd.add_argument("target", help="graph.json path, pattern id, or proof.json bundle")
|
|
463
|
+
rd.add_argument("-o", "--out", help="output path (default: the target path with .html)")
|
|
464
|
+
rd.add_argument("--renderer", default="html", help="registered renderer name (default: html)")
|
|
465
|
+
rd.add_argument(
|
|
466
|
+
"--graph",
|
|
467
|
+
metavar="PATH",
|
|
468
|
+
help="bundle browsing: draw the canvas from this graph "
|
|
469
|
+
"(must match the bundle's pinned graph_sha256)",
|
|
470
|
+
)
|
|
471
|
+
rd.set_defaults(fn=cmd_render)
|
|
472
|
+
|
|
473
|
+
pv = sub.add_parser("prove", help="build a proof bundle: scenarios + invariants + digests")
|
|
474
|
+
pv.add_argument("target", help="pattern id, or path to a graph.json")
|
|
475
|
+
pv.add_argument(
|
|
476
|
+
"--fixture",
|
|
477
|
+
action="append",
|
|
478
|
+
help="fixture name (pattern) or YAML path (graph file); repeatable; "
|
|
479
|
+
"default: all of the pattern's fixtures",
|
|
480
|
+
)
|
|
481
|
+
pv.add_argument(
|
|
482
|
+
"--approve",
|
|
483
|
+
action="append",
|
|
484
|
+
metavar="GATE=approve|reject",
|
|
485
|
+
help="approval decision applied to every scenario (repeatable)",
|
|
486
|
+
)
|
|
487
|
+
pv.add_argument("-o", "--out", default="proof.json", help="proof bundle path")
|
|
488
|
+
pv.add_argument("--html", help="also write a self-contained HTML report")
|
|
489
|
+
pv.add_argument(
|
|
490
|
+
"--no-events", action="store_true", help="omit the canonical event streams (smaller bundle)"
|
|
491
|
+
)
|
|
492
|
+
pv.add_argument(
|
|
493
|
+
"--junit", metavar="PATH", help="also write the verdicts as JUnit XML (CI test report)"
|
|
494
|
+
)
|
|
495
|
+
pv.add_argument(
|
|
496
|
+
"--sarif",
|
|
497
|
+
metavar="PATH",
|
|
498
|
+
help="also write the violations as SARIF 2.1.0 (code-scanning report)",
|
|
499
|
+
)
|
|
500
|
+
pv.add_argument(
|
|
501
|
+
"--sign",
|
|
502
|
+
action="store_true",
|
|
503
|
+
help="attach an Ed25519 attestation (key auto-created on first "
|
|
504
|
+
"use; needs the [sign] extra)",
|
|
505
|
+
)
|
|
506
|
+
pv.add_argument(
|
|
507
|
+
"--key",
|
|
508
|
+
metavar="PATH",
|
|
509
|
+
help="signing key path (default ~/.evarness/keys/proof_ed25519.pem)",
|
|
510
|
+
)
|
|
511
|
+
pv.set_defaults(fn=cmd_prove)
|
|
512
|
+
|
|
513
|
+
vf = sub.add_parser(
|
|
514
|
+
"verify", help="re-check a proof bundle: digests, chains, verdict, signature"
|
|
515
|
+
)
|
|
516
|
+
vf.add_argument("bundle", help="path to a proof.json")
|
|
517
|
+
vf.add_argument("--pubkey", help="pin the signer: base64 key or a .pub file path")
|
|
518
|
+
vf.add_argument(
|
|
519
|
+
"--require-signature",
|
|
520
|
+
action="store_true",
|
|
521
|
+
help="fail if the bundle is unsigned or unverifiable",
|
|
522
|
+
)
|
|
523
|
+
vf.set_defaults(fn=cmd_verify)
|
|
524
|
+
|
|
525
|
+
ex = sub.add_parser(
|
|
526
|
+
"export",
|
|
527
|
+
help="unpack a verified proof bundle into standard interchange files "
|
|
528
|
+
"(per-scenario JSONL + OTLP traces, JUnit/SARIF verdicts, manifest)",
|
|
529
|
+
)
|
|
530
|
+
ex.add_argument("bundle", help="path to a proof.json")
|
|
531
|
+
ex.add_argument("-o", "--out", help="output directory (default: <bundle>-export/)")
|
|
532
|
+
ex.add_argument(
|
|
533
|
+
"--format",
|
|
534
|
+
action="append",
|
|
535
|
+
metavar="FMT",
|
|
536
|
+
help="trace format to export (repeatable; default: jsonl and otlp; "
|
|
537
|
+
"any registered exporter format works)",
|
|
538
|
+
)
|
|
539
|
+
ex.set_defaults(fn=cmd_export)
|
|
540
|
+
|
|
541
|
+
p = sub.add_parser("patterns", help="list built-in and user patterns")
|
|
542
|
+
p.set_defaults(fn=cmd_patterns)
|
|
543
|
+
|
|
544
|
+
args = ap.parse_args(argv)
|
|
545
|
+
try:
|
|
546
|
+
return args.fn(args)
|
|
547
|
+
except EvarnessError as exc:
|
|
548
|
+
# Refusals and misconfigurations are messages, not tracebacks (E9):
|
|
549
|
+
# the family covers provider refusals, graph validation, registry
|
|
550
|
+
# misses, and unknown export formats. Anything OUTSIDE the family is
|
|
551
|
+
# a genuine bug and keeps its stack trace.
|
|
552
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
553
|
+
return 1
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
if __name__ == "__main__":
|
|
557
|
+
sys.exit(main())
|