agentcrash-core 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
agentcrash/__init__.py ADDED
@@ -0,0 +1,45 @@
1
+ """AgentCrash — installable reliability framework for agentic workflows.
2
+
3
+ Core package: SDK instrumentation, versioned events, generic invariants,
4
+ fault policies, and the control-plane server. Domain logic (ODRL, DSPy,
5
+ SHACL) lives in host apps / extensions, never in core imports.
6
+ """
7
+
8
+ from agentcrash.evaluate import compare_reports, evaluate_trace
9
+ from agentcrash.events import SCHEMA_VERSION, make_event, validate_event
10
+ from agentcrash.invariants import DuplicateMutatingEffect, RequiredStep
11
+ from agentcrash.sdk import AgentCrash, FaultPolicy, RunContext, WorkflowConfig
12
+ from agentcrash.server import (
13
+ evaluation_summary,
14
+ ingest_events,
15
+ prune_runs,
16
+ register_project,
17
+ reliability_summary,
18
+ reset_store,
19
+ run_detail,
20
+ submit_evaluation,
21
+ )
22
+
23
+ __version__ = "0.2.0"
24
+
25
+ __all__ = [
26
+ "SCHEMA_VERSION",
27
+ "AgentCrash",
28
+ "DuplicateMutatingEffect",
29
+ "FaultPolicy",
30
+ "RequiredStep",
31
+ "RunContext",
32
+ "WorkflowConfig",
33
+ "compare_reports",
34
+ "evaluate_trace",
35
+ "evaluation_summary",
36
+ "ingest_events",
37
+ "make_event",
38
+ "prune_runs",
39
+ "register_project",
40
+ "reliability_summary",
41
+ "reset_store",
42
+ "run_detail",
43
+ "submit_evaluation",
44
+ "validate_event",
45
+ ]
agentcrash/__main__.py ADDED
@@ -0,0 +1,6 @@
1
+ """Allow `python -m agentcrash ...` (see docs 13 §9 and 14 §2 quickstarts)."""
2
+
3
+ from agentcrash.cli import main
4
+
5
+ if __name__ == "__main__":
6
+ raise SystemExit(main())
@@ -0,0 +1,68 @@
1
+ """Domain-free invariant registry: verdict type, plugin protocol, lookup.
2
+
3
+ Split out of :mod:`agentcrash.verifiers` so the installable core can
4
+ register and evaluate invariants without importing ODRL/SHACL stacks.
5
+ ``verifiers.py`` re-exports every name defined here for backward
6
+ compatibility.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from typing import Any, Protocol
12
+
13
+
14
+ class VerifierError(Exception):
15
+ """Harness bug: the trace/state cannot be judged (exit 3, never a verdict)."""
16
+
17
+
18
+ class UnknownVerifierError(ValueError):
19
+ """Unknown invariant id requested (``E_CONFIG``, exit 2)."""
20
+
21
+
22
+ class TraceCorruptError(ValueError):
23
+ """Structural trace/state problem found by strict validation (exit 3)."""
24
+
25
+
26
+ class Verdict:
27
+ """One invariant judgement with human-readable evidence."""
28
+
29
+ def __init__(self, invariant_id: str, passed: bool, evidence: str) -> None:
30
+ self.invariant_id = invariant_id
31
+ self.passed = passed
32
+ self.evidence = evidence
33
+
34
+ def to_dict(self) -> dict[str, Any]:
35
+ """Return the JSON-serializable verdict row."""
36
+ return {"id": self.invariant_id, "passed": self.passed, "evidence": self.evidence}
37
+
38
+
39
+ class Verifier(Protocol):
40
+ """Plugin interface: pure check over ``(trace, state)``."""
41
+
42
+ id: str
43
+
44
+ def check(self, trace: list[dict[str, Any]], state: dict[str, Any]) -> Verdict: ...
45
+
46
+
47
+ _REGISTRY: dict[str, Verifier] = {}
48
+
49
+
50
+ def register_verifier(verifier: Verifier) -> None:
51
+ """Register a verifier under its invariant id."""
52
+ _REGISTRY[verifier.id] = verifier
53
+
54
+
55
+ def get_verifier(invariant_id: str) -> Verifier:
56
+ """Return the verifier for ``invariant_id`` or raise ``E_CONFIG``."""
57
+ try:
58
+ return _REGISTRY[invariant_id]
59
+ except KeyError:
60
+ known = ", ".join(sorted(_REGISTRY)) or "none registered"
61
+ raise UnknownVerifierError(
62
+ f"unknown invariant {invariant_id!r}; known verifiers: {known}"
63
+ ) from None
64
+
65
+
66
+ def verifier_ids() -> list[str]:
67
+ """Return the registered invariant ids in registration order."""
68
+ return list(_REGISTRY)
agentcrash/cli.py ADDED
@@ -0,0 +1,274 @@
1
+ """AgentCrash core command-line entry point: ``check`` + ``serve``.
2
+
3
+ ``check`` verifies registry invariants against an SDK-schema JSONL trace;
4
+ ``serve`` runs the loopback control plane (Lab + ``/v1`` API).
5
+
6
+ Host-owned commands (``run`` / ``diagnose`` / ``replay`` / ``seed`` /
7
+ ``optimize-prompts``) live in the host CLI (e.g. ``odrl_agent.cli``) and
8
+ reuse the shared helpers below (exit codes, :class:`ConfigError`,
9
+ ``resolve_scenario`` / ``resolve_policy`` / ``load_yaml`` /
10
+ ``validate_run_inputs``). Core never imports host or domain stacks.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import argparse
16
+ import json
17
+ import sys
18
+ from pathlib import Path
19
+ from typing import Any
20
+
21
+ import yaml
22
+
23
+ from agentcrash import __version__
24
+ from agentcrash.injector import FaultSchedule, UnknownFaultError
25
+ from agentcrash.verifiers import (
26
+ TraceCorruptError,
27
+ UnknownVerifierError,
28
+ VerifierError,
29
+ load_state_snapshot,
30
+ load_trace_events,
31
+ run_verifiers,
32
+ )
33
+
34
+ EXIT_OK = 0
35
+ EXIT_VERDICT_FAIL = 1
36
+ EXIT_USAGE = 2
37
+ EXIT_CORRUPT = 3
38
+ EXIT_POLICY_REJECTED = 4
39
+ EXIT_RUNABORTED = 5
40
+
41
+
42
+ class ConfigError(ValueError):
43
+ """Bad flags, missing files, or invalid scenario/policy (E_CONFIG/E_SCHEMA)."""
44
+
45
+
46
+ def resolve_scenario(ref: str) -> Path:
47
+ """Resolve a scenario id or path to its scenario.yaml file."""
48
+ direct = Path(ref)
49
+ if direct.is_file():
50
+ return direct
51
+ candidates = [
52
+ Path("scenarios") / ref / "scenario.yaml",
53
+ Path("scenarios") / f"{ref}.yaml",
54
+ ]
55
+ for candidate in candidates:
56
+ if candidate.is_file():
57
+ return candidate
58
+ raise ConfigError(
59
+ f"unknown --scenario {ref!r}: tried {direct}, " + ", ".join(str(c) for c in candidates)
60
+ )
61
+
62
+
63
+ def resolve_policy(ref: str | None, scenario_path: Path, scenario: dict[str, Any]) -> Path:
64
+ """Resolve the policy override, the scenario default, or fail."""
65
+ raw = ref or str(scenario.get("policy", ""))
66
+ if not raw:
67
+ raise ConfigError("no policy: pass --policy or set scenario 'policy'")
68
+ direct = Path(raw)
69
+ if direct.is_file():
70
+ return direct
71
+ sibling = scenario_path.parent / raw
72
+ if sibling.is_file():
73
+ return sibling
74
+ raise ConfigError(f"policy file not found: {raw!r}")
75
+
76
+
77
+ def load_yaml(path: Path) -> tuple[dict[str, Any], str]:
78
+ """Parse a YAML mapping file; return (data, raw text)."""
79
+ try:
80
+ text = path.read_text(encoding="utf-8")
81
+ except OSError as exc:
82
+ raise ConfigError(f"cannot read {path}: {exc}") from exc
83
+ try:
84
+ data = yaml.safe_load(text)
85
+ except yaml.YAMLError as exc:
86
+ raise ConfigError(f"malformed YAML in {path}: {exc}") from exc
87
+ if not isinstance(data, dict):
88
+ raise ConfigError(f"{path} must contain a YAML mapping")
89
+ return data, text
90
+
91
+
92
+ def validate_run_inputs(scenario: dict[str, Any], policy: dict[str, Any]) -> tuple[int, int]:
93
+ """Light structural validation; returns (seed, max_steps)."""
94
+ for key in ("scenario", "task", "tools", "policy"):
95
+ if key not in scenario:
96
+ raise ConfigError(f"scenario is missing required key {key!r}")
97
+ if not isinstance(scenario["tools"], list) or not scenario["tools"]:
98
+ raise ConfigError("scenario 'tools' must be a non-empty list")
99
+ if policy.get("schema_version") != "policy/v1":
100
+ raise ConfigError(
101
+ f"policy schema_version must be 'policy/v1' (got {policy.get('schema_version')!r})"
102
+ )
103
+ try:
104
+ FaultSchedule.from_scenario(scenario)
105
+ except UnknownFaultError as exc:
106
+ raise ConfigError(str(exc)) from exc
107
+ seed = scenario.get("seed", 42)
108
+ max_steps = scenario.get("max_steps", 12)
109
+ if type(seed) is not int or type(max_steps) is not int:
110
+ raise ConfigError("scenario 'seed' and 'max_steps' must be integers")
111
+ return seed, max_steps
112
+
113
+
114
+ def cmd_check(argv: list[str]) -> int:
115
+ """Execute ``agentcrash check`` (exit 0 PASS, 1 VERDICT_FAIL, 3 corrupt).
116
+
117
+ Verifies registry invariants against an SDK-schema JSONL trace.
118
+ Invariant ids come from ``--invariant`` (repeatable, at least one);
119
+ hosts pass their own ids explicitly instead of via scenario files.
120
+ """
121
+ parser = argparse.ArgumentParser(prog="agentcrash check")
122
+ parser.add_argument("--trace", required=True)
123
+ parser.add_argument("--invariant", action="append", default=[])
124
+ parser.add_argument(
125
+ "--plugin",
126
+ action="append",
127
+ default=[],
128
+ help="dotted module to import before judging (registers host "
129
+ "invariants, e.g. odrl_agent.verifiers); repeatable",
130
+ )
131
+ parser.add_argument("--state", default=None)
132
+ parser.add_argument("--report-out", default=None)
133
+ parser.add_argument("--format", choices=("text", "json"), default="text")
134
+ parser.add_argument(
135
+ "--scan-pii",
136
+ action="store_true",
137
+ help="scan the raw trace + state files for leaked emails/card-like "
138
+ "runs (redaction placeholders exempt); exit non-zero on any hit",
139
+ )
140
+ args = parser.parse_args(argv)
141
+ as_json = args.format == "json"
142
+
143
+ def fail(code: str, hint: str, exit_code: int) -> int:
144
+ if as_json:
145
+ print(
146
+ json.dumps(
147
+ {"api_version": "v1", "code": code, "exit": exit_code, "hint": hint},
148
+ sort_keys=True,
149
+ )
150
+ )
151
+ else:
152
+ print(f"agentcrash check: {code}: {hint}", file=sys.stderr)
153
+ return exit_code
154
+
155
+ def emit(text: str) -> None:
156
+ print(text, file=sys.stderr if as_json else sys.stdout)
157
+
158
+ if not args.invariant:
159
+ return fail(
160
+ "E_CONFIG",
161
+ "no invariants: pass at least one --invariant <id>",
162
+ EXIT_USAGE,
163
+ )
164
+ for plugin in args.plugin:
165
+ try:
166
+ __import__(plugin)
167
+ except ImportError as exc:
168
+ return fail("E_CONFIG", f"cannot import --plugin {plugin!r}: {exc}", EXIT_USAGE)
169
+ trace_path = Path(args.trace)
170
+ if not trace_path.is_file():
171
+ return fail("E_CONFIG", f"trace file not found: {args.trace!r}", EXIT_USAGE)
172
+ try:
173
+ events = load_trace_events(str(trace_path))
174
+ except TraceCorruptError as exc:
175
+ emit(f"trace {trace_path} is corrupt: {exc}")
176
+ return fail("E_TRACE_CORRUPT", str(exc), EXIT_CORRUPT)
177
+
178
+ state: dict[str, Any] = {}
179
+ state_ref = args.state
180
+ if state_ref:
181
+ if not Path(state_ref).is_file():
182
+ return fail("E_CONFIG", f"state file not found: {state_ref!r}", EXIT_USAGE)
183
+ try:
184
+ state = load_state_snapshot(state_ref)
185
+ except TraceCorruptError as exc:
186
+ emit(f"state {state_ref} is corrupt: {exc}")
187
+ return fail("E_TRACE_CORRUPT", str(exc), EXIT_CORRUPT)
188
+
189
+ try:
190
+ outcome = run_verifiers(events, state, args.invariant)
191
+ except UnknownVerifierError as exc:
192
+ return fail("E_CONFIG", str(exc), EXIT_USAGE)
193
+ except VerifierError as exc:
194
+ emit(f"trace {trace_path} cannot be judged: {exc}")
195
+ return fail("E_TRACE_CORRUPT", str(exc), EXIT_CORRUPT)
196
+
197
+ run_id = str(events[0].get("run_id", "core-check")) if events else "core-check"
198
+ report = {
199
+ "api_version": "v1",
200
+ "results": outcome["results"],
201
+ "run_id": run_id,
202
+ "verdict": outcome["verdict"],
203
+ }
204
+ report_out = args.report_out or str(Path("reports") / f"{run_id}.verdict.json")
205
+ Path(report_out).parent.mkdir(parents=True, exist_ok=True)
206
+ Path(report_out).write_text(
207
+ json.dumps(report, sort_keys=True, indent=2) + "\n", encoding="utf-8"
208
+ )
209
+
210
+ if args.scan_pii:
211
+ from agentcrash.scan_pii import format_hits, scan_files
212
+
213
+ targets = [str(trace_path)] + ([state_ref] if state_ref else [])
214
+ hits = scan_files(targets)
215
+ if hits:
216
+ emit(f"PII scan: {len(hits)} raw-PII hit(s) in trace/state:")
217
+ emit(format_hits(hits))
218
+ return fail(
219
+ "E_PII_LEAK",
220
+ f"{len(hits)} raw-PII hit(s); see locations above",
221
+ EXIT_VERDICT_FAIL,
222
+ )
223
+ emit(f"PII scan: clean ({', '.join(targets)})")
224
+
225
+ for row in report["results"]:
226
+ mark = "PASS" if row["passed"] else "FAIL"
227
+ verb = "satisfied" if row["passed"] else "violated!"
228
+ emit(f"[{mark}] Invariant {row['id']} {verb} ({row['evidence']})")
229
+ emit(f"verdict={report['verdict']} report={report_out}")
230
+ if as_json:
231
+ print(json.dumps(report, sort_keys=True, indent=2))
232
+ return EXIT_OK if report["verdict"] == "PASS" else EXIT_VERDICT_FAIL
233
+
234
+
235
+ def cmd_serve(argv: list[str]) -> int:
236
+ """Serve the AgentCrash control plane (Lab + ``/v1`` API, local only)."""
237
+ parser = argparse.ArgumentParser(prog="agentcrash serve")
238
+ parser.add_argument("--port", type=int, default=8901)
239
+ parser.add_argument("--bind", default="127.0.0.1")
240
+ args = parser.parse_args(argv)
241
+
242
+ from agentcrash.server import serve
243
+
244
+ server, port = serve(args.bind, args.port)
245
+ print(f"AgentCrash control plane on http://{args.bind}:{port}/ (Lab + /v1 API)")
246
+ try:
247
+ server.serve_forever()
248
+ except KeyboardInterrupt:
249
+ pass
250
+ return EXIT_OK
251
+
252
+
253
+ def main(argv: list[str] | None = None) -> int:
254
+ """Dispatch subcommands: check, serve."""
255
+ raw = list(sys.argv[1:] if argv is None else argv)
256
+ if "--version" in raw:
257
+ print(f"agentcrash {__version__}")
258
+ return EXIT_OK
259
+ if not raw or raw[0] in ("-h", "--help"):
260
+ print("usage: agentcrash [--version] {check, serve} ...")
261
+ print(" check: verify registry invariants against an SDK trace")
262
+ print(" serve: run the control plane (Lab + /v1 API, local only)")
263
+ return EXIT_OK if raw else EXIT_USAGE
264
+ command, rest = raw[0], raw[1:]
265
+ if command == "check":
266
+ return cmd_check(rest)
267
+ if command == "serve":
268
+ return cmd_serve(rest)
269
+ print(f"agentcrash: unknown command {command!r}", file=sys.stderr)
270
+ return EXIT_USAGE
271
+
272
+
273
+ if __name__ == "__main__":
274
+ raise SystemExit(main())
agentcrash/clock.py ADDED
@@ -0,0 +1,37 @@
1
+ """Virtual clock: the only time source for synthetic tools (ADR-003, ADR-006).
2
+
3
+ All time is integer virtual milliseconds. The clock starts at ``t=0`` and
4
+ advances only via :meth:`VirtualClock.sleep`. Wall-clock time is never read
5
+ here (see ``tests/test_acr001.py``).
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+
11
+ class VirtualClock:
12
+ """Harness-owned virtual clock (tick 50ms, ``t=0`` at run start)."""
13
+
14
+ def __init__(self, tick_ms: int = 50, t: int = 0) -> None:
15
+ if tick_ms <= 0:
16
+ raise ValueError("tick_ms must be positive")
17
+ if t < 0:
18
+ raise ValueError("t must be non-negative")
19
+ self.tick_ms = tick_ms
20
+ self._t = t
21
+
22
+ def now(self) -> int:
23
+ """Return the current virtual time in ms."""
24
+ return self._t
25
+
26
+ def sleep(self, ms: int) -> int:
27
+ """Advance virtual time by ``ms`` and return the new time."""
28
+ if ms < 0:
29
+ raise ValueError("sleep ms must be non-negative")
30
+ self._t += ms
31
+ return self._t
32
+
33
+ def deadline(self, ms: int) -> int:
34
+ """Return the virtual timestamp ``ms`` after now (clock unchanged)."""
35
+ if ms < 0:
36
+ raise ValueError("deadline ms must be non-negative")
37
+ return self._t + ms