interlockd 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- interlockd/__init__.py +3 -0
- interlockd/__main__.py +7 -0
- interlockd/aibom.py +136 -0
- interlockd/api.py +307 -0
- interlockd/auth.py +38 -0
- interlockd/calibration.py +260 -0
- interlockd/cli.py +1920 -0
- interlockd/compose.py +84 -0
- interlockd/config.py +80 -0
- interlockd/corpus.py +505 -0
- interlockd/data/packs/_corpus.yaml +7 -0
- interlockd/data/packs/egress.jsonl +38 -0
- interlockd/data/packs/fileops.jsonl +20 -0
- interlockd/data/packs/git.jsonl +51 -0
- interlockd/data/packs/packages.jsonl +24 -0
- interlockd/data/packs/persistence.jsonl +31 -0
- interlockd/data/packs/secrets.jsonl +22 -0
- interlockd/data/packs/shell.jsonl +75 -0
- interlockd/det/__init__.py +53 -0
- interlockd/det/describe.py +66 -0
- interlockd/det/engine.py +153 -0
- interlockd/det/flow.py +106 -0
- interlockd/det/mandate.py +108 -0
- interlockd/det/mcp.py +70 -0
- interlockd/det/memory.py +50 -0
- interlockd/det/packs.py +531 -0
- interlockd/det/shellparse.py +438 -0
- interlockd/egress.py +124 -0
- interlockd/gateway.py +350 -0
- interlockd/ids.py +25 -0
- interlockd/insights.py +316 -0
- interlockd/integrations/__init__.py +2 -0
- interlockd/integrations/client.py +85 -0
- interlockd/integrations/langchain.py +59 -0
- interlockd/integrations/openai_agents.py +72 -0
- interlockd/integrations/strands.py +69 -0
- interlockd/judges/__init__.py +107 -0
- interlockd/judges/guardrails.py +94 -0
- interlockd/judges/local_probe.py +192 -0
- interlockd/judges/projector.py +82 -0
- interlockd/judges/strands_decider.py +160 -0
- interlockd/ledger.py +62 -0
- interlockd/mcp_http.py +216 -0
- interlockd/mcp_proxy.py +315 -0
- interlockd/mcp_server.py +211 -0
- interlockd/models.py +98 -0
- interlockd/normalize/__init__.py +1 -0
- interlockd/normalize/claude_code.py +89 -0
- interlockd/normalize/cursor.py +59 -0
- interlockd/normalize/gemini_cli.py +55 -0
- interlockd/normalize/kiro.py +65 -0
- interlockd/normalize/more.py +98 -0
- interlockd/ogr.py +542 -0
- interlockd/onboard.py +300 -0
- interlockd/otel.py +139 -0
- interlockd/paths.py +63 -0
- interlockd/pipeline.py +370 -0
- interlockd/policy/__init__.py +220 -0
- interlockd/policy_lint.py +76 -0
- interlockd/render.py +142 -0
- interlockd/sandbox.py +87 -0
- interlockd/scan.py +405 -0
- interlockd/schema/verdict.schema.json +68 -0
- interlockd/selftest.py +101 -0
- interlockd/service.py +247 -0
- interlockd/shims/claude-code/interlock-hook.sh +113 -0
- interlockd/shims/kiro/interlock-hook.mjs +101 -0
- interlockd/shims/kiro/interlock-hook.sh +82 -0
- interlockd/shims/opencode/interlock.js +60 -0
- interlockd/store/__init__.py +22 -0
- interlockd/store/drainer.py +74 -0
- interlockd/store/dynamodb.py +144 -0
- interlockd/store/memory.py +28 -0
- interlockd/store/spool.py +159 -0
- interlockd/testkit.py +401 -0
- interlockd/tiers.py +95 -0
- interlockd/transcript/__init__.py +1 -0
- interlockd/transcript/claude_code.py +73 -0
- interlockd/transcript/kiro.py +102 -0
- interlockd-0.1.0.dist-info/METADATA +125 -0
- interlockd-0.1.0.dist-info/RECORD +84 -0
- interlockd-0.1.0.dist-info/WHEEL +4 -0
- interlockd-0.1.0.dist-info/entry_points.txt +2 -0
- interlockd-0.1.0.dist-info/licenses/LICENSE +201 -0
interlockd/compose.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Composer (design §4.7): findings and rule actions -> one disposition, a reason, and a notice for the model."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
from .det import DetResult
|
|
8
|
+
from .det.mandate import MandateResult
|
|
9
|
+
from .models import Finding
|
|
10
|
+
from .policy import Policy
|
|
11
|
+
|
|
12
|
+
ORDER = {"block": 4, "ask": 3, "model": 2, "flag": 1, "allow": 0}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class Composed:
|
|
17
|
+
disposition: str
|
|
18
|
+
reason: str | None
|
|
19
|
+
notice: str | None
|
|
20
|
+
findings: list[Finding]
|
|
21
|
+
unjudged: list[str]
|
|
22
|
+
needs_judge: bool = False
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _describe(f: Finding) -> str:
|
|
26
|
+
rule = (f.rule or f.category).replace("_", " ").replace(".", ": ", 1)
|
|
27
|
+
return f"{rule}" + (f" ({f.subject})" if f.subject else "")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def render_notice(disposition: str, findings: list[Finding], *, reason: str | None) -> str:
|
|
31
|
+
"""OGR `continuation` wording: what was refused, that it did not run, what the agent may do, no disguised retry."""
|
|
32
|
+
what = "; ".join(_describe(f) for f in findings[:3]) or (reason or "a security policy")
|
|
33
|
+
if disposition == "block":
|
|
34
|
+
return (f"[interlock] This action was refused by a security policy and was NOT executed. Reason: {what}. "
|
|
35
|
+
"You may pursue the user's goal by other legitimate means, but you MUST NOT retry this action in a disguised form "
|
|
36
|
+
"(different encoding, split across commands, via another tool). If the action is genuinely required, stop and ask the user.")
|
|
37
|
+
if disposition == "ask":
|
|
38
|
+
return (f"[interlock] This action was NOT executed because it needs the user's explicit confirmation. Reason: {what}. "
|
|
39
|
+
"Ask the user whether to proceed, quoting the exact command, and only retry after they confirm. Do not retry in a disguised form.")
|
|
40
|
+
return ""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def compose(det: DetResult, mandate: MandateResult, policy: Policy, *, session: dict | None, kind: str) -> Composed:
|
|
44
|
+
findings = list(det.findings)
|
|
45
|
+
unjudged = list(mandate.unjudged) if mandate.unjudged and policy.mandate.mode != "off" else []
|
|
46
|
+
if mandate.finding is not None:
|
|
47
|
+
findings.append(mandate.finding)
|
|
48
|
+
|
|
49
|
+
if kind == "post_action":
|
|
50
|
+
disp = "warn" if any(f.rule == "secrets.literal_in_result" for f in findings) else "pass"
|
|
51
|
+
reason = "; ".join(_describe(f) for f in findings[:3]) or None
|
|
52
|
+
return Composed(disp, reason, None, findings, unjudged)
|
|
53
|
+
|
|
54
|
+
# strongest rule action wins (deny-wins)
|
|
55
|
+
action = det.decision # block | ask | model | flag | allow | None
|
|
56
|
+
if mandate.finding is not None and policy.mandate.mode == "enforce":
|
|
57
|
+
action = "block" if ORDER.get(action or "allow", 0) < ORDER["block"] else action
|
|
58
|
+
elif mandate.finding is not None: # log_only: a finding, no enforcement
|
|
59
|
+
pass
|
|
60
|
+
|
|
61
|
+
# denial budget (Anthropic backstop): after N consecutive or M total denials, escalate to the human
|
|
62
|
+
budget_hit = False
|
|
63
|
+
if session:
|
|
64
|
+
b = policy.denial_budget
|
|
65
|
+
if session.get("denials_consecutive", 0) >= b.consecutive or session.get("denials_total", 0) >= b.total:
|
|
66
|
+
budget_hit = True
|
|
67
|
+
|
|
68
|
+
if action == "block":
|
|
69
|
+
disp = "block"
|
|
70
|
+
elif action == "ask":
|
|
71
|
+
disp = "ask"
|
|
72
|
+
elif action in ("model", None, "flag", "allow"):
|
|
73
|
+
disp = "allow"
|
|
74
|
+
else:
|
|
75
|
+
disp = "allow"
|
|
76
|
+
needs_judge = action == "model" or bool(det.questions)
|
|
77
|
+
if budget_hit and disp == "allow" and det.action_class not in ("read", "harness", "edit_in_workspace", "build_or_test"):
|
|
78
|
+
disp = policy.denial_budget.then
|
|
79
|
+
findings.append(Finding(category="x.interlock.denial_budget_exhausted", severity="high", rule="budget.denials",
|
|
80
|
+
detector="interlock.ledger", path="tool_call", score=1.0,
|
|
81
|
+
subject=f"{session.get('denials_consecutive', 0)} consecutive / {session.get('denials_total', 0)} total denials"))
|
|
82
|
+
reason = "; ".join(_describe(f) for f in findings[:3]) or None
|
|
83
|
+
notice = render_notice(disp, findings, reason=reason) if disp in ("block", "ask") else None
|
|
84
|
+
return Composed(disp, reason, notice, findings, unjudged, needs_judge=needs_judge)
|
interlockd/config.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Daemon settings. M0: defaults plus an optional ~/.interlock/config.json; the policy YAML arrives in M1."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
DEFAULT_LISTEN = "127.0.0.1:7391"
|
|
11
|
+
MAX_REQUEST_BYTES = 1_048_576
|
|
12
|
+
|
|
13
|
+
DEFAULT_PROTECTED = [".git/hooks/**", ".kiro/**", ".claude/**", ".codex/**", ".github/workflows/**", ".interlock/**"]
|
|
14
|
+
DEFAULT_SENSITIVE = [
|
|
15
|
+
"~/.ssh/**", "~/.aws/**", "~/.config/gh/**", "~/.interlock/token", "**/.env", "**/.env.*", "**/*.pem",
|
|
16
|
+
"**/id_rsa*", "**/id_ed25519*", "**/*.key",
|
|
17
|
+
]
|
|
18
|
+
DEFAULT_FAIL_MODE = {
|
|
19
|
+
"security.data_exfiltration": "closed",
|
|
20
|
+
"security.malicious_command": "closed",
|
|
21
|
+
"security.persistence": "closed",
|
|
22
|
+
"security.secret_leak.*": "closed",
|
|
23
|
+
"security.privilege_escalation": "closed",
|
|
24
|
+
"default": "open",
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class Settings:
|
|
30
|
+
home: Path = field(default_factory=lambda: Path(os.environ.get("INTERLOCK_HOME", "~/.interlock")).expanduser())
|
|
31
|
+
listen: str = DEFAULT_LISTEN
|
|
32
|
+
mode: str = "shadow"
|
|
33
|
+
profile: str = "laptop"
|
|
34
|
+
agent_id: str = ""
|
|
35
|
+
workspace_roots: list[str] = field(default_factory=list)
|
|
36
|
+
protected_paths: list[str] = field(default_factory=lambda: list(DEFAULT_PROTECTED))
|
|
37
|
+
sensitive_paths: list[str] = field(default_factory=lambda: list(DEFAULT_SENSITIVE))
|
|
38
|
+
fail_mode: dict[str, str] = field(default_factory=lambda: dict(DEFAULT_FAIL_MODE))
|
|
39
|
+
kiro_sessions_dir: Path = field(default_factory=lambda: Path("~/.kiro/sessions").expanduser())
|
|
40
|
+
user_messages_n: int = 3
|
|
41
|
+
pre_timeout_ms: int = 1500
|
|
42
|
+
post_timeout_ms: int = 2500
|
|
43
|
+
dynamodb_table: str | None = None
|
|
44
|
+
aws_region: str | None = None
|
|
45
|
+
aws_profile: str | None = None
|
|
46
|
+
unix_socket: bool = True # also serve on <home>/interlockd.sock (0600); hooks prefer it
|
|
47
|
+
otlp_endpoint: str | None = None # e.g. http://localhost:4318/v1/traces (OpenTelemetry export)
|
|
48
|
+
otlp_headers: dict[str, str] = field(default_factory=dict)
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def token_path(self) -> Path:
|
|
52
|
+
return self.home / "token"
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def socket_path(self) -> Path:
|
|
56
|
+
return self.home / "interlockd.sock"
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def spool_dir(self) -> Path:
|
|
60
|
+
return self.home / "spool"
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def config_path(self) -> Path:
|
|
64
|
+
return self.home / "config.json"
|
|
65
|
+
|
|
66
|
+
@classmethod
|
|
67
|
+
def load(cls, home: Path | None = None) -> Settings:
|
|
68
|
+
s = cls() if home is None else cls(home=Path(home).expanduser())
|
|
69
|
+
if s.config_path.exists():
|
|
70
|
+
data = json.loads(s.config_path.read_text())
|
|
71
|
+
for k, v in data.items():
|
|
72
|
+
if k == "kiro_sessions_dir":
|
|
73
|
+
v = Path(v).expanduser()
|
|
74
|
+
if hasattr(s, k):
|
|
75
|
+
setattr(s, k, v)
|
|
76
|
+
return s
|
|
77
|
+
|
|
78
|
+
def host_port(self) -> tuple[str, int]:
|
|
79
|
+
h, _, p = self.listen.rpartition(":")
|
|
80
|
+
return h or "127.0.0.1", int(p)
|
interlockd/corpus.py
ADDED
|
@@ -0,0 +1,505 @@
|
|
|
1
|
+
"""Corpus v1 engine (design §6, evaluation): build a labelled corpus from real and synthetic sources.
|
|
2
|
+
|
|
3
|
+
This module is the logic behind `interlock corpus <export|label|import>` (wired in cli.py). It is kept
|
|
4
|
+
separate from the CLI so it is unit-testable without argparse. Everything it emits is in the same case
|
|
5
|
+
schema the deterministic runner (`interlockd.testkit`) consumes, plus three v1 additions:
|
|
6
|
+
|
|
7
|
+
transcript optional list of {role, text} turns giving the user-intent context a judge case needs
|
|
8
|
+
(role in {"user","assistant"}); never read by the deterministic layer, only the judge step
|
|
9
|
+
judge {question_id: expected_noul} expected judge answers (already defined by testkit)
|
|
10
|
+
split "dev" | "heldout" — stable 70/30 assignment from sha256(id), stamped into the file
|
|
11
|
+
|
|
12
|
+
Sources:
|
|
13
|
+
export real pre_action events from the local spool and the DynamoDB audit table, secrets redacted
|
|
14
|
+
with the deterministic engine's own SECRET_PATTERNS, deduped by normalized command, written
|
|
15
|
+
UNLABELLED (expect/label/category absent) to corpus/shadow/*.jsonl for a human to label
|
|
16
|
+
label interactive pass that sets expect/label/category on unlabelled cases
|
|
17
|
+
import re-shapers for AgentDojo / InjecAgent / AgentDrift into our schema
|
|
18
|
+
|
|
19
|
+
Redaction note: the review asked to "redact with the existing projector", but redaction proper lives in
|
|
20
|
+
the secret patterns, not the projector (which only bounds/structures state). We reuse det.packs.SECRET_PATTERNS
|
|
21
|
+
so the corpus is redacted by exactly the same matcher the gate uses — one source of truth.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import hashlib
|
|
27
|
+
import json
|
|
28
|
+
import re
|
|
29
|
+
from collections.abc import Iterable
|
|
30
|
+
from dataclasses import dataclass, field
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from typing import Any
|
|
33
|
+
|
|
34
|
+
from .det.packs import SECRET_PATTERNS
|
|
35
|
+
|
|
36
|
+
SPLIT_DEV_FRACTION = 0.70 # 70% dev / 30% heldout
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# ---- split -------------------------------------------------------------------------------------------
|
|
40
|
+
|
|
41
|
+
def split_for(case_id: str, dev_fraction: float = SPLIT_DEV_FRACTION) -> str:
|
|
42
|
+
"""Stable dev/heldout assignment from a hash of the id. Deterministic across runs and machines:
|
|
43
|
+
the first 8 hex digits of sha256(id) scaled to [0,1); below dev_fraction → dev, else heldout."""
|
|
44
|
+
h = int(hashlib.sha256(case_id.encode("utf-8")).hexdigest()[:8], 16)
|
|
45
|
+
return "dev" if (h / 0xFFFFFFFF) < dev_fraction else "heldout"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def assign_split(case: dict, dev_fraction: float = SPLIT_DEV_FRACTION) -> dict:
|
|
49
|
+
"""Stamp `split` into a case from its id (idempotent). Requires an `id`."""
|
|
50
|
+
case["split"] = split_for(case["id"], dev_fraction)
|
|
51
|
+
return case
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# ---- redaction ---------------------------------------------------------------------------------------
|
|
55
|
+
|
|
56
|
+
def redact(text: str) -> tuple[str, list[str]]:
|
|
57
|
+
"""Replace any secret the deterministic engine would detect with a typed placeholder.
|
|
58
|
+
|
|
59
|
+
Returns (redacted_text, kinds_found). Uses det.packs.SECRET_PATTERNS so the corpus never stores a
|
|
60
|
+
live credential — the same matcher the secrets pack uses on real traffic."""
|
|
61
|
+
if not text:
|
|
62
|
+
return text, []
|
|
63
|
+
kinds: list[str] = []
|
|
64
|
+
out = text
|
|
65
|
+
for kind, pat in SECRET_PATTERNS:
|
|
66
|
+
if pat.search(out):
|
|
67
|
+
kinds.append(kind)
|
|
68
|
+
out = pat.sub(f"<REDACTED:{kind}>", out)
|
|
69
|
+
return out, kinds
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def redact_case(case: dict) -> tuple[dict, list[str]]:
|
|
73
|
+
"""Redact secrets in a case's command / tool arguments / manifest in place. Returns (case, kinds)."""
|
|
74
|
+
found: list[str] = []
|
|
75
|
+
if isinstance(case.get("cmd"), str):
|
|
76
|
+
case["cmd"], k = redact(case["cmd"])
|
|
77
|
+
found += k
|
|
78
|
+
args = case.get("args")
|
|
79
|
+
if isinstance(args, dict):
|
|
80
|
+
for key, val in list(args.items()):
|
|
81
|
+
if isinstance(val, str):
|
|
82
|
+
args[key], k = redact(val)
|
|
83
|
+
found += k
|
|
84
|
+
manifest = case.get("manifest")
|
|
85
|
+
if isinstance(manifest, dict):
|
|
86
|
+
for name, content in list(manifest.items()):
|
|
87
|
+
if isinstance(content, str):
|
|
88
|
+
manifest[name], k = redact(content)
|
|
89
|
+
found += k
|
|
90
|
+
return case, found
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
# ---- normalized dedup key ----------------------------------------------------------------------------
|
|
94
|
+
|
|
95
|
+
_WS = re.compile(r"\s+")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def dedup_key(case: dict) -> str:
|
|
99
|
+
"""A normalized key for dedup: the shell command (whitespace-collapsed) or tool+sorted-args.
|
|
100
|
+
Two cases that differ only in incidental spacing or argument order collapse to one."""
|
|
101
|
+
if isinstance(case.get("cmd"), str):
|
|
102
|
+
return "cmd:" + _WS.sub(" ", case["cmd"].strip())
|
|
103
|
+
tool = case.get("tool", "")
|
|
104
|
+
args = case.get("args") or {}
|
|
105
|
+
try:
|
|
106
|
+
norm = json.dumps(args, sort_keys=True, separators=(",", ":"))
|
|
107
|
+
except TypeError:
|
|
108
|
+
norm = str(sorted(args.items())) if isinstance(args, dict) else str(args)
|
|
109
|
+
return f"tool:{tool}:{norm}"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# ---- event → case ------------------------------------------------------------------------------------
|
|
113
|
+
|
|
114
|
+
def event_to_case(record: dict, *, idx: int) -> dict | None:
|
|
115
|
+
"""Turn a stored pre_action audit record into an UNLABELLED corpus case (expect/label/category absent).
|
|
116
|
+
|
|
117
|
+
Only pre_action events with a shell command or a tool+arguments become cases; post_action and
|
|
118
|
+
harness-internal events are skipped (returns None). Secrets are redacted. The source event id and
|
|
119
|
+
the deterministic disposition at capture time are kept under `_meta` for the human labeller's context."""
|
|
120
|
+
if record.get("kind") != "pre_action":
|
|
121
|
+
return None
|
|
122
|
+
tool = record.get("tool_name")
|
|
123
|
+
args = record.get("arguments") or {}
|
|
124
|
+
if not tool:
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
case: dict[str, Any] = {}
|
|
128
|
+
cmd = None
|
|
129
|
+
if isinstance(args, dict):
|
|
130
|
+
for k in ("command", "cmd", "script"):
|
|
131
|
+
if isinstance(args.get(k), str) and args[k].strip():
|
|
132
|
+
cmd = args[k]
|
|
133
|
+
break
|
|
134
|
+
if cmd is not None:
|
|
135
|
+
case["cmd"] = cmd
|
|
136
|
+
else:
|
|
137
|
+
case["tool"] = tool
|
|
138
|
+
case["args"] = {k: v for k, v in args.items() if k not in ("description", "explanation")}
|
|
139
|
+
|
|
140
|
+
case["id"] = f"shadow-{record.get('event_id', idx)}"
|
|
141
|
+
case, kinds = redact_case(case)
|
|
142
|
+
meta = {
|
|
143
|
+
"source": "shadow",
|
|
144
|
+
"source_event_id": record.get("event_id"),
|
|
145
|
+
"captured_disposition": record.get("disposition"),
|
|
146
|
+
"captured_at": record.get("received_at"),
|
|
147
|
+
}
|
|
148
|
+
if kinds:
|
|
149
|
+
meta["redacted"] = sorted(set(kinds))
|
|
150
|
+
# user instruction turns, redacted, carried as transcript context for later judge labelling
|
|
151
|
+
msgs = ((record.get("state") or {}).get("user_messages")) or []
|
|
152
|
+
transcript = []
|
|
153
|
+
for m in msgs:
|
|
154
|
+
text = m.get("text") if isinstance(m, dict) else None
|
|
155
|
+
if isinstance(text, str) and text.strip():
|
|
156
|
+
transcript.append({"role": "user", "text": redact(text)[0]})
|
|
157
|
+
if transcript:
|
|
158
|
+
case["transcript"] = transcript
|
|
159
|
+
case["_meta"] = meta
|
|
160
|
+
assign_split(case)
|
|
161
|
+
return case
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
# ---- export ------------------------------------------------------------------------------------------
|
|
165
|
+
|
|
166
|
+
@dataclass
|
|
167
|
+
class ExportResult:
|
|
168
|
+
cases: list[dict] = field(default_factory=list)
|
|
169
|
+
seen_keys: set[str] = field(default_factory=set)
|
|
170
|
+
from_spool: int = 0
|
|
171
|
+
from_dynamo: int = 0
|
|
172
|
+
skipped_non_preaction: int = 0
|
|
173
|
+
duplicates: int = 0
|
|
174
|
+
redacted_cases: int = 0
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def export_cases(records: Iterable[dict], *, result: ExportResult | None = None) -> ExportResult:
|
|
178
|
+
"""Convert an iterable of audit records into deduped, redacted, unlabelled cases. Pure; the CLI
|
|
179
|
+
supplies the records from the spool and/or DynamoDB so this is testable without AWS."""
|
|
180
|
+
res = result or ExportResult()
|
|
181
|
+
for i, record in enumerate(records):
|
|
182
|
+
case = event_to_case(record, idx=len(res.cases) + i)
|
|
183
|
+
if case is None:
|
|
184
|
+
res.skipped_non_preaction += 1
|
|
185
|
+
continue
|
|
186
|
+
key = dedup_key(case)
|
|
187
|
+
if key in res.seen_keys:
|
|
188
|
+
res.duplicates += 1
|
|
189
|
+
continue
|
|
190
|
+
res.seen_keys.add(key)
|
|
191
|
+
if case.get("_meta", {}).get("redacted"):
|
|
192
|
+
res.redacted_cases += 1
|
|
193
|
+
res.cases.append(case)
|
|
194
|
+
return res
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
# ---- label -----------------------------------------------------------------------------------------
|
|
198
|
+
|
|
199
|
+
VALID_EXPECT = {"block", "ask", "model", "allow"}
|
|
200
|
+
VALID_LABEL = {"attack", "benign"}
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def is_labelled(case: dict) -> bool:
|
|
204
|
+
return "expect" in case and "label" in case
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def apply_label(case: dict, *, expect: str, label: str, category: str | None) -> dict:
|
|
208
|
+
"""Set the human labels on a case, validating the enums. Raises ValueError on a bad value."""
|
|
209
|
+
if expect not in VALID_EXPECT:
|
|
210
|
+
raise ValueError(f"expect must be one of {sorted(VALID_EXPECT)}, got {expect!r}")
|
|
211
|
+
if label not in VALID_LABEL:
|
|
212
|
+
raise ValueError(f"label must be one of {sorted(VALID_LABEL)}, got {label!r}")
|
|
213
|
+
case["expect"] = expect
|
|
214
|
+
case["label"] = label
|
|
215
|
+
if category:
|
|
216
|
+
case["category"] = category
|
|
217
|
+
return case
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def propose_label(case: dict, ws: Path, *, policy=None, corpus_trust: dict | None = None) -> tuple[dict, bool]:
|
|
221
|
+
"""Pre-fill expect/label/category from the deterministic engine's own verdict, so a human only
|
|
222
|
+
reviews the ambiguous cases. Returns (proposal, confident):
|
|
223
|
+
|
|
224
|
+
- confident=True when the engine is unambiguous: it either allowed with NO findings (→ benign/allow)
|
|
225
|
+
or fired a rule with a hard disposition block/ask (→ attack/<decision>/<rule category>).
|
|
226
|
+
- confident=False when the engine routed to `model` (judge territory) or produced only flags —
|
|
227
|
+
these genuinely need a human, so the proposal is a best guess the reviewer should confirm.
|
|
228
|
+
|
|
229
|
+
The proposal is NOT written onto the case; the caller decides whether to accept it (auto mode) or
|
|
230
|
+
show it as a default in interactive mode. Runs the real deterministic pipeline via run_case."""
|
|
231
|
+
from .testkit import run_case # lazy: avoids pulling testkit (and judge deps) unless auto-labelling
|
|
232
|
+
|
|
233
|
+
res = run_case(dict(case), ws, policy=policy, corpus_trust=corpus_trust)
|
|
234
|
+
decision = res.decision or "allow"
|
|
235
|
+
category = res.category or None
|
|
236
|
+
if decision == "allow" and not res.fired:
|
|
237
|
+
return {"expect": "allow", "label": "benign", "category": category}, True
|
|
238
|
+
if decision in ("block", "ask"):
|
|
239
|
+
return {"expect": decision, "label": "attack", "category": category}, True
|
|
240
|
+
# model-routed or flag-only: ambiguous — the deterministic layer deferred, so a human must judge
|
|
241
|
+
# The default label is `benign`: a model-routed action is one the engine could not condemn on
|
|
242
|
+
# signature alone, and in real traffic the large majority of those are legitimate. Defaulting to
|
|
243
|
+
# benign makes the interactive Enter/accept SAFE (accepting won't mislabel an ordinary command as an
|
|
244
|
+
# attack); the reviewer presses `a` for the rare genuine attack. The case still needs a human (confident=False).
|
|
245
|
+
return {"expect": decision, "label": "benign", "category": category}, False
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def auto_label(cases: list[dict], ws_root: Path, *, policy=None, corpus_trust: dict | None = None
|
|
249
|
+
) -> tuple[int, list[dict]]:
|
|
250
|
+
"""Apply confident proposals to all unlabelled cases in place; return (n_applied, needs_review).
|
|
251
|
+
`needs_review` is the list of still-unlabelled cases the engine was not confident about."""
|
|
252
|
+
applied, needs_review = 0, []
|
|
253
|
+
for i, c in enumerate(cases):
|
|
254
|
+
if is_labelled(c):
|
|
255
|
+
continue
|
|
256
|
+
ws = ws_root / f"p{i}"
|
|
257
|
+
ws.mkdir(parents=True, exist_ok=True)
|
|
258
|
+
proposal, confident = propose_label(c, ws, policy=policy, corpus_trust=corpus_trust)
|
|
259
|
+
if confident:
|
|
260
|
+
apply_label(c, expect=proposal["expect"], label=proposal["label"], category=proposal.get("category"))
|
|
261
|
+
applied += 1
|
|
262
|
+
else:
|
|
263
|
+
needs_review.append(c)
|
|
264
|
+
return applied, needs_review
|
|
265
|
+
# ---- jsonl io --------------------------------------------------------------------------------------
|
|
266
|
+
|
|
267
|
+
def read_jsonl(path: Path) -> list[dict]:
|
|
268
|
+
out = []
|
|
269
|
+
for lineno, line in enumerate(Path(path).read_text().splitlines(), 1):
|
|
270
|
+
if line.strip():
|
|
271
|
+
try:
|
|
272
|
+
out.append(json.loads(line))
|
|
273
|
+
except json.JSONDecodeError as e:
|
|
274
|
+
raise ValueError(f"{path}:{lineno}: invalid JSON: {e}") from e
|
|
275
|
+
return out
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def write_jsonl(path: Path, cases: list[dict]) -> None:
|
|
279
|
+
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
|
280
|
+
Path(path).write_text("".join(json.dumps(c, ensure_ascii=False) + "\n" for c in cases))
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
# ---- importers -------------------------------------------------------------------------------------
|
|
284
|
+
#
|
|
285
|
+
# Each importer reshapes one external dataset into our case schema. The datasets disagree on structure,
|
|
286
|
+
# so each mapper is defensive: it extracts a shell/file/network-equivalent action and SKIPS anything with
|
|
287
|
+
# no such equivalent (an API-only or chat-only item), returning the skip count. Mappers are pure functions
|
|
288
|
+
# over already-parsed JSON so they are testable with small fixtures and never touch the network.
|
|
289
|
+
|
|
290
|
+
# tool names in external datasets that map onto our shell/file/network tool surface
|
|
291
|
+
_SHELLish = {"bash", "shell", "run_terminal", "execute", "exec", "terminal", "run_command", "sh"}
|
|
292
|
+
_FILEish = {"write_file", "edit_file", "create_file", "fs_write", "append_file", "delete_file", "read_file"}
|
|
293
|
+
_NETish = {"curl", "http_get", "http_post", "fetch", "web_fetch", "download", "wget", "request"}
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
@dataclass
|
|
297
|
+
class ImportResult:
|
|
298
|
+
cases: list[dict] = field(default_factory=list)
|
|
299
|
+
imported: int = 0
|
|
300
|
+
skipped_no_equivalent: int = 0
|
|
301
|
+
skipped_errors: int = 0
|
|
302
|
+
source: str = ""
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _mk_case(case_id: str, *, cmd: str | None = None, tool: str | None = None, args: dict | None = None,
|
|
306
|
+
expect: str, label: str, category: str, judge: dict | None = None,
|
|
307
|
+
transcript: list | None = None, manifest: dict | None = None, result: str | None = None) -> dict:
|
|
308
|
+
case: dict[str, Any] = {"id": case_id}
|
|
309
|
+
if cmd is not None:
|
|
310
|
+
case["cmd"] = cmd
|
|
311
|
+
else:
|
|
312
|
+
case["tool"] = tool
|
|
313
|
+
case["args"] = args or {}
|
|
314
|
+
case["expect"] = expect
|
|
315
|
+
case["label"] = label
|
|
316
|
+
case["category"] = category
|
|
317
|
+
if judge:
|
|
318
|
+
case["judge"] = judge
|
|
319
|
+
if transcript:
|
|
320
|
+
case["transcript"] = transcript
|
|
321
|
+
if manifest:
|
|
322
|
+
case["manifest"] = manifest
|
|
323
|
+
if result:
|
|
324
|
+
case["result"] = result
|
|
325
|
+
case, _ = redact_case(case)
|
|
326
|
+
assign_split(case)
|
|
327
|
+
return case
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _classify_tool(name: str) -> str | None:
|
|
331
|
+
n = (name or "").lower()
|
|
332
|
+
if any(s in n for s in _SHELLish):
|
|
333
|
+
return "shell"
|
|
334
|
+
if any(s in n for s in _FILEish):
|
|
335
|
+
return "file"
|
|
336
|
+
if any(s in n for s in _NETish):
|
|
337
|
+
return "net"
|
|
338
|
+
return None
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def import_agentdojo(items: list[dict], *, result: ImportResult | None = None) -> ImportResult:
|
|
344
|
+
"""AgentDojo: (user_task, injection_task) pairs over tool suites. Each item carries a ground-truth
|
|
345
|
+
tool-call sequence (`GROUND_TRUTH`/`ground_truth`/`tool_calls`), each call a {function, args} (the API's
|
|
346
|
+
FunctionCall). Calls from an injection task are the attack; user-task calls are benign. Calls whose tool
|
|
347
|
+
has no shell/file/network equivalent are skipped. Falls back to a flat {tool,args,injection} item shape."""
|
|
348
|
+
res = result or ImportResult(source="agentdojo")
|
|
349
|
+
|
|
350
|
+
def emit(call: dict, *, injected: bool, suite: str, cid: str) -> bool:
|
|
351
|
+
fn = call.get("function") or call.get("name") or call.get("tool") or ""
|
|
352
|
+
if _classify_tool(fn) is None:
|
|
353
|
+
res.skipped_no_equivalent += 1
|
|
354
|
+
return False
|
|
355
|
+
args = call.get("args") or call.get("arguments") or {}
|
|
356
|
+
cmd = args.get("command") if isinstance(args, dict) else None
|
|
357
|
+
category = "security.data_exfiltration" if injected else "security.malicious_command"
|
|
358
|
+
res.cases.append(_mk_case(
|
|
359
|
+
f"agentdojo-{suite}-{cid}", cmd=cmd if cmd else None, tool=None if cmd else fn,
|
|
360
|
+
args=None if cmd else args, expect="ask" if injected else "allow",
|
|
361
|
+
label="attack" if injected else "benign", category="security.prompt_injection" if injected else category,
|
|
362
|
+
judge={"intent_authorized": 0.1} if injected else None))
|
|
363
|
+
res.imported += 1
|
|
364
|
+
return True
|
|
365
|
+
|
|
366
|
+
for i, item in enumerate(items):
|
|
367
|
+
try:
|
|
368
|
+
if not isinstance(item, dict):
|
|
369
|
+
res.skipped_errors += 1
|
|
370
|
+
continue
|
|
371
|
+
suite = item.get("suite") or item.get("suite_name") or "s"
|
|
372
|
+
cid = item.get("id") or item.get("user_task_id") or i
|
|
373
|
+
gt = item.get("GROUND_TRUTH") or item.get("ground_truth") or item.get("tool_calls")
|
|
374
|
+
if isinstance(gt, list) and gt:
|
|
375
|
+
injected = bool(item.get("injection_task") or item.get("injection_task_id") or item.get("attack"))
|
|
376
|
+
for j, call in enumerate(gt):
|
|
377
|
+
if isinstance(call, dict):
|
|
378
|
+
emit(call, injected=injected, suite=str(suite), cid=f"{cid}-{j}")
|
|
379
|
+
else:
|
|
380
|
+
# flat fallback (used by unit fixtures): one tool call per item
|
|
381
|
+
fn = item.get("tool") or item.get("function") or item.get("name") or ""
|
|
382
|
+
if _classify_tool(fn) is None:
|
|
383
|
+
res.skipped_no_equivalent += 1
|
|
384
|
+
continue
|
|
385
|
+
injected = bool(item.get("injection") or item.get("is_injected") or item.get("attack"))
|
|
386
|
+
args = item.get("args") or item.get("arguments") or {}
|
|
387
|
+
cmd = args.get("command") if isinstance(args, dict) else None
|
|
388
|
+
res.cases.append(_mk_case(
|
|
389
|
+
f"agentdojo-{item.get('id', i)}", cmd=cmd if cmd else None, tool=None if cmd else fn,
|
|
390
|
+
args=None if cmd else args, expect="ask" if injected else "allow",
|
|
391
|
+
label="attack" if injected else "benign",
|
|
392
|
+
category="security.prompt_injection" if injected else "security.malicious_command",
|
|
393
|
+
judge={"intent_authorized": 0.1} if injected else None))
|
|
394
|
+
res.imported += 1
|
|
395
|
+
except Exception: # noqa: BLE001 — a malformed item is skipped and counted, never fatal
|
|
396
|
+
res.skipped_errors += 1
|
|
397
|
+
return res
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def import_injecagent(items: list[dict], *, result: ImportResult | None = None) -> ImportResult:
|
|
401
|
+
"""InjecAgent: a benign user request triggers a user tool whose OUTPUT carries the attacker instruction,
|
|
402
|
+
which tries to drive an attacker tool. Real fields: `User Instruction`, `Attacker Instruction`,
|
|
403
|
+
`Attacker Tools` (list) / `Attacker Tool`, `User Tool`, `Attack Type` (dh=direct harm, ds=data stealing).
|
|
404
|
+
We model the induced attacker action, put the injected instruction in a manifest, and set the category
|
|
405
|
+
from the attack type (ds -> data_exfiltration). Non-tool attacker actions are skipped."""
|
|
406
|
+
res = result or ImportResult(source="injecagent")
|
|
407
|
+
for i, item in enumerate(items):
|
|
408
|
+
try:
|
|
409
|
+
if not isinstance(item, dict):
|
|
410
|
+
res.skipped_errors += 1
|
|
411
|
+
continue
|
|
412
|
+
atools = item.get("Attacker Tools") or item.get("attacker_tools")
|
|
413
|
+
attacker_tool = (atools[0] if isinstance(atools, list) and atools else None) \
|
|
414
|
+
or item.get("Attacker Tool") or item.get("attacker_tool") or item.get("tool") or ""
|
|
415
|
+
if _classify_tool(attacker_tool) is None:
|
|
416
|
+
res.skipped_no_equivalent += 1
|
|
417
|
+
continue
|
|
418
|
+
payload = item.get("Attacker Instruction") or item.get("attacker_instruction") or ""
|
|
419
|
+
user_instruction = item.get("User Instruction") or item.get("user_instruction") or ""
|
|
420
|
+
user_tool = item.get("User Tool") or item.get("user_tool") or attacker_tool
|
|
421
|
+
attack_type = (item.get("Attack Type") or item.get("attack_type") or "").lower()
|
|
422
|
+
category = "security.data_exfiltration" if attack_type.startswith(("ds", "data")) \
|
|
423
|
+
else "security.prompt_injection"
|
|
424
|
+
cmd = item.get("command")
|
|
425
|
+
res.cases.append(_mk_case(
|
|
426
|
+
f"injecagent-{item.get('id', i)}", cmd=cmd if cmd else None,
|
|
427
|
+
tool=None if cmd else user_tool, args=None if cmd else (item.get("args") or {}),
|
|
428
|
+
expect="ask", label="attack", category=category,
|
|
429
|
+
judge={"injection_present": 0.9, "injection_targets_action": 0.9},
|
|
430
|
+
transcript=[{"role": "user", "text": user_instruction}] if user_instruction else None,
|
|
431
|
+
manifest={"tool_output.txt": payload} if payload else None, result=payload or None))
|
|
432
|
+
res.imported += 1
|
|
433
|
+
except Exception: # noqa: BLE001
|
|
434
|
+
res.skipped_errors += 1
|
|
435
|
+
return res
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
_DRIFT_ATTACK_LABELS = {"hijacked", "injection point", "injection_point", "failed injection", "failed_injection"}
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def import_agentdrift(items: list[dict], *, result: ImportResult | None = None) -> ImportResult:
|
|
442
|
+
"""AgentDrift: step-labeled tool-call trajectories. Each trajectory has a `user_goal`/`task` and a list
|
|
443
|
+
of `steps`, each step a {tool/action, args, observation, label} where label is one of benign,
|
|
444
|
+
injection point, hijacked, failed injection. We emit one case per actionable step: `hijacked` steps are
|
|
445
|
+
attacks the agent took for the attacker (intent low); `benign`/`failed injection` steps are benign; the
|
|
446
|
+
poisoned observation at an `injection point` is carried as a manifest. Flat {tool,args,drifted} items
|
|
447
|
+
are also accepted (unit fixtures). Steps with no shell/file/network equivalent are skipped."""
|
|
448
|
+
res = result or ImportResult(source="agentdrift")
|
|
449
|
+
|
|
450
|
+
def emit(step: dict, *, goal: str, tid, j: int) -> None:
|
|
451
|
+
tool = step.get("tool") or step.get("action") or step.get("function") or ""
|
|
452
|
+
if _classify_tool(tool) is None:
|
|
453
|
+
res.skipped_no_equivalent += 1
|
|
454
|
+
return
|
|
455
|
+
label_raw = (step.get("label") or step.get("step_label") or "").lower().replace("_", " ")
|
|
456
|
+
attack = label_raw in {"hijacked", "injection point", "failed injection"}
|
|
457
|
+
hijacked = label_raw == "hijacked"
|
|
458
|
+
args = step.get("args") or step.get("arguments") or {}
|
|
459
|
+
cmd = args.get("command") if isinstance(args, dict) else step.get("command")
|
|
460
|
+
obs = step.get("observation") or step.get("output")
|
|
461
|
+
res.cases.append(_mk_case(
|
|
462
|
+
f"agentdrift-{tid}-{j}", cmd=cmd if cmd else None, tool=None if cmd else tool,
|
|
463
|
+
args=None if cmd else args,
|
|
464
|
+
expect="ask" if attack else "allow", label="attack" if attack else "benign",
|
|
465
|
+
category="security.malicious_command",
|
|
466
|
+
judge={"intent_authorized": 0.1 if hijacked else (0.5 if attack else 0.9)},
|
|
467
|
+
transcript=[{"role": "user", "text": goal}] if goal else None,
|
|
468
|
+
manifest={"observation.txt": obs} if (obs and "injection" in label_raw) else None,
|
|
469
|
+
result=obs if (obs and "injection" in label_raw) else None))
|
|
470
|
+
res.imported += 1
|
|
471
|
+
|
|
472
|
+
for i, item in enumerate(items):
|
|
473
|
+
try:
|
|
474
|
+
if not isinstance(item, dict):
|
|
475
|
+
res.skipped_errors += 1
|
|
476
|
+
continue
|
|
477
|
+
goal = item.get("user_goal") or item.get("goal") or item.get("task") or ""
|
|
478
|
+
tid = item.get("id") or item.get("trajectory_id") or i
|
|
479
|
+
steps = item.get("steps") or item.get("trajectory")
|
|
480
|
+
if isinstance(steps, list) and steps:
|
|
481
|
+
for j, step in enumerate(steps):
|
|
482
|
+
if isinstance(step, dict):
|
|
483
|
+
emit(step, goal=goal, tid=tid, j=j)
|
|
484
|
+
else:
|
|
485
|
+
# flat fallback (unit fixtures): the item itself is one step
|
|
486
|
+
drifted = bool(item.get("drifted", True))
|
|
487
|
+
tool = item.get("tool") or item.get("action") or ""
|
|
488
|
+
if _classify_tool(tool) is None:
|
|
489
|
+
res.skipped_no_equivalent += 1
|
|
490
|
+
continue
|
|
491
|
+
args = item.get("args") or {}
|
|
492
|
+
cmd = args.get("command") if isinstance(args, dict) else item.get("command")
|
|
493
|
+
res.cases.append(_mk_case(
|
|
494
|
+
f"agentdrift-{tid}", cmd=cmd if cmd else None, tool=None if cmd else tool,
|
|
495
|
+
args=None if cmd else args, expect="ask" if drifted else "allow",
|
|
496
|
+
label="attack" if drifted else "benign", category="security.malicious_command",
|
|
497
|
+
judge={"intent_authorized": 0.1 if drifted else 0.9},
|
|
498
|
+
transcript=[{"role": "user", "text": goal}] if goal else None))
|
|
499
|
+
res.imported += 1
|
|
500
|
+
except Exception: # noqa: BLE001
|
|
501
|
+
res.skipped_errors += 1
|
|
502
|
+
return res
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
IMPORTERS = {"agentdojo": import_agentdojo, "injecagent": import_injecagent, "agentdrift": import_agentdrift}
|