killchain 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- killchain/__init__.py +10 -0
- killchain/__main__.py +5 -0
- killchain/config.toml +78 -0
- killchain/contract.py +154 -0
- killchain/detector.py +180 -0
- killchain/doctor.py +164 -0
- killchain/fakes.py +182 -0
- killchain/frontier.py +63 -0
- killchain/gate.py +99 -0
- killchain/harness.py +377 -0
- killchain/localmodel.py +74 -0
- killchain/paths.py +277 -0
- killchain/prompts/frontier_build.md +34 -0
- killchain/prompts/local_implement.md +36 -0
- killchain/prompts/partition.md +100 -0
- killchain/routing.py +310 -0
- killchain/stubgen.py +151 -0
- killchain-0.1.0.dist-info/METADATA +268 -0
- killchain-0.1.0.dist-info/RECORD +22 -0
- killchain-0.1.0.dist-info/WHEEL +4 -0
- killchain-0.1.0.dist-info/entry_points.txt +2 -0
- killchain-0.1.0.dist-info/licenses/LICENSE +21 -0
killchain/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""killchain: split a build so a frontier agent never sees the risky half.
|
|
2
|
+
|
|
3
|
+
The prompt is partitioned by *refusal risk*, not confidentiality. Anything a
|
|
4
|
+
commercial coding agent would refuse, hedge, simplify, or guess at is routed to
|
|
5
|
+
an abliterated orchestrator model, which then implements it locally. A frontier
|
|
6
|
+
agent (Claude Code, Codex) builds the rest against stubs.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
__version__ = "0.1.0"
|
|
10
|
+
__all__ = ["__version__"]
|
killchain/__main__.py
ADDED
killchain/config.toml
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# killchain configuration. Paths may use ~.
|
|
2
|
+
#
|
|
3
|
+
# This file is the shipped default. On first run it is copied to
|
|
4
|
+
# ~/.config/killchain/config.toml, which is what you should edit.
|
|
5
|
+
|
|
6
|
+
[orchestrator]
|
|
7
|
+
# The abliterated orchestrator model: the only model that sees the full prompt.
|
|
8
|
+
# Any OpenAI-compatible chat endpoint. Default: refuseless, settings mirrored
|
|
9
|
+
# from ~/.hermes/config.yaml. The key is read from $REFUSELESS_API_KEY, or
|
|
10
|
+
# from env_file if that is unset. It is never written to disk by killchain.
|
|
11
|
+
base_url = "https://api.refuseless.com/v1"
|
|
12
|
+
model = "deepseek-v4.1-flash-abliterated" # also: glm-5.3-abliterated, glm-5.3-flash-abliterated
|
|
13
|
+
api_key_env = "REFUSELESS_API_KEY"
|
|
14
|
+
env_file = "~/.hermes/.env"
|
|
15
|
+
timeout = 900
|
|
16
|
+
max_partition_tries = 4
|
|
17
|
+
max_impl_tries = 4
|
|
18
|
+
|
|
19
|
+
[frontier]
|
|
20
|
+
backend = "claude" # claude | codex | fake
|
|
21
|
+
claude_model = "" # blank = CLI default
|
|
22
|
+
claude_extra_args = ["--dangerously-skip-permissions"] # it runs confined to the public dir
|
|
23
|
+
codex_model = ""
|
|
24
|
+
codex_extra_args = ["--full-auto"]
|
|
25
|
+
timeout = 3600
|
|
26
|
+
|
|
27
|
+
[risk]
|
|
28
|
+
# Jev (TypeSafe) via the OpenRouter Decisions API. Scores each span of the
|
|
29
|
+
# candidate public view on every refusal dimension in routing.py. Everything
|
|
30
|
+
# scored is sent to OpenRouter, so scoring happens on the public view and, if
|
|
31
|
+
# score_raw_prompt is on, on spans of the raw prompt.
|
|
32
|
+
enabled = true
|
|
33
|
+
base_url = "https://openrouter.ai/api/alpha/decisions"
|
|
34
|
+
model = "typesafe/jev-1.14"
|
|
35
|
+
api_key_env = "OPENROUTER_API_KEY"
|
|
36
|
+
# Dimensions, thresholds and the per-span decision live in routing.py.
|
|
37
|
+
timeout = 60
|
|
38
|
+
# Pre-score the RAW PROMPT span by span to build the risk map, and let the
|
|
39
|
+
# orchestrator ask for extra probes on subsets. Refusal risk is carried by the
|
|
40
|
+
# original phrasing, so this is the useful mode. All of it goes to
|
|
41
|
+
# OpenRouter/TypeSafe; turn off to score only the assembled public view.
|
|
42
|
+
probe_raw_prompt = true
|
|
43
|
+
max_probe_rounds = 3
|
|
44
|
+
|
|
45
|
+
[routing]
|
|
46
|
+
# Shipped defaults live in killchain/routing.py; anything set here wins. Read
|
|
47
|
+
# routing.py first - these change what the frontier model is allowed to build.
|
|
48
|
+
# Nothing here is required; the section exists so retuning needs no code edit.
|
|
49
|
+
|
|
50
|
+
# Drop a dimension entirely (it stops being asked and stops routing spans).
|
|
51
|
+
# disabled_dimensions = ["hedge_framing"]
|
|
52
|
+
|
|
53
|
+
# Per-dimension bar, 0..1: the risk probability needed to pull a span out of
|
|
54
|
+
# the public view. Lower = more paranoid (more goes private).
|
|
55
|
+
# Valid keys: attribution, reverse_engineering, offensive_security, extraction,
|
|
56
|
+
# regulated, proprietary_model, hedge_framing, scale_or_scope,
|
|
57
|
+
# overall, default
|
|
58
|
+
# [routing.thresholds]
|
|
59
|
+
# hedge_framing = 0.75
|
|
60
|
+
# attribution = 0.40
|
|
61
|
+
|
|
62
|
+
# Adjustment per span kind, added to the bar. Fixtures are bare JSON numbers
|
|
63
|
+
# and score near 0.5 whatever they contain, so they get a higher bar.
|
|
64
|
+
# [routing.kind_slack]
|
|
65
|
+
# fixture = 0.15
|
|
66
|
+
|
|
67
|
+
# Extra regexes that block an outbound view outright, no model involved.
|
|
68
|
+
# Prefix with re: for a regex, otherwise it is a literal. Prefer the denylist
|
|
69
|
+
# file for terms; use this for patterns that should ship with your team's config.
|
|
70
|
+
# hard_rules = ["TICKER-[A-Z]{4}", { pattern = "re:\\bsk-[a-z0-9]{20,}", why = "looks like an API key" }]
|
|
71
|
+
|
|
72
|
+
[paths]
|
|
73
|
+
# Defaults are user directories so an installed command never writes into
|
|
74
|
+
# site-packages. Override with env vars (KILLCHAIN_PRIVATE_ROOT) or per run
|
|
75
|
+
# with --workspace / --private-root. Relative paths resolve against the cwd.
|
|
76
|
+
workspace = "" # blank = <project>/.killchain/workspace, else ~/.local/share/killchain/workspace
|
|
77
|
+
private_root = "~/killchain-private" # mode 0700: prompts, contracts, private impls
|
|
78
|
+
denylist = "" # blank = ~/.config/killchain/denylist.txt
|
killchain/contract.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""Contract: the agreed interface between the public (frontier-built) half and
|
|
2
|
+
the private (abliterated-model-built) half of a system.
|
|
3
|
+
|
|
4
|
+
The private half exists because the frontier may refuse, hedge, simplify, or
|
|
5
|
+
guess at the risky parts. The contract is produced by the LOCAL model from the
|
|
6
|
+
user's full prompt. Only `Contract.public_view()` ever leaves the machine;
|
|
7
|
+
`private_notes` and the original prompt never do.
|
|
8
|
+
|
|
9
|
+
Public fixtures are decoys on purpose: shape without behaviour. A handful of
|
|
10
|
+
correct input/output points is enough for a reader to fit a rule, and a
|
|
11
|
+
reverse-engineered rule is one the frontier no longer needs to have been told.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
from dataclasses import dataclass, field, asdict
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
IDENT = re.compile(r"^[a-z_][a-z0-9_]*$")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class ContractError(ValueError):
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class Param:
|
|
29
|
+
name: str
|
|
30
|
+
type: str
|
|
31
|
+
description: str = ""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class Fixture:
|
|
36
|
+
inputs: dict[str, Any]
|
|
37
|
+
output: Any
|
|
38
|
+
note: str = ""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class Interface:
|
|
43
|
+
module: str
|
|
44
|
+
function: str
|
|
45
|
+
description: str
|
|
46
|
+
params: list[Param]
|
|
47
|
+
returns: str
|
|
48
|
+
fixtures: list[Fixture] # PUBLIC decoys: shape only
|
|
49
|
+
private_fixtures: list[Fixture] = field(default_factory=list) # real behaviour, never leaves
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class Contract:
|
|
54
|
+
public_brief: str
|
|
55
|
+
interfaces: list[Interface]
|
|
56
|
+
private_notes: str = "" # NEVER leaves the machine
|
|
57
|
+
private_plan: str = "" # NEVER leaves the machine
|
|
58
|
+
|
|
59
|
+
# ---- construction -------------------------------------------------
|
|
60
|
+
@staticmethod
|
|
61
|
+
def from_dict(d: dict) -> "Contract":
|
|
62
|
+
try:
|
|
63
|
+
ifaces = []
|
|
64
|
+
for i in d.get("interfaces", []):
|
|
65
|
+
params = [Param(**p) for p in i.get("params", [])]
|
|
66
|
+
fixtures = [Fixture(**f) for f in i.get("fixtures", [])]
|
|
67
|
+
pfix = [Fixture(**f) for f in i.get("private_fixtures", [])]
|
|
68
|
+
ifaces.append(Interface(
|
|
69
|
+
module=i["module"], function=i["function"],
|
|
70
|
+
description=i.get("description", ""),
|
|
71
|
+
params=params, returns=i.get("returns", "Any"),
|
|
72
|
+
fixtures=fixtures, private_fixtures=pfix,
|
|
73
|
+
))
|
|
74
|
+
return Contract(
|
|
75
|
+
public_brief=d["public_brief"], interfaces=ifaces,
|
|
76
|
+
private_notes=d.get("private_notes", ""),
|
|
77
|
+
private_plan=d.get("private_plan", ""),
|
|
78
|
+
)
|
|
79
|
+
except (KeyError, TypeError) as e:
|
|
80
|
+
raise ContractError(f"malformed contract: {e}") from e
|
|
81
|
+
|
|
82
|
+
@staticmethod
|
|
83
|
+
def from_json(s: str) -> "Contract":
|
|
84
|
+
return Contract.from_dict(json.loads(s))
|
|
85
|
+
|
|
86
|
+
# ---- validation ---------------------------------------------------
|
|
87
|
+
def validate(self) -> None:
|
|
88
|
+
if not self.public_brief.strip():
|
|
89
|
+
raise ContractError("public_brief is empty")
|
|
90
|
+
if not self.interfaces:
|
|
91
|
+
raise ContractError("contract has no interfaces; nothing is risky, "
|
|
92
|
+
"so there is nothing to partition")
|
|
93
|
+
seen = set()
|
|
94
|
+
for i in self.interfaces:
|
|
95
|
+
for ident in (i.module, i.function):
|
|
96
|
+
if not IDENT.match(ident):
|
|
97
|
+
raise ContractError(f"bad identifier {ident!r} (snake_case only)")
|
|
98
|
+
key = (i.module, i.function)
|
|
99
|
+
if key in seen:
|
|
100
|
+
raise ContractError(f"duplicate interface {i.module}.{i.function}")
|
|
101
|
+
seen.add(key)
|
|
102
|
+
if not i.fixtures:
|
|
103
|
+
raise ContractError(f"{i.module}.{i.function} has no public fixtures; "
|
|
104
|
+
"every interface needs at least one synthetic decoy example")
|
|
105
|
+
if not i.private_fixtures:
|
|
106
|
+
raise ContractError(f"{i.module}.{i.function} has no private_fixtures; "
|
|
107
|
+
"every interface needs at least one real example for conformance")
|
|
108
|
+
pnames = {p.name for p in i.params}
|
|
109
|
+
for p in i.params:
|
|
110
|
+
if not IDENT.match(p.name):
|
|
111
|
+
raise ContractError(f"bad param name {p.name!r} in {i.function}")
|
|
112
|
+
for kind, fixtures in (("fixture", i.fixtures), ("private_fixture", i.private_fixtures)):
|
|
113
|
+
for n, f in enumerate(fixtures):
|
|
114
|
+
extra = set(f.inputs) - pnames
|
|
115
|
+
missing = pnames - set(f.inputs)
|
|
116
|
+
if extra or missing:
|
|
117
|
+
raise ContractError(
|
|
118
|
+
f"{kind} {n} of {i.module}.{i.function} inputs mismatch params: "
|
|
119
|
+
f"extra={sorted(extra)} missing={sorted(missing)}")
|
|
120
|
+
try:
|
|
121
|
+
json.dumps(f.inputs); json.dumps(f.output)
|
|
122
|
+
except TypeError as e:
|
|
123
|
+
raise ContractError(f"{kind} {n} of {i.function} not JSON: {e}")
|
|
124
|
+
real = {json.dumps([f.inputs, f.output], sort_keys=True) for f in i.private_fixtures}
|
|
125
|
+
for n, f in enumerate(i.fixtures):
|
|
126
|
+
if json.dumps([f.inputs, f.output], sort_keys=True) in real:
|
|
127
|
+
raise ContractError(
|
|
128
|
+
f"public fixture {n} of {i.module}.{i.function} equals a private fixture; "
|
|
129
|
+
"public fixtures must be decoys, not real behaviour")
|
|
130
|
+
|
|
131
|
+
# ---- views --------------------------------------------------------
|
|
132
|
+
def public_view(self) -> dict:
|
|
133
|
+
"""Everything the frontier model is allowed to see. private_fixtures excluded."""
|
|
134
|
+
ifaces = []
|
|
135
|
+
for i in self.interfaces:
|
|
136
|
+
d = asdict(i)
|
|
137
|
+
d.pop("private_fixtures", None)
|
|
138
|
+
ifaces.append(d)
|
|
139
|
+
return {"public_brief": self.public_brief, "interfaces": ifaces}
|
|
140
|
+
|
|
141
|
+
def private_fixtures_dict(self) -> dict:
|
|
142
|
+
"""{module: {function: [{inputs, output}]}} from the REAL examples."""
|
|
143
|
+
out: dict = {}
|
|
144
|
+
for i in self.interfaces:
|
|
145
|
+
out.setdefault(i.module, {})[i.function] = [
|
|
146
|
+
{"inputs": f.inputs, "output": f.output} for f in i.private_fixtures]
|
|
147
|
+
return out
|
|
148
|
+
|
|
149
|
+
def to_dict(self) -> dict:
|
|
150
|
+
return asdict(self)
|
|
151
|
+
|
|
152
|
+
def public_text(self) -> str:
|
|
153
|
+
"""Flattened outbound text, for the redaction gate."""
|
|
154
|
+
return json.dumps(self.public_view(), indent=2, sort_keys=True)
|
killchain/detector.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""Model-based refusal-risk detector: Jev (TypeSafe) via the OpenRouter
|
|
2
|
+
Decisions API.
|
|
3
|
+
|
|
4
|
+
Jev is a structured decision model: it returns probabilities, not prose. For
|
|
5
|
+
each span we ask one `noul` (yes/no) question per dimension in
|
|
6
|
+
routing.DIMENSIONS, in a single request, and read all the answers back. A span
|
|
7
|
+
that trips any dimension is pulled out of the public view and handed to the
|
|
8
|
+
abliterated orchestrator to implement instead.
|
|
9
|
+
|
|
10
|
+
Dimensions come from routing.py. Everything scored here is sent to OpenRouter
|
|
11
|
+
and TypeSafe, so the harness only scores the CANDIDATE PUBLIC VIEW and, when
|
|
12
|
+
`probe_raw_prompt` is on, spans of the raw prompt.
|
|
13
|
+
|
|
14
|
+
`base_url = "fake"` selects a canned backend for pipeline tests.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
import os
|
|
20
|
+
import re
|
|
21
|
+
import urllib.request
|
|
22
|
+
import urllib.error
|
|
23
|
+
from dataclasses import dataclass, field
|
|
24
|
+
|
|
25
|
+
from killchain import routing
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class DetectorError(RuntimeError):
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class RiskVerdict:
|
|
34
|
+
label: str # where the span came from, e.g. "brief[2]"
|
|
35
|
+
text: str
|
|
36
|
+
scores: dict[str, float] = field(default_factory=dict)
|
|
37
|
+
flagged: bool = False
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def worst(self) -> tuple[str, float]:
|
|
41
|
+
return routing.worst_dimension(self.scores)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def spans_from_public_view(pv: dict) -> list[tuple[str, str]]:
|
|
45
|
+
"""Split the public view into independently scorable spans."""
|
|
46
|
+
out: list[tuple[str, str]] = []
|
|
47
|
+
paras = [p.strip() for p in re.split(r"\n\s*\n", pv["public_brief"]) if p.strip()]
|
|
48
|
+
for n, p in enumerate(paras):
|
|
49
|
+
out.append((f"brief[{n}]", p))
|
|
50
|
+
for i in pv["interfaces"]:
|
|
51
|
+
name = f"{i['module']}.{i['function']}"
|
|
52
|
+
out.append((f"iface {name} desc", i["description"]))
|
|
53
|
+
sig = ", ".join(f"{p['name']}: {p['type']} ({p.get('description', '')})" for p in i["params"])
|
|
54
|
+
out.append((f"iface {name} sig", f"{name}({sig}) -> {i['returns']}"))
|
|
55
|
+
for k, f in enumerate(i["fixtures"]):
|
|
56
|
+
out.append((f"iface {name} fixture[{k}]", json.dumps(f, sort_keys=True)))
|
|
57
|
+
return out
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def whole_view_text(pv: dict) -> str:
|
|
61
|
+
"""The public view flattened, for the holistic pass.
|
|
62
|
+
|
|
63
|
+
Refusals are triggered by the shape of a task, not only by its sentences:
|
|
64
|
+
every span can be innocent while the assembled brief reads as 'clone this
|
|
65
|
+
product'. This text is scored once as the `overall` span.
|
|
66
|
+
"""
|
|
67
|
+
parts = [pv["public_brief"]]
|
|
68
|
+
for i in pv["interfaces"]:
|
|
69
|
+
parts.append(f"{i['module']}.{i['function']}: {i['description']}")
|
|
70
|
+
return "\n".join(parts)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _questions(risky_terms: list[str]) -> dict:
|
|
74
|
+
terms = ", ".join(risky_terms) if risky_terms else "(none listed)"
|
|
75
|
+
safe = routing.SAFE.strip() + "\nKnown-risky terms and obvious variants of them: " + terms + "."
|
|
76
|
+
qs: dict = {}
|
|
77
|
+
for key, d in routing.DIMENSIONS.items():
|
|
78
|
+
qs[key] = {
|
|
79
|
+
"type": "noul",
|
|
80
|
+
"instructions": d["question"],
|
|
81
|
+
"criteria": {"true": d["risky"].strip() + "\nAlso true if: " + terms + ".",
|
|
82
|
+
"false": safe},
|
|
83
|
+
}
|
|
84
|
+
qs["overall"] = {
|
|
85
|
+
"type": "noul",
|
|
86
|
+
"instructions": ("Taken as one task, is this the kind of build a frontier coding agent "
|
|
87
|
+
"would hedge, ask about, simplify, or refuse?"),
|
|
88
|
+
"criteria": {
|
|
89
|
+
"true": ("Any of these, even once: brand or clone framing, reverse engineering, "
|
|
90
|
+
"offensive security, harvesting, a regulated domain, another vendor's "
|
|
91
|
+
"model, hedged claims, or scope big enough to negotiate away."),
|
|
92
|
+
"false": safe,
|
|
93
|
+
},
|
|
94
|
+
}
|
|
95
|
+
return qs
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _call(cfg: dict, state: str, questions: dict) -> dict:
|
|
99
|
+
key = os.environ.get(cfg.get("api_key_env", "OPENROUTER_API_KEY"), "")
|
|
100
|
+
if not key:
|
|
101
|
+
raise DetectorError(f"env {cfg.get('api_key_env', 'OPENROUTER_API_KEY')} is not set")
|
|
102
|
+
body = {"model": cfg.get("model", "typesafe/jev-1.14"), "state": state, "questions": questions}
|
|
103
|
+
req = urllib.request.Request(
|
|
104
|
+
cfg.get("base_url", "https://openrouter.ai/api/alpha/decisions"),
|
|
105
|
+
data=json.dumps(body).encode(),
|
|
106
|
+
headers={"Content-Type": "application/json", "Authorization": f"Bearer {key}",
|
|
107
|
+
"HTTP-Referer": "https://github.com/local/harness", "X-Title": "skoal-risk"})
|
|
108
|
+
try:
|
|
109
|
+
with urllib.request.urlopen(req, timeout=cfg.get("timeout", 60)) as r:
|
|
110
|
+
return json.loads(r.read())
|
|
111
|
+
except urllib.error.HTTPError as e:
|
|
112
|
+
raise DetectorError(f"detector HTTP {e.code}: {e.read()[:500]!r}") from e
|
|
113
|
+
except urllib.error.URLError as e:
|
|
114
|
+
raise DetectorError(f"detector unreachable: {e}") from e
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _read_scores(resp: dict) -> dict[str, float]:
|
|
118
|
+
answers = resp.get("answers") or {}
|
|
119
|
+
scores: dict[str, float] = {}
|
|
120
|
+
for dim in routing.DIMENSIONS:
|
|
121
|
+
try:
|
|
122
|
+
scores[dim] = float(answers[dim]["noul"])
|
|
123
|
+
except (KeyError, TypeError, ValueError):
|
|
124
|
+
continue
|
|
125
|
+
if not scores:
|
|
126
|
+
raise DetectorError(f"unexpected detector response: {json.dumps(resp)[:500]}")
|
|
127
|
+
return scores
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _verdict(label: str, text: str, scores: dict[str, float]) -> RiskVerdict:
|
|
131
|
+
return RiskVerdict(label, text, scores, routing.classify(label, text, scores))
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def score_spans(cfg: dict, spans: list[tuple[str, str]],
|
|
135
|
+
risky_terms: list[str]) -> list[RiskVerdict]:
|
|
136
|
+
"""Score each span against every dimension, then let routing.classify() rule."""
|
|
137
|
+
if cfg.get("base_url") == "fake":
|
|
138
|
+
from killchain.fakes import fake_detector
|
|
139
|
+
return [_verdict(l, t, s) for (l, t), s in zip(spans, fake_detector(spans))]
|
|
140
|
+
qs = _questions(risky_terms)
|
|
141
|
+
out: list[RiskVerdict] = []
|
|
142
|
+
for label, text in spans:
|
|
143
|
+
out.append(_verdict(label, text, _read_scores(_call(cfg, text, qs))))
|
|
144
|
+
return out
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def score_whole_view(cfg: dict, pv: dict, risky_terms: list[str]) -> RiskVerdict:
|
|
148
|
+
"""The holistic pass over the assembled public view."""
|
|
149
|
+
text = whole_view_text(pv)
|
|
150
|
+
if cfg.get("base_url") == "fake":
|
|
151
|
+
from killchain.fakes import fake_overall
|
|
152
|
+
return _verdict("overall", text, fake_overall(text))
|
|
153
|
+
resp = _call(cfg, text, {"overall": _questions(risky_terms)["overall"]})
|
|
154
|
+
return _verdict("overall", text, _read_scores(resp))
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def describe(verdicts: list[RiskVerdict], only_flagged: bool = True) -> str:
|
|
158
|
+
rows = [v for v in verdicts if v.flagged] if only_flagged else verdicts
|
|
159
|
+
if not rows:
|
|
160
|
+
return "detector: no spans over threshold"
|
|
161
|
+
lines = [f"DETECTOR ROUTED {len(rows)} span(s) PRIVATE:"]
|
|
162
|
+
for v in rows:
|
|
163
|
+
dim, p = v.worst
|
|
164
|
+
lines.append(f" {dim} p={p:.2f} {v.label}: {v.text[:160]!r}")
|
|
165
|
+
return "\n".join(lines)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def risk_map(verdicts: list[RiskVerdict]) -> str:
|
|
169
|
+
"""Human/LLM-readable table of span scores for the partition prompt."""
|
|
170
|
+
lines = []
|
|
171
|
+
for v in verdicts:
|
|
172
|
+
dim, p = v.worst
|
|
173
|
+
tag = f"PRIVATE {dim} p={p:.2f}" if v.flagged else f"public {dim} p={p:.2f}"
|
|
174
|
+
lines.append(f"- [{tag}] {v.text}")
|
|
175
|
+
return "\n".join(lines) or "- (no spans)"
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def spans_from_prompt(prompt: str) -> list[tuple[str, str]]:
|
|
179
|
+
"""Delegates to routing.split_prompt(). Only used when probe_raw_prompt is on."""
|
|
180
|
+
return routing.split_prompt(prompt)
|
killchain/doctor.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""killchain doctor: tell a new user exactly what is missing.
|
|
2
|
+
|
|
3
|
+
Every check reports ok / warn / fail and nothing here raises. The point is that
|
|
4
|
+
a first run should never be a stack trace about a missing key three steps in.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import shutil
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import killchain.localmodel as localmodel
|
|
14
|
+
from killchain import paths
|
|
15
|
+
|
|
16
|
+
OK, WARN, FAIL = "ok", "warn", "FAIL"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class Check:
|
|
21
|
+
name: str
|
|
22
|
+
state: str
|
|
23
|
+
detail: str
|
|
24
|
+
fix: str = ""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _key_state(cfg: dict, label: str) -> tuple[str, str, str]:
|
|
28
|
+
name = cfg.get("api_key_env", "")
|
|
29
|
+
if name and os.environ.get(name):
|
|
30
|
+
return f"${name} is set", "", f"{label} key found in the environment"
|
|
31
|
+
env_file = Path(cfg.get("env_file", "~/.hermes/.env")).expanduser()
|
|
32
|
+
if name and env_file.exists():
|
|
33
|
+
try:
|
|
34
|
+
found = any(line.startswith(name + "=") for line in env_file.read_text().splitlines())
|
|
35
|
+
except OSError:
|
|
36
|
+
found = False
|
|
37
|
+
if found:
|
|
38
|
+
return f"${name} found in {env_file}", "", f"{label} key read from {env_file}"
|
|
39
|
+
return "", f"set ${name}", f"{label} key not found"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def run(cfg: dict) -> int:
|
|
43
|
+
checks: list[Check] = []
|
|
44
|
+
|
|
45
|
+
# --- config
|
|
46
|
+
try:
|
|
47
|
+
cp = paths.resolve_config(None)
|
|
48
|
+
checks.append(Check("config", OK, str(cp)))
|
|
49
|
+
except SystemExit as e:
|
|
50
|
+
checks.append(Check("config", FAIL, str(e), "run: killchain doctor --write-config"))
|
|
51
|
+
|
|
52
|
+
# --- frontier agent
|
|
53
|
+
backend = cfg.get("frontier", {}).get("backend", "claude")
|
|
54
|
+
exe = {"claude": "claude", "codex": "codex"}.get(backend)
|
|
55
|
+
if backend == "fake":
|
|
56
|
+
checks.append(Check("frontier", WARN, "backend=fake (canned replies)", "set backend in [frontier]"))
|
|
57
|
+
elif exe and shutil.which(exe):
|
|
58
|
+
checks.append(Check("frontier", OK, f"{exe} on PATH ({shutil.which(exe)})"))
|
|
59
|
+
else:
|
|
60
|
+
checks.append(Check("frontier", FAIL, f"{backend} backend but no `{exe}` on PATH",
|
|
61
|
+
f"install {exe}, or use --frontier codex/claude/fake"))
|
|
62
|
+
|
|
63
|
+
# --- orchestrator key
|
|
64
|
+
och = cfg.get("orchestrator", {})
|
|
65
|
+
detail, fix, msg = _key_state(och, "orchestrator")
|
|
66
|
+
if och.get("base_url") == "fake":
|
|
67
|
+
checks.append(Check("orchestrator", WARN, "base_url=fake (canned replies)"))
|
|
68
|
+
else:
|
|
69
|
+
checks.append(Check("orchestrator", OK if detail else FAIL, detail or msg, fix))
|
|
70
|
+
|
|
71
|
+
# --- risk detector key
|
|
72
|
+
risk = cfg.get("risk", {})
|
|
73
|
+
if not risk.get("enabled", True):
|
|
74
|
+
checks.append(Check("risk scorer", WARN, "disabled (gate only)", "enable [risk] to score spans"))
|
|
75
|
+
elif risk.get("base_url") == "fake":
|
|
76
|
+
checks.append(Check("risk scorer", WARN, "base_url=fake (canned replies)"))
|
|
77
|
+
else:
|
|
78
|
+
detail, fix, msg = _key_state(risk, "OpenRouter")
|
|
79
|
+
checks.append(Check("risk scorer", OK if detail else FAIL, detail or msg, fix))
|
|
80
|
+
|
|
81
|
+
# --- private root: must exist, be writable, and NOT be world-readable
|
|
82
|
+
root = Path(cfg.get("paths", {}).get("private_root", str(paths.private_root()))).expanduser()
|
|
83
|
+
try:
|
|
84
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
85
|
+
probe = root / ".write-test"
|
|
86
|
+
probe.write_text("ok")
|
|
87
|
+
probe.unlink()
|
|
88
|
+
mode = root.stat().st_mode
|
|
89
|
+
if mode & 0o077:
|
|
90
|
+
# Holds the raw prompt and real fixtures; fix it rather than nag.
|
|
91
|
+
paths.secure_mkdir(root)
|
|
92
|
+
mode = root.stat().st_mode
|
|
93
|
+
if mode & 0o077:
|
|
94
|
+
checks.append(Check("private root", WARN, f"{root} still group/world-accessible",
|
|
95
|
+
f"chmod 700 {root}"))
|
|
96
|
+
else:
|
|
97
|
+
checks.append(Check("private root", OK, f"{root} (tightened to 0700)"))
|
|
98
|
+
else:
|
|
99
|
+
checks.append(Check("private root", OK, f"{root} (mode {oct(mode & 0o777)})"))
|
|
100
|
+
except OSError as e:
|
|
101
|
+
checks.append(Check("private root", FAIL, f"{root}: {e}", "pass --private-root to a writable dir"))
|
|
102
|
+
|
|
103
|
+
# --- workspace
|
|
104
|
+
ws = Path(cfg.get("paths", {}).get("workspace", "")).expanduser()
|
|
105
|
+
try:
|
|
106
|
+
ws.mkdir(parents=True, exist_ok=True)
|
|
107
|
+
checks.append(Check("workspace", OK, str(ws)))
|
|
108
|
+
except OSError as e:
|
|
109
|
+
checks.append(Check("workspace", FAIL, f"{ws}: {e}", "pass --workspace"))
|
|
110
|
+
|
|
111
|
+
# --- denylist
|
|
112
|
+
dl = paths.denylist_path(cfg)
|
|
113
|
+
try:
|
|
114
|
+
import killchain.gate as gate
|
|
115
|
+
rules = gate.load_denylist(dl)
|
|
116
|
+
if rules:
|
|
117
|
+
checks.append(Check("denylist", OK, f"{len(rules)} rule(s) in {dl}"))
|
|
118
|
+
else:
|
|
119
|
+
checks.append(Check("denylist", WARN, f"empty ({dl}) — gate will only catch builtin secrets",
|
|
120
|
+
"add product names, codenames and licence names"))
|
|
121
|
+
except SystemExit as e:
|
|
122
|
+
checks.append(Check("denylist", FAIL, str(e), f"fix or delete {dl}"))
|
|
123
|
+
|
|
124
|
+
# --- the one check that proves the network path works
|
|
125
|
+
if risk.get("enabled", True) and risk.get("base_url") != "fake" and _key_state(risk, "OpenRouter")[0]:
|
|
126
|
+
import killchain.detector as detector
|
|
127
|
+
try:
|
|
128
|
+
spans = detector.spans_from_prompt("a small test span about sorting a list")
|
|
129
|
+
got = detector.score_spans(risk, spans[:1], [])
|
|
130
|
+
score = got[0].scores.get("overall", got[0].scores)
|
|
131
|
+
checks.append(Check("risk round-trip", OK, f"scored 1 span: {score}"))
|
|
132
|
+
except Exception as e: # noqa: BLE001 - doctor must never raise
|
|
133
|
+
checks.append(Check("risk round-trip", FAIL, f"{type(e).__name__}: {e}",
|
|
134
|
+
"check OPENROUTER_API_KEY and network"))
|
|
135
|
+
|
|
136
|
+
# --- orchestrator round-trip (cheap, skipped when the base_url is fake)
|
|
137
|
+
if och.get("base_url") != "fake" and _key_state(och, "orchestrator")[0]:
|
|
138
|
+
try:
|
|
139
|
+
localmodel.chat({**och, "timeout": 30}, [{"role": "user", "content": "reply with: ok"}],
|
|
140
|
+
max_tokens=8)
|
|
141
|
+
checks.append(Check("orchestrator round-trip", OK, "chat endpoint answered"))
|
|
142
|
+
except Exception as e: # noqa: BLE001
|
|
143
|
+
checks.append(Check("orchestrator round-trip", FAIL, f"{type(e).__name__}: {e}",
|
|
144
|
+
"check the key, base_url and model name"))
|
|
145
|
+
|
|
146
|
+
# --- report
|
|
147
|
+
width = max(len(c.name) for c in checks)
|
|
148
|
+
icon = {OK: "ok ", WARN: "warn", FAIL: "FAIL"}
|
|
149
|
+
failed = 0
|
|
150
|
+
for c in checks:
|
|
151
|
+
print(f" {icon[c.state]} {c.name.ljust(width)} {c.detail}")
|
|
152
|
+
if c.fix:
|
|
153
|
+
print(f" {''.ljust(width)} -> {c.fix}")
|
|
154
|
+
failed += c.state == FAIL
|
|
155
|
+
fatal = [c for c in checks if c.state == FAIL and c.name in ("config", "frontier", "orchestrator")]
|
|
156
|
+
print()
|
|
157
|
+
if failed:
|
|
158
|
+
print(f"{failed} check(s) failed.")
|
|
159
|
+
else:
|
|
160
|
+
print("all checks passed — try: killchain \"your task\"")
|
|
161
|
+
if failed:
|
|
162
|
+
print("Still stuck? War stories and how this works: "
|
|
163
|
+
"https://youtube.com/@drjoshcsimmons")
|
|
164
|
+
return 1 if fatal else 0
|