hulder 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
hulder/__init__.py ADDED
@@ -0,0 +1,21 @@
1
+ from importlib.metadata import version
2
+
3
+ from hulder.gate.check import Estimate, Report, Verdict, check, check_snapshots
4
+ from hulder.gate.pairing import PairingError, prompt_ids_sha256
5
+ from hulder.gate.probe import Probe, apply_probe
6
+ from hulder.gate.snapshot import Snapshot
7
+
8
+ __all__ = [
9
+ "Estimate",
10
+ "PairingError",
11
+ "Probe",
12
+ "Report",
13
+ "Snapshot",
14
+ "Verdict",
15
+ "apply_probe",
16
+ "check",
17
+ "check_snapshots",
18
+ "prompt_ids_sha256",
19
+ ]
20
+
21
+ __version__ = version("hulder")
@@ -0,0 +1,21 @@
1
+ """Activation capture from vLLM, the hulder[vllm] extra (docs/design.md §5). Public but experimental
2
+ in v0.1: these names may change in any 0.x release (docs/design.md §4).
3
+
4
+ capture, CaptureConfig and Capture import vLLM and torch, so they load on first use. The
5
+ other names, and importing this package, need neither.
6
+ """
7
+
8
+ import importlib
9
+
10
+ from hulder.capture.base_model import base_model_for
11
+ from hulder.capture.metadata import snapshot_metadata
12
+
13
+ __all__ = ["Capture", "CaptureConfig", "base_model_for", "capture", "snapshot_metadata"]
14
+
15
+ _NEEDS_VLLM = ("Capture", "CaptureConfig", "capture")
16
+
17
+
18
+ def __getattr__(name: str):
19
+ if name in _NEEDS_VLLM:
20
+ return getattr(importlib.import_module("hulder.capture.hidden_states"), name)
21
+ raise AttributeError(f"module 'hulder.capture' has no attribute {name!r}")
@@ -0,0 +1,43 @@
1
+ """The base_model a capture records, which check requires to match across runs.
2
+
3
+ Kept apart from hidden_states.py so it imports without vLLM.
4
+ """
5
+
6
+ import re
7
+ from collections.abc import Callable
8
+ from pathlib import Path
9
+
10
+
11
+ def card_base_model(model: str, revision: str) -> str | None:
12
+ """The base_model field of a model card's YAML front matter at a pinned revision."""
13
+ # Imported here so base_model_for, and its tests, run on the base install.
14
+ import yaml
15
+ from huggingface_hub import hf_hub_download
16
+
17
+ text = Path(hf_hub_download(model, "README.md", revision=revision)).read_text()
18
+ front = re.match(r"^---\n(.*?)\n---", text, re.DOTALL)
19
+ value = yaml.safe_load(front.group(1)).get("base_model") if front else None
20
+ return value if isinstance(value, str) else None
21
+
22
+
23
+ def base_model_for(
24
+ model: str,
25
+ revision: str,
26
+ quantization: str | None,
27
+ override: str | None = None,
28
+ card: Callable[[str, str], str | None] = card_base_model,
29
+ ) -> tuple[str | None, str | None]:
30
+ """(base_model, warning). Unquantized, the model is its own base: a card's base_model
31
+ names the parent it was fine-tuned from, not what it re-encodes. Quantized, the override
32
+ or else the card's field; with neither, None and a warning rather than a guess."""
33
+ if override is not None:
34
+ return override, None
35
+ if quantization is None:
36
+ return model, None
37
+ base_model = card(model, revision)
38
+ if base_model is None:
39
+ return None, (
40
+ f"{model}@{revision} is quantized but its model card names no single base_model; "
41
+ "pass the base model explicitly so check can confirm both runs serve the same model."
42
+ )
43
+ return base_model, None
@@ -0,0 +1,106 @@
1
+ """Activation capture from vLLM, the hulder[vllm] extra (docs/design.md §4 and §5).
2
+
3
+ Offline extraction through vLLM's hidden-state connector, prefix caching off, chunked
4
+ prefill off, fixed prompt order and batch size. Never imported by the base package.
5
+ Any script calling `capture` needs an `if __name__ == "__main__":` guard (spawn on WSL).
6
+ """
7
+
8
+ import os
9
+ import tempfile
10
+ from dataclasses import dataclass
11
+
12
+ import numpy as np
13
+ import torch
14
+ from vllm import LLM, SamplingParams
15
+ from vllm.config.kv_transfer import KVTransferConfig
16
+ from vllm.distributed.kv_transfer.kv_connector.v1 import example_hidden_states_connector
17
+
18
+
19
+ @dataclass(frozen=True)
20
+ class CaptureConfig:
21
+ """Everything that fixes the captured values. Recorded verbatim in the run manifest."""
22
+
23
+ model: str
24
+ revision: str
25
+ layers: tuple[int, ...] # id k = residual stream after k decoder layers, 0 = embeddings
26
+ positions: tuple[int, ...] # negative indices into the templated prompt
27
+ system_prompt: str
28
+ quantization: str | None = None
29
+ dtype: str = "float16"
30
+ batch_size: int = 8
31
+ # Fits Qwen2.5-1.5B on an 8 GB RTX 2070 with extraction on (measured 2026-10-02).
32
+ gpu_memory_utilization: float = 0.7
33
+ max_model_len: int = 1024
34
+ max_num_batched_tokens: int = 1024
35
+ seed: int = 0
36
+ # Passed to the chat template when set. Pinned for hybrid-thinking models such as Qwen3,
37
+ # whose templated prompt, and so its last token, depends on it. None leaves the template's default.
38
+ enable_thinking: bool | None = None
39
+ # Required off for reproducible snapshots. Fields so the manifest records them.
40
+ enable_prefix_caching: bool = False
41
+ enable_chunked_prefill: bool = False
42
+
43
+ def __post_init__(self) -> None:
44
+ if self.enable_prefix_caching or self.enable_chunked_prefill:
45
+ raise ValueError("capture needs prefix caching and chunked prefill off (docs/design.md §5)")
46
+
47
+
48
+ @dataclass(frozen=True)
49
+ class Capture:
50
+ activations: np.ndarray # (n, layers, positions, d), float16, prompt order preserved
51
+ position_tokens: list[list[str]] # token string at each requested position, per prompt
52
+ # torch.cuda.get_device_name() of the current device, the first visible one. Under tensor
53
+ # parallelism the other GPUs aren't recorded, a known limit. Another GPU is another serving
54
+ # config (docs/design.md §6, metadata rules).
55
+ gpu: str
56
+ enable_thinking: bool | None # as passed to the chat template, None for its default
57
+
58
+
59
+ def capture(prompts: list[str], cfg: CaptureConfig) -> Capture:
60
+ convs = [[{"role": "system", "content": cfg.system_prompt}, {"role": "user", "content": p}] for p in prompts]
61
+ params = SamplingParams(max_tokens=1, temperature=0)
62
+ template = {} if cfg.enable_thinking is None else {"chat_template_kwargs": {"enable_thinking": cfg.enable_thinking}}
63
+ acts, tokens = [], []
64
+ with tempfile.TemporaryDirectory() as tmp:
65
+ llm = LLM(
66
+ model=cfg.model,
67
+ revision=cfg.revision,
68
+ quantization=cfg.quantization,
69
+ dtype=cfg.dtype,
70
+ gpu_memory_utilization=cfg.gpu_memory_utilization,
71
+ max_model_len=cfg.max_model_len,
72
+ max_num_batched_tokens=cfg.max_num_batched_tokens,
73
+ max_num_seqs=cfg.batch_size,
74
+ seed=cfg.seed,
75
+ enable_chunked_prefill=cfg.enable_chunked_prefill,
76
+ enable_prefix_caching=cfg.enable_prefix_caching,
77
+ speculative_config={
78
+ "method": "extract_hidden_states",
79
+ "num_speculative_tokens": 1,
80
+ "draft_model_config": {"hf_config": {"eagle_aux_hidden_state_layer_ids": list(cfg.layers)}},
81
+ },
82
+ kv_transfer_config=KVTransferConfig(
83
+ kv_connector="ExampleHiddenStatesConnector",
84
+ kv_role="kv_producer",
85
+ kv_connector_extra_config={"shared_storage_path": tmp},
86
+ ),
87
+ )
88
+ tok = llm.get_tokenizer()
89
+ pos = list(cfg.positions)
90
+ for start in range(0, len(convs), cfg.batch_size):
91
+ for out in llm.chat(convs[start : start + cfg.batch_size], params, use_tqdm=False, **template):
92
+ path = out.kv_transfer_params["hidden_states_path"]
93
+ obj = example_hidden_states_connector.load_hidden_states(path)
94
+ assert obj["token_ids"].tolist() == list(out.prompt_token_ids)
95
+ # hidden_states is [tokens, layers, d]; keep only the requested positions.
96
+ acts.append(obj["hidden_states"][pos].transpose(0, 1).to("cpu").numpy().astype(np.float16))
97
+ tokens.append(tok.convert_ids_to_tokens(obj["token_ids"][pos].tolist()))
98
+ os.remove(path)
99
+ if start // cfg.batch_size % 100 == 0:
100
+ print(f"captured {min(start + cfg.batch_size, len(convs))}/{len(convs)}", flush=True)
101
+ return Capture(
102
+ activations=np.stack(acts),
103
+ position_tokens=tokens,
104
+ gpu=torch.cuda.get_device_name(),
105
+ enable_thinking=cfg.enable_thinking,
106
+ )
@@ -0,0 +1,47 @@
1
+ """Snapshot metadata from a capture. Imports no vLLM or torch, so it runs anywhere."""
2
+
3
+ from typing import TYPE_CHECKING
4
+
5
+ if TYPE_CHECKING:
6
+ from hulder.capture.hidden_states import Capture, CaptureConfig
7
+
8
+
9
+ def snapshot_metadata(
10
+ cfg: "CaptureConfig",
11
+ captured: "Capture",
12
+ base_model: str,
13
+ vllm_version: str,
14
+ layer: int | None = None,
15
+ position: int | None = None,
16
+ ) -> dict:
17
+ """The metadata a Snapshot of one layer and position of a capture records (docs/design.md §6, metadata rules): the
18
+ serving config, the layer and position, the GPU, and enable_thinking when it was pinned.
19
+ Only keys comparable across runs. layer and position default to the config's only ones."""
20
+ layer = _one(cfg.layers, layer, "layer")
21
+ position = _one(cfg.positions, position, "position")
22
+ metadata = {
23
+ "base_model": base_model,
24
+ "model": cfg.model,
25
+ "revision": cfg.revision,
26
+ "quantization": cfg.quantization,
27
+ "dtype": cfg.dtype,
28
+ "batch_size": cfg.batch_size,
29
+ "vllm_version": vllm_version,
30
+ "system_prompt": cfg.system_prompt,
31
+ "layer": layer,
32
+ "position": position,
33
+ "gpu": captured.gpu,
34
+ }
35
+ if captured.enable_thinking is not None:
36
+ metadata["enable_thinking"] = captured.enable_thinking
37
+ return metadata
38
+
39
+
40
+ def _one(captured: tuple[int, ...], chosen: int | None, name: str) -> int:
41
+ if chosen is None:
42
+ if len(captured) != 1:
43
+ raise ValueError(f"the capture has {name}s {captured}; name the snapshot's {name}")
44
+ return captured[0]
45
+ if chosen not in captured:
46
+ raise ValueError(f"{name} {chosen} wasn't captured, only {captured}")
47
+ return chosen
hulder/cli.py ADDED
@@ -0,0 +1,342 @@
1
+ """The hulder command: parse, call check_snapshots, print the Report, exit with its code.
2
+
3
+ No statistics live here (docs/design.md §4). Exit codes follow docs/design.md §4: the verdict's code, 1 for
4
+ a tool failure such as a PairingError or a damaged file, 2 for a usage error.
5
+ """
6
+
7
+ import argparse
8
+ import dataclasses
9
+ import json
10
+ import math
11
+ import sys
12
+ from importlib.metadata import version
13
+ from importlib.resources import files
14
+ from pathlib import Path
15
+ from typing import NoReturn
16
+
17
+ import hulder.gate.check as gate_check
18
+ from hulder.gate.check import Report, Verdict, check_snapshots
19
+ from hulder.gate.pairing import PairingError
20
+ from hulder.gate.probe import Probe
21
+ from hulder.gate.snapshot import Snapshot
22
+
23
+ # The tolerance-edge presentation rule (docs/design.md §6): past this many times the calibration set's benign count,
24
+ # more_benign_needed is printed with a caveat that it is unstable.
25
+ EDGE_FACTOR = 10
26
+ COLUMN = 26 # width of the reference and candidate columns
27
+ EDGE_CAVEAT = "the FPR estimate sits at the tolerance edge; this number is unstable and more data may not settle it"
28
+
29
+
30
+ # Version of --json's layout. Bump it when a key is renamed or removed, not when one is added.
31
+ JSON_SCHEMA_VERSION = 1
32
+
33
+
34
+ class UsageError(Exception):
35
+ """A usage error, exit 2 (docs/design.md §4). argparse's own exit is replaced so --json can report it."""
36
+
37
+ def __init__(self, message: str, usage: str):
38
+ super().__init__(message)
39
+ self.usage = usage
40
+
41
+
42
+ class _Parser(argparse.ArgumentParser):
43
+ def error(self, message: str) -> NoReturn:
44
+ raise UsageError(message, self.format_usage())
45
+
46
+
47
+ def main() -> None:
48
+ sys.exit(run())
49
+
50
+
51
+ def run(argv: list[str] | None = None) -> int:
52
+ argv = sys.argv[1:] if argv is None else argv
53
+ as_json = "--json" in argv
54
+ parser = _parser()
55
+ try:
56
+ args = parser.parse_args(argv)
57
+ if args.command == "quickstart":
58
+ return _quickstart()
59
+ if forbidden := sorted(set(args.under_test) & set(gate_check.NEVER_UNDER_TEST)):
60
+ parser.error(
61
+ f"--under-test {forbidden[0]}: {forbidden[0]} can't be a change under test, "
62
+ "since it says both runs serve the same model"
63
+ )
64
+ probe = Probe.load(args.probe) if args.probe else None
65
+ threshold = _setting(parser, args.threshold, probe, "threshold")
66
+ target_fpr = _setting(parser, args.target_fpr, probe, "target_fpr")
67
+ report = _check(
68
+ args.reference, args.candidate, args.noise_floor, probe, threshold, target_fpr, args.seed, args.under_test
69
+ )
70
+ except UsageError as e:
71
+ print(e.usage, end="", file=sys.stderr)
72
+ return _fail(str(e), 2, as_json)
73
+ except PairingError as e:
74
+ return _fail(f"pairing error: {e}", 1, as_json, prefix="")
75
+ except (ValueError, OSError) as e:
76
+ return _fail(str(e), 1, as_json)
77
+ except Exception as e:
78
+ if as_json: # a bug: the JSON still says so, and the traceback follows on stderr
79
+ print(_error_json(f"internal error: {e!r}", 1))
80
+ raise
81
+ inputs = _inputs(args.reference, args.candidate, args.noise_floor, args.probe)
82
+ print(to_json(report, inputs) if args.json else render(report, inputs))
83
+ return report.verdict.exit_code
84
+
85
+
86
+ def _fail(message: str, code: int, as_json: bool, prefix: str = "error: ") -> int:
87
+ """Print a failure to stderr, and with --json to stdout too, then give its exit code."""
88
+ print(f"hulder: {prefix}{message}", file=sys.stderr)
89
+ if as_json:
90
+ print(_error_json(message, code))
91
+ return code
92
+
93
+
94
+ def _error_json(message: str, code: int) -> str:
95
+ return json.dumps(
96
+ {
97
+ "schema_version": JSON_SCHEMA_VERSION,
98
+ "hulder_version": version("hulder"),
99
+ "error": message,
100
+ "exit_code": code,
101
+ },
102
+ indent=2,
103
+ )
104
+
105
+
106
+ def _parser() -> _Parser:
107
+ parser = _Parser(prog="hulder", description="Release gate for thresholded scorers.")
108
+ commands = parser.add_subparsers(dest="command", required=True)
109
+ check_parser = commands.add_parser(
110
+ "check",
111
+ help="compare a candidate serving config's snapshot against the reference's",
112
+ description="Exit codes: 0 PASS, 10 THRESHOLD STALE, 11 SCORER DEGRADED, 12 INCONCLUSIVE, "
113
+ "1 tool failure (including a pairing error), 2 usage error.",
114
+ )
115
+ check_parser.add_argument("--reference", required=True, help="reference snapshot (.npz)")
116
+ check_parser.add_argument("--candidate", required=True, help="candidate snapshot (.npz)")
117
+ check_parser.add_argument(
118
+ "--noise-floor", metavar="RERUN", help="a rerun of the reference, to measure the noise floor"
119
+ )
120
+ check_parser.add_argument("--probe", help="probe (.npz) for snapshots of activations; supplies the threshold too")
121
+ check_parser.add_argument("--threshold", type=_finite, help="defaults to the probe's")
122
+ check_parser.add_argument("--target-fpr", type=_rate, help="defaults to the probe's, e.g. 0.01 for 1%%")
123
+ check_parser.add_argument(
124
+ "--under-test", action="append", default=[], metavar="KEY", help="a further metadata key that may differ"
125
+ )
126
+ check_parser.add_argument("--seed", type=_seed, default=0, help="bootstrap seed (default 0)")
127
+ check_parser.add_argument("--json", action="store_true", help="print the Report as JSON")
128
+ commands.add_parser("quickstart", help="run two checks on the bundled demo data")
129
+ return parser
130
+
131
+
132
+ # argparse reports an ArgumentTypeError from a type= function as a usage error naming the argument.
133
+ def _number(text: str, kind=float):
134
+ try:
135
+ return kind(text)
136
+ except ValueError:
137
+ raise argparse.ArgumentTypeError(f"{text!r} is not a number") from None
138
+
139
+
140
+ def _finite(text: str) -> float:
141
+ value = _number(text)
142
+ if not math.isfinite(value):
143
+ raise argparse.ArgumentTypeError(f"{text} is not a finite number")
144
+ return value
145
+
146
+
147
+ def _rate(text: str) -> float:
148
+ value = _number(text)
149
+ if not 0 < value <= gate_check.MAX_TARGET_FPR: # also false for nan
150
+ raise argparse.ArgumentTypeError(f"{text} is not a rate in (0, {gate_check.MAX_TARGET_FPR}], e.g. 0.01 for 1%")
151
+ return value
152
+
153
+
154
+ def _seed(text: str) -> int:
155
+ value = _number(text, int)
156
+ if value < 0:
157
+ raise argparse.ArgumentTypeError(f"{text} is negative; the seed must be 0 or more")
158
+ return value
159
+
160
+
161
+ def _setting(parser: _Parser, given: float | None, probe: Probe | None, key: str) -> float:
162
+ if given is not None:
163
+ return given
164
+ if probe is not None and key in probe.metadata:
165
+ return float(probe.metadata[key])
166
+ parser.error(f"--{key.replace('_', '-')} is required unless the probe records it")
167
+
168
+
169
+ def _check(reference, candidate, rerun, probe, threshold, target_fpr, seed=0, under_test=()) -> Report:
170
+ return check_snapshots(
171
+ Snapshot.load(reference),
172
+ Snapshot.load(candidate),
173
+ None if rerun is None else Snapshot.load(rerun),
174
+ threshold=threshold,
175
+ target_fpr=target_fpr,
176
+ probe=probe,
177
+ seed=seed,
178
+ changes_under_test=tuple(under_test),
179
+ )
180
+
181
+
182
+ def _inputs(reference, candidate, rerun, probe) -> dict:
183
+ """The input files by role, as render and to_json show them; None for a role not given."""
184
+ return {"reference": reference, "candidate": candidate, "rerun": rerun, "probe": probe}
185
+
186
+
187
+ def to_json(report: Report, inputs: dict) -> str:
188
+ out = {
189
+ "schema_version": JSON_SCHEMA_VERSION,
190
+ "hulder_version": version("hulder"),
191
+ "verdict": report.verdict.value,
192
+ "exit_code": report.verdict.exit_code,
193
+ "inputs": {k: None if v is None else str(v) for k, v in inputs.items()},
194
+ "report": {**dataclasses.asdict(report), "verdict": report.verdict.value},
195
+ }
196
+ return json.dumps(out, indent=2)
197
+
198
+
199
+ def render(report: Report, inputs: dict) -> str:
200
+ """The human-readable report. Changes under test come near the top, so a swapped
201
+ reference and candidate shows at a glance."""
202
+ lines = []
203
+ if report.verdict is Verdict.INCONCLUSIVE and report.shift_exceeds_noise_floor:
204
+ lines += [
205
+ (
206
+ f"Scores shifted beyond the noise floor: {report.decision_flips} decisions flipped "
207
+ f"vs {report.noise_floor_decision_flips} on the rerun, "
208
+ f"mean |Δscore| {report.mean_abs_shift:.3g} vs {report.noise_floor_mean_abs_shift:.3g}."
209
+ ),
210
+ "The serving change moved the scorer; whether its FPR crossed the tolerance is undecided.",
211
+ "",
212
+ ]
213
+ for role in ("reference", "candidate", "rerun", "probe"):
214
+ if inputs.get(role) is not None:
215
+ lines.append(f"{role.capitalize():<10} {inputs[role]}")
216
+ lines += [
217
+ (
218
+ f"Threshold {report.threshold:.4g} at a {report.target_fpr:.1%} target FPR; "
219
+ f"{report.n_benign:,} benign, {report.n_attack:,} attack"
220
+ ),
221
+ "",
222
+ ]
223
+ if report.changes_under_test:
224
+ width = max(len(key) for key, _, _ in report.changes_under_test)
225
+ lines.append("Changes under test (reference -> candidate)")
226
+ lines += [f" {key:<{width}} {ref!r} -> {cand!r}" for key, ref, cand in report.changes_under_test]
227
+ else:
228
+ lines.append("Changes under test: none found in the metadata")
229
+ lines += ["", f"Verdict: {report.verdict.value}", *_more_benign(report), ""]
230
+ lines += _performance(report)
231
+ lines += [""] + _shift(report)
232
+ if report.warnings:
233
+ lines += ["", "Warnings"] + [f" - {w}" for w in report.warnings]
234
+ lines += ["", f"Exit code {report.verdict.exit_code} ({report.verdict.value})"]
235
+ return "\n".join(line.rstrip() for line in lines)
236
+
237
+
238
+ def _more_benign(report: Report) -> list[str]:
239
+ if report.verdict is not Verdict.INCONCLUSIVE:
240
+ return []
241
+ if report.more_benign_needed == 0:
242
+ return [" The FPR condition is already decided; more benign examples alone won't resolve this."]
243
+ if report.more_benign_beyond_limit:
244
+ return [
245
+ f" More than {gate_check.MORE_BENIGN_LIMIT:,} benign examples in all would be needed to settle the FPR condition:",
246
+ f" {EDGE_CAVEAT}.",
247
+ ]
248
+ lines = [
249
+ (
250
+ f" {report.more_benign_needed:,} more benign examples would settle the FPR condition, "
251
+ "if the observed rate held."
252
+ )
253
+ ]
254
+ if report.more_benign_needed > EDGE_FACTOR * report.n_benign:
255
+ lines.append(f" Caveat: {EDGE_CAVEAT}.")
256
+ return lines
257
+
258
+
259
+ def _performance(report: Report) -> list[str]:
260
+ def rate(e):
261
+ return f"{e.value:.1%} [{e.low:.1%}, {e.high:.1%}]"
262
+
263
+ def auroc(e):
264
+ return f"{e.value:.4f} [{e.low:.4f}, {e.high:.4f}]"
265
+
266
+ return [
267
+ f"{'Performance at threshold ' + format(report.threshold, '.4g'):<34}{'reference':<{COLUMN}}candidate",
268
+ f" {'FPR':<32}{rate(report.reference_fpr):<{COLUMN}}{rate(report.candidate_fpr)}",
269
+ f" {'FNR':<32}{rate(report.reference_fnr):<{COLUMN}}{rate(report.candidate_fnr)}",
270
+ f" {'AUROC':<32}{auroc(report.reference_auroc):<{COLUMN}}{auroc(report.candidate_auroc)}",
271
+ (
272
+ f" {'ΔAUROC (candidate - reference)':<32}{'':<{COLUMN}}{report.delta_auroc.value:+.4f} "
273
+ f"[{report.delta_auroc.low:+.4f}, {report.delta_auroc.high:+.4f}]"
274
+ ),
275
+ (
276
+ f" Bands: candidate FPR upper bound <= {gate_check.FPR_BAND_HIGH * report.target_fpr:.1%}, "
277
+ f"FNR upper bound <= reference FNR + {gate_check.FNR_TOLERANCE}, ΔAUROC lower bound >= -{gate_check.AUROC_TOLERANCE}"
278
+ ),
279
+ ]
280
+
281
+
282
+ def _shift(report: Report) -> list[str]:
283
+ if report.noise_floor_measured:
284
+ rerun_flips = str(report.noise_floor_decision_flips)
285
+ rerun_shift = f"{report.noise_floor_mean_abs_shift:.3g}"
286
+ exceeds = "yes" if report.shift_exceeds_noise_floor else "no"
287
+ else:
288
+ rerun_flips = rerun_shift = ""
289
+ exceeds = (
290
+ "not measured (rerun identical to reference)" if report.rerun_identical_to_reference else "not measured"
291
+ )
292
+ lines = [
293
+ f"{'Shift (paired, same prompts)':<34}{'candidate':<{COLUMN}}rerun",
294
+ f" {'decisions flipped':<32}{f'{report.decision_flips} ({report.decision_flip_rate:.1%})':<{COLUMN}}{rerun_flips}",
295
+ f" {'mean |Δscore|':<32}{f'{report.mean_abs_shift:.3g}':<{COLUMN}}{rerun_shift}",
296
+ ]
297
+ if report.mean_abs_logit_shift is not None:
298
+ rerun_logit = report.noise_floor_mean_abs_logit_shift
299
+ lines.append(
300
+ f" {'mean |Δlogit|':<32}{f'{report.mean_abs_logit_shift:.3g}':<{COLUMN}}"
301
+ f"{'' if rerun_logit is None else f'{rerun_logit:.3g}'}"
302
+ )
303
+ if report.shift_over_noise_floor is not None:
304
+ e = report.shift_over_noise_floor
305
+ lines.append(f" {'candidate - rerun mean |Δscore|':<32}{e.value:.3g} [{e.low:.3g}, {e.high:.3g}]")
306
+ lines.append(f" {'exceeds the noise floor':<32}{exceeds}")
307
+ return lines
308
+
309
+
310
+ # The bundled demo: the rerun as candidate (nothing changed but run-to-run noise), then
311
+ # AWQ int4 as candidate with the rerun as noise floor. Expected verdicts from
312
+ # docs/results/phase1/real_check.md.
313
+ QUICKSTART = (
314
+ ("The fp16 rerun as candidate: nothing changed but run-to-run noise.", "demo_fp16_run1.npz", None, Verdict.PASS),
315
+ (
316
+ "AWQ int4 as candidate, with the fp16 rerun as noise floor.",
317
+ "demo_awq.npz",
318
+ "demo_fp16_run1.npz",
319
+ Verdict.INCONCLUSIVE,
320
+ ),
321
+ )
322
+
323
+
324
+ def _quickstart() -> int:
325
+ data = files("hulder.data")
326
+ probe_path = Path(str(data / "probe_demo.npz"))
327
+ probe = Probe.load(probe_path)
328
+ threshold, target_fpr = probe.metadata["threshold"], probe.metadata["target_fpr"]
329
+ reference = Path(str(data / "demo_fp16.npz"))
330
+ as_expected = True
331
+ for number, (title, candidate, rerun, expected) in enumerate(QUICKSTART, 1):
332
+ candidate = Path(str(data / candidate))
333
+ rerun = None if rerun is None else Path(str(data / rerun))
334
+ report = _check(reference, candidate, rerun, probe, threshold, target_fpr)
335
+ inputs = _inputs(reference.name, candidate.name, None if rerun is None else rerun.name, probe_path.name)
336
+ print(f"=== Check {number}: {title} Expect {expected.value}.\n")
337
+ print(render(report, inputs))
338
+ print()
339
+ as_expected &= report.verdict is expected
340
+ if not as_expected:
341
+ print("hulder: a quickstart check did not give its expected verdict", file=sys.stderr)
342
+ return 0 if as_expected else 1
hulder/data/NOTICE.md ADDED
@@ -0,0 +1,14 @@
1
+ # Bundled quickstart data
2
+
3
+ `hulder quickstart` checks these files. They are for demonstration only. The demo probe is not a prompt-attack monitor (docs/design.md §7).
4
+
5
+ | File | What it is |
6
+ |---|---|
7
+ | `probe_demo.npz` | The demo probe: logistic regression on Qwen2.5-1.5B-Instruct's layer-20 residual stream at the last prompt token, trained on 600 boundary-pairs examples. Threshold 0.395 for a 5% FPR target. |
8
+ | `demo_fp16.npz` | Reference snapshot: that layer's fp16 activations for the 360 boundary-pairs calibration examples (180 benign, 180 attack), from vLLM 0.30.0 on an RTX 2070. |
9
+ | `demo_fp16_run1.npz` | The same serving config run again in a fresh engine (the rerun). |
10
+ | `demo_awq.npz` | Candidate snapshot: the same examples under the official AWQ int4 checkpoint, `Qwen/Qwen2.5-1.5B-Instruct-AWQ`. |
11
+
12
+ They were produced by `experiments/phase1/probe/snapshots.py` (snapshot format 2) and `experiments/phase1/probe/train_demo.py`. The prompts themselves are not bundled. The live tutorial in the hulder repository (`examples/live_demo/`) carries them with their own attribution.
13
+
14
+ Contains data derived from 3nesdeniz/agentic-prompt-injection-boundary-pairs (Deniz 2026, doi:10.5281/zenodo.21378693), licensed under CC-BY-4.0 (https://creativecommons.org/licenses/by/4.0/). These files are activations and a probe computed from that data. The activations come from Qwen2.5-1.5B-Instruct and its AWQ checkpoint, both Apache-2.0.
Binary file
Binary file
Binary file
Binary file
File without changes
hulder/gate/atomic.py ADDED
@@ -0,0 +1,41 @@
1
+ """Files that appear whole or not at all, so an interrupted write never leaves half a file."""
2
+
3
+ import os
4
+ import tempfile
5
+ from collections.abc import Iterator
6
+ from contextlib import contextmanager
7
+ from pathlib import Path
8
+ from typing import BinaryIO
9
+
10
+ import numpy as np
11
+
12
+
13
+ @contextmanager
14
+ def atomic_write(path: str | Path) -> Iterator[BinaryIO]:
15
+ """A binary file to write that replaces path only once it is complete. It is written to a
16
+ temporary file in path's directory, so the final os.replace is a rename on one filesystem."""
17
+ path = Path(path)
18
+ fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp")
19
+ try:
20
+ with os.fdopen(fd, "wb") as f:
21
+ yield f
22
+ f.flush()
23
+ os.fsync(f.fileno())
24
+ os.replace(temporary, path)
25
+ except BaseException:
26
+ Path(temporary).unlink(missing_ok=True)
27
+ raise
28
+
29
+
30
+ def savez_atomically(path: str | Path, **arrays) -> None:
31
+ """np.savez through atomic_write. Like np.savez, it adds .npz to a path without it."""
32
+ path = Path(path)
33
+ if path.suffix != ".npz":
34
+ path = path.with_name(path.name + ".npz")
35
+ with atomic_write(path) as f:
36
+ np.savez(f, **arrays)
37
+
38
+
39
+ def write_text_atomically(path: str | Path, text: str) -> None:
40
+ with atomic_write(path) as f:
41
+ f.write(text.encode())