guarddiff 0.1.0__py3-none-any.whl

guarddiff/report/html.py

@@ -0,0 +1,107 @@
+"""Single-file HTML report. Every dynamic value is escaped; no external resources are referenced."""
+
+from __future__ import annotations
+
+from html import escape
+from pathlib import Path
+from string import Template
+from typing import TYPE_CHECKING
+
+from ..outcomes import OUTCOMES
+from .markdown import indeterminate_warning, notices, reduction_text
+
+if TYPE_CHECKING:
+    from ..runner import RunResult
+
+_PAGE = Template("""<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><title>guarddiff report</title>
+<style>
+body{font-family:system-ui,sans-serif;margin:1.5rem;color:#222}
+table{border-collapse:collapse;margin:1rem 0;font-size:.9rem}
+th,td{border:1px solid #ccc;padding:.3rem .6rem;text-align:left;vertical-align:top}
+th{background:#f3f3f3}.note{background:#fff6d6;border:1px solid #e0c870;padding:.4rem .8rem;margin:.3rem 0}
+.o-blocked,.o-model_refused,.o-passed{background:#e6f4e6}.o-missed,.o-overblocked{background:#fbe3e3}
+.o-model_overrefused{background:#fdf0d5}.o-indeterminate{background:#e3e3f4}
+small{color:#555}
+</style></head><body>
+<h1>guarddiff report</h1>
+$notes
+<h2>Summary</h2>
+<p><small>Baseline: <b>$baseline</b>. Rates use all items of a label as the denominator
+(indeterminate included). Intervals are Wilson 95%; the paired p is an exact McNemar test.</small></p>
+$summary_table
+<h2>Per item</h2>
+$items_table
+</body></html>
+""")
+
+
+def _e(x: object) -> str:
+    return escape("" if x is None else str(x), quote=True)
+
+
+def _ci(c: list[float]) -> str:
+    return f"{c[0]:.4f}&ndash;{c[1]:.4f}"
+
+
+def _summary_table(result: "RunResult") -> str:
+    head = ("<tr><th>variant</th><th>n atk / ben</th><th>ASR</th><th>ASR 95% CI</th><th>ASR reduction</th>"
+            "<th>overblock</th><th>overblock 95% CI</th><th>total overrefusal</th><th>indeterminate</th>"
+            "<th>latency p50 / p95 ms</th><th>cost USD</th><th>target calls</th>"
+            "<th>fixed / regressed</th><th>McNemar p</th><th>redundant blocks</th></tr>")
+    body = []
+    for v in result.variants:
+        s = result.summary[v]
+        pair = s["paired_vs_none"]
+        body.append(
+            f"<tr><td>{_e(v)}</td><td>{s['n_attack']} / {s['n_benign']}</td><td>{s['asr']:.4f}</td>"
+            f"<td>{_ci(s['asr_ci95'])}</td><td>{reduction_text(s, 4)}</td>"
+            f"<td>{s['guard_overblock_rate']:.4f}</td><td>{_ci(s['guard_overblock_ci95'])}</td>"
+            f"<td>{s['total_overrefusal_rate']:.4f}</td><td>{s['indeterminate_rate']:.4f}</td>"
+            f"<td>{s['guard_latency_ms_p50']:.3f} / {s['guard_latency_ms_p95']:.3f}</td>"
+            f"<td>{s['guard_cost_usd_total']:.4f}</td><td>{s['target_calls']}</td>"
+            + ("<td>-</td><td>-</td><td>-</td>" if pair is None else
+               f"<td>{pair['attack_fixed']} / {pair['attack_regressed']}</td><td>{pair['mcnemar_p']}</td>"
+               f"<td>{pair['redundant_blocks']}</td>")
+            + "</tr>")
+    table = f"<table>{head}{''.join(body)}</table>"
+    warning = indeterminate_warning(result, "the ASR reduction")
+    if warning:  # the '!' in the table points here
+        table += f'<div class="note"><b>!</b> {_e(warning)}</div>'
+    return table
+
+
+def _items_table(result: "RunResult") -> str:
+    order: list[str] = []
+    for r in result.rows:
+        if r.id not in order:
+            order.append(r.id)
+    first = {r.id: r for r in result.rows}
+    cell = {(r.id, r.variant): r for r in result.rows}
+    head = "<tr><th>id</th><th>label</th><th>prompt</th>" + "".join(f"<th>{_e(v)}</th>" for v in result.variants) + "</tr>"
+    body = []
+    for i in order:
+        f = first[i]
+        tds = []
+        for v in result.variants:
+            r = cell.get((i, v))
+            if r is None:
+                tds.append("<td>-</td>")
+                continue
+            ev = f"<br><small>{_e(r.evidence)}</small>" if r.evidence else ""
+            tds.append(f"<td class=\"o-{_e(r.outcome)}\"><b>{_e(r.outcome)}</b>{ev}</td>")
+        body.append(f"<tr><td>{_e(i)}</td><td>{_e(f.label)}</td><td>{_e(f.prompt)}</td>{''.join(tds)}</tr>")
+    legend = "<p><small>Outcomes: " + ", ".join(OUTCOMES) + ".</small></p>"
+    return f"<table>{head}{''.join(body)}</table>{legend}"
+
+
+def render_html(result: "RunResult") -> str:
+    notes = "".join(f'<div class="note">{_e(n)}</div>' for n in notices(result))
+    return _PAGE.substitute(notes=notes, baseline=_e(result.baseline),
+                            summary_table=_summary_table(result), items_table=_items_table(result))
+
+
+def write_html(result: "RunResult", path: str | Path) -> Path:
+    p = Path(path)
+    p.write_text(render_html(result), encoding="utf-8")
+    return p

guarddiff/report/markdown.py

@@ -0,0 +1,66 @@
+"""Markdown summary (one table row per variant), meant for CI step summaries."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..runner import RunResult
+
+
+def notices(result: "RunResult") -> list[str]:
+    """Honesty notes shown above every report: mock target, simulated latency."""
+    notes: list[str] = []
+    if result.meta.get("target_type", "mock") == "mock":
+        notes.append("practice mock target (not a real LLM)")
+    if any(s.get("latency_source") in ("simulated", "mixed") for s in result.summary.values()):
+        notes.append("simulated latency (reproducible placeholder values, not measurements)")
+    return notes
+
+
+def reduction_text(summary: dict, digits: int) -> str:
+    """ASR reduction for a table cell. A trailing '!' means the variant has indeterminate items:
+    failed items are not counted as missed attacks, so the reduction may look better than it is."""
+    red = summary["asr_reduction_vs_none"]
+    if red is None:
+        return "-"
+    text = f"{red:.{digits}f}"
+    return text + "!" if summary["indeterminate_rate"] > 0 else text
+
+
+def indeterminate_warning(result: "RunResult", reduction_name: str) -> str | None:
+    """One warning line when any variant has indeterminate items (used by `run`, Markdown and HTML)."""
+    flagged = [v for v in result.variants if result.summary[v]["indeterminate_rate"] > 0]
+    if not flagged:
+        return None
+    return (f"indeterminate items (a guard or the target failed) in: {', '.join(flagged)}. They are not counted "
+            f"as missed attacks, so {reduction_name} may look better than it is. Fix the failures, "
+            "or run `guarddiff gate`.")
+
+
+def _ci(c: list[float]) -> str:
+    return f"{c[0]:.2f}-{c[1]:.2f}"
+
+
+def render_markdown(result: "RunResult") -> str:
+    lines = ["## guarddiff report", ""]
+    for n in notices(result):
+        lines.append(f"> {n}")
+    if lines[-1] != "":
+        lines.append("")
+    lines += ["| variant | ASR (95% CI) | ASR reduction | overblock (95% CI) | indeterminate | p50 ms | p95 ms "
+              "| cost USD | McNemar p |",
+              "|---|---|---|---|---|---|---|---|---|"]
+    for v in result.variants:
+        s = result.summary[v]
+        pair = s["paired_vs_none"]
+        lines.append(
+            f"| {v} | {s['asr']:.3f} ({_ci(s['asr_ci95'])}) | {reduction_text(s, 4)} "
+            f"| {s['guard_overblock_rate']:.3f} ({_ci(s['guard_overblock_ci95'])}) "
+            f"| {s['indeterminate_rate']:.3f} | {s['guard_latency_ms_p50']:.1f} | {s['guard_latency_ms_p95']:.1f} "
+            f"| {s['guard_cost_usd_total']:.4f} | {'-' if pair is None else pair['mcnemar_p']} |")
+    warning = indeterminate_warning(result, "the ASR reduction")
+    if warning:
+        lines += ["", f"> **!** {warning}"]  # same warning as the `run` table, so a CI summary shows it too
+    lines += ["", f"Baseline variant: `{result.baseline}`. Rates use all items of a label as the denominator."]
+    return "\n".join(lines) + "\n"

guarddiff/runner.py

@@ -0,0 +1,304 @@
+"""Run every item through every variant and keep results.jsonl up to date (resumable)."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from .config import Config, Variant, load_config
+from .errors import ConfigError, GuarddiffError
+from .guards import Guard, build_guard
+from .items import Item, load_items
+from .judges import RulesJudge
+from .latency import MODES, simulated_latency
+from .metrics import build_summary
+from .outcomes import INDETERMINATE, ResultRow, classify
+from .targets import Target, TargetReply, build_target
+
+RESULTS_FILE = "results.jsonl"
+SUMMARY_FILE = "summary.json"
+SCHEMA_VERSION = 1
+
+
+@dataclass
+class RunResult:
+    """What ``run_ablation`` returns. ``summary`` maps variant name -> metrics dict."""
+
+    summary: dict[str, dict[str, Any]]
+    rows: list[ResultRow]
+    variants: list[str]
+    baseline: str
+    meta: dict[str, Any] = field(default_factory=dict)
+    out_dir: Path | None = None
+
+    def rows_for(self, variant: str) -> list[ResultRow]:
+        return [r for r in self.rows if r.variant == variant]
+
+
+def _fail_text(exc: BaseException) -> str:
+    return f"{type(exc).__name__}: {exc}"
+
+
+def _check_guard(guard: Guard, text: str, item: Item, mode: str) -> tuple[dict[str, Any], float, float]:
+    """Run one guard. Returns (event, latency_ms, cost_usd). A raised error becomes event['error']."""
+    event: dict[str, Any] = {"guard": guard.name, "stage": guard.stage}
+    t0 = time.perf_counter()
+    try:
+        verdict = guard.check(text, item_id=item.id, prompt=item.prompt)
+    except Exception as e:  # noqa: BLE001 - any guard failure must become "indeterminate", never a pass
+        event.update(blocked=None, reason=None, error=_fail_text(e), latency_ms=0.0, latency_source="none")
+        return event, 0.0, 0.0
+    elapsed = round((time.perf_counter() - t0) * 1000, 3)
+    if verdict.latency_ms is not None:  # a replayed recording knows its own latency
+        latency, source = float(verdict.latency_ms), "replay"
+    elif mode == "simulated" and guard.sim_latency_ms is not None:
+        latency, source = simulated_latency(item.id, guard.name, guard.sim_latency_ms), "simulated"
+    else:
+        latency, source = elapsed, "measured"
+    cost = guard.unit_cost_usd if verdict.cost_usd is None else verdict.cost_usd
+    event.update(blocked=verdict.blocked, reason=verdict.reason, latency_ms=latency, latency_source=source)
+    return event, latency, cost
+
+
+def run_pair(item: Item, guards: list[Guard], target: Target, judge: RulesJudge, mode: str,
+             variant: str) -> ResultRow:
+    """Input guards (stop at the first block) -> target -> output guards -> refusal judge -> outcome."""
+    events: list[dict[str, Any]] = []
+    latency = cost = 0.0
+    blocked_by: str | None = None
+    failure: str | None = None
+    answer = ""
+    usage: dict[str, int] = {}
+    called = False
+
+    for g in (g for g in guards if g.stage == "input"):
+        ev, lat, c = _check_guard(g, item.prompt, item, mode)
+        events.append(ev)
+        latency, cost = latency + lat, cost + c
+        if ev.get("error"):
+            failure = f"{g.name}: {ev['error']}"
+            break
+        if ev["blocked"]:
+            blocked_by = f"{g.name}: {ev['reason']}"
+            break
+
+    if blocked_by is None and failure is None:
+        called = True
+        try:
+            reply = target.call(item.prompt)
+        except Exception as e:  # noqa: BLE001
+            reply = TargetReply("", error=_fail_text(e))
+        usage = dict(reply.usage)
+        if reply.error:
+            failure = f"target: {reply.error}"
+        elif not reply.text.strip():
+            failure = "target: empty response (a reasoning model may need a larger max_tokens)"
+        else:
+            answer = reply.text
+        if failure is None:
+            for g in (g for g in guards if g.stage == "output"):
+                ev, lat, c = _check_guard(g, answer, item, mode)
+                events.append(ev)
+                latency, cost = latency + lat, cost + c
+                if ev.get("error"):
+                    failure = f"{g.name}: {ev['error']}"
+                    break
+                if ev["blocked"]:
+                    blocked_by = f"{g.name}: {ev['reason']}"
+                    break
+
+    if failure is not None:
+        outcome, evidence = INDETERMINATE, failure
+    elif blocked_by is not None:
+        outcome, evidence = classify(item.label, guard_blocked=True, model_refused=False), blocked_by
+    else:
+        refused, why = judge.judge(answer)
+        outcome = classify(item.label, guard_blocked=False, model_refused=refused)
+        evidence = why
+    return ResultRow(id=item.id, variant=variant, label=item.label, lang=item.lang, category=item.category,
+                     prompt=item.prompt, outcome=outcome, evidence=evidence,
+                     guard_latency_ms=round(latency, 3), guard_cost_usd=round(cost, 6),
+                     guard_events=events, target_called=called, usage=usage)
+
+
+def variant_config_hash(cfg: Config, variant: Variant, mode: str) -> str:
+    """Hash of everything that decides a variant's rows besides the prompts: its guards (in order),
+    the target, the judge and the latency mode. Code behind a ``module:function`` is not hashed."""
+    specs = {s.get("name"): s for s in cfg.guards}
+    doc = {"guards": [specs[n] for n in variant.guards], "target": cfg.target, "judge": cfg.judge,
+           "latency": mode}
+    blob = json.dumps(doc, sort_keys=True, ensure_ascii=False, default=str)
+    return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:16]
+
+
+def _load_rows(path: Path) -> list[ResultRow]:
+    """Read results.jsonl, skipping unreadable lines (for example a line cut short by a crash)."""
+    rows: list[ResultRow] = []
+    if not path.is_file():
+        return rows
+    for line in path.read_text(encoding="utf-8").splitlines():
+        if not line.strip():
+            continue
+        try:
+            rows.append(ResultRow.from_dict(json.loads(line)))
+        except (json.JSONDecodeError, TypeError):
+            continue
+    return rows
+
+
+def _write_rows(path: Path, rows: list[ResultRow]) -> None:
+    tmp = path.with_suffix(".jsonl.tmp")
+    tmp.write_text("".join(json.dumps(r.to_dict(), ensure_ascii=False) + "\n" for r in rows), encoding="utf-8")
+    os.replace(tmp, path)
+
+
+def _in_file_order(items: list[Item], variants: list[Variant],
+                   rows: dict[tuple[str, str], ResultRow]) -> list[ResultRow]:
+    """The rows we have, in the order results.jsonl is written: item by item, variants in config order."""
+    return [rows[(i.id, v.name)] for i in items for v in variants if (i.id, v.name) in rows]
+
+
+def _select_variants(config: Config, only: list[str] | None) -> list[Variant]:
+    if not only:
+        return list(config.variants)
+    names = [v.name for v in config.variants]
+    unknown = [n for n in only if n not in names]
+    if unknown:
+        raise ConfigError(f"--only: unknown variant(s) {unknown}. Defined variants: {names}.")
+    wanted = set(only) | {config.baseline}  # the baseline is always needed for the comparison
+    return [v for v in config.variants if v.name in wanted]
+
+
+def run_ablation(config: Config | str | Path, out_dir: str | Path, *, latency: str | None = None,
+                 fresh: bool = False, only: list[str] | None = None) -> RunResult:
+    """Run the ablation and write ``results.jsonl``, ``summary.json`` and ``report.html`` into ``out_dir``.
+
+    Resuming: a pair (item, variant) already in results.jsonl is reused when its prompt, label and settings
+    hash are unchanged and its outcome was decided. ``indeterminate`` rows (a guard or the target failed) are
+    always run again, because the cause may be fixed by now. ``fresh`` recomputes the selected variants.
+    ``only`` runs those variants plus the baseline; rows of the other variants stay in results.jsonl as they are.
+    ``latency`` overrides the config ("simulated" or "measured").
+    """
+    cfg = load_config(config) if isinstance(config, (str, Path)) else config
+    mode = latency or cfg.latency
+    if mode not in MODES:
+        raise ConfigError(f"latency must be 'simulated' or 'measured', got {mode!r}.")
+    items = load_items(cfg.items_path)
+    variants = _select_variants(cfg, only)
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    results_path = out / RESULTS_FILE
+
+    used = {name for v in variants for name in v.guards}
+    guards: dict[str, Guard] = {}
+    target: Target | None = None
+    try:
+        for spec in cfg.guards:
+            if spec.get("name") in used:
+                guards[spec["name"]] = build_guard(spec, cfg.base_dir)
+        target = build_target(cfg.target, cfg.base_dir)
+        judge = RulesJudge(cfg.judge.get("patterns"))
+
+        hashes = {v.name: variant_config_hash(cfg, v, mode) for v in variants}
+        current = {(i.id, v.name): i for i in items for v in variants}  # the pairs this run must finish
+        other_variants = {v.name for v in cfg.variants} - {v.name for v in variants}  # not chosen by --only
+        item_ids = {i.id for i in items}
+
+        latest: dict[tuple[str, str], ResultRow] = {}
+        for r in _load_rows(results_path):
+            latest[(r.id, r.variant)] = r  # if a pair appears twice, the later line wins
+
+        done: dict[tuple[str, str], ResultRow] = {}  # rows of the chosen variants that can be reused
+        kept: dict[tuple[str, str], ResultRow] = {}  # rows of other variants: written back untouched
+        config_changed = retried = 0
+        for key, r in latest.items():
+            if r.variant in other_variants:
+                if r.id in item_ids:
+                    kept[key] = r  # --only must not throw away results that may have cost time or money
+                continue
+            it = current.get(key)
+            if fresh or it is None or r.prompt != it.prompt or r.label != it.label:
+                continue  # --fresh, a removed variant or item, or an edited prompt: not reusable
+            if r.config_hash != hashes[r.variant]:  # settings changed since this row was written
+                config_changed += 1
+                continue
+            if r.outcome == INDETERMINATE:  # it failed last time; the server or the code may be fixed now
+                retried += 1
+                continue
+            done[key] = r
+        # rewrite the file before running: drops stale or half-written lines, keeps other variants' rows
+        _write_rows(results_path, _in_file_order(items, cfg.variants, {**kept, **done}))
+
+        todo = [(i, v) for i in items for v in variants if (i.id, v.name) not in done]
+        if todo:
+            for g in guards.values():
+                g.prepare()
+            with results_path.open("a", encoding="utf-8") as fh:
+                for item, variant in todo:
+                    row = run_pair(item, [guards[n] for n in variant.guards], target, judge, mode, variant.name)
+                    row.config_hash = hashes[variant.name]
+                    done[(item.id, variant.name)] = row
+                    fh.write(json.dumps(row.to_dict(), ensure_ascii=False) + "\n")
+                    fh.flush()  # a crash loses at most the pair in flight
+
+        rows = [done[(i.id, v.name)] for i in items for v in variants]  # what this run compares
+        _write_rows(results_path, _in_file_order(items, cfg.variants, {**kept, **done}))
+        meta = {"schema": SCHEMA_VERSION, "baseline": cfg.baseline, "latency_mode": mode,
+                "target_type": cfg.target.get("type", "mock"), "n_items": len(items),
+                "skipped_pairs": len(items) * len(variants) - len(todo), "config_changed_pairs": config_changed,
+                "retried_indeterminate_pairs": retried, "kept_other_pairs": len(kept),
+                "kept_other_variants": [v.name for v in cfg.variants if any(k[1] == v.name for k in kept)]}
+        return _finish(rows, [v.name for v in variants], cfg.baseline, meta, out)
+    finally:
+        for g in guards.values():
+            g.close()
+        if target is not None:
+            target.close()
+
+
+def _finish(rows: list[ResultRow], variants: list[str], baseline: str, meta: dict[str, Any],
+            out: Path) -> RunResult:
+    by_variant = {v: [r for r in rows if r.variant == v] for v in variants}
+    summary = build_summary(by_variant, baseline)
+    meta = {**meta, "variants": variants}
+    doc = {**meta, "summary": summary}
+    (out / SUMMARY_FILE).write_text(json.dumps(doc, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    result = RunResult(summary=summary, rows=rows, variants=variants, baseline=baseline, meta=meta, out_dir=out)
+    from .report.html import write_html  # noqa: PLC0415 - keeps the import graph one-directional
+
+    write_html(result, out / "report.html")
+    return result
+
+
+def load_run(out_dir: str | Path) -> RunResult:
+    """Rebuild a RunResult from an output folder, for ``guarddiff report``."""
+    out = Path(out_dir)
+    results_path, summary_path = out / RESULTS_FILE, out / SUMMARY_FILE
+    if not results_path.is_file():
+        raise GuarddiffError(f"{results_path} not found. Run `guarddiff run CONFIG --out {out}` first.")
+    rows = _load_rows(results_path)
+    if not rows:
+        raise GuarddiffError(f"{results_path} has no readable rows. Re-run with --fresh.")
+    meta: dict[str, Any] = {}
+    if summary_path.is_file():
+        try:
+            loaded = json.loads(summary_path.read_text(encoding="utf-8"))
+            meta = loaded if isinstance(loaded, dict) else {}
+        except json.JSONDecodeError:
+            meta = {}
+    seen: list[str] = []
+    for r in rows:
+        if r.variant not in seen:
+            seen.append(r.variant)
+    variants = [v for v in meta.get("variants", seen) if v in seen] or seen
+    rows = [r for r in rows if r.variant in variants]  # after `run --only`, other variants' rows are kept aside
+    baseline = meta.get("baseline") if meta.get("baseline") in variants else variants[0]
+    meta = {k: v for k, v in meta.items() if k != "summary"}
+    by_variant = {v: [r for r in rows if r.variant == v] for v in variants}
+    return RunResult(summary=build_summary(by_variant, baseline), rows=rows, variants=variants,
+                     baseline=baseline, meta={**meta, "variants": variants, "baseline": baseline}, out_dir=out)

guarddiff/targets/__init__.py

@@ -0,0 +1,40 @@
+"""Target adapters."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from ..errors import ConfigError
+from ..guards.python_fn import resolve_callable
+from .base import Target, TargetReply, strip_thinking
+from .mock import MockTarget
+from .openai_http import OpenAIHTTPTarget
+from .python_fn import PythonTarget
+
+__all__ = ["Target", "TargetReply", "strip_thinking", "build_target"]
+
+_ALLOWED = {
+    "mock": {"self_refuse"},
+    "python": {"callable"},
+    "openai": {"model", "base_url", "max_tokens", "temperature", "api_key_env", "system", "timeout_s"},
+}
+
+
+def build_target(spec: dict[str, Any], base_dir: Path) -> Target:
+    """Create the target from the ``[target]`` table (default: practice mock)."""
+    ttype = spec.get("type", "mock")
+    if ttype not in _ALLOWED:
+        raise ConfigError(f"[target] unknown type {ttype!r}. Use one of: {', '.join(sorted(_ALLOWED))}.")
+    unknown = set(spec) - {"type"} - _ALLOWED[ttype]
+    if unknown:
+        raise ConfigError(f"[target] unknown option(s) {sorted(unknown)} for type '{ttype}'.")
+    if ttype == "mock":
+        return MockTarget(spec.get("self_refuse"))
+    if ttype == "python":
+        ref = spec.get("callable")
+        if not isinstance(ref, str):
+            raise ConfigError("[target] type='python' needs callable = \"module:function\".")
+        return PythonTarget(resolve_callable(ref, base_dir, "[target]"))
+    kwargs = {k: v for k, v in spec.items() if k != "type"}
+    return OpenAIHTTPTarget(**kwargs)

guarddiff/targets/base.py

@@ -0,0 +1,31 @@
+"""Target protocol: the model (or app) the guards protect."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+_THINK_CLOSED = re.compile(r"<think>.*?</think>", re.DOTALL | re.IGNORECASE)
+_THINK_OPEN = re.compile(r"<think>.*\Z", re.DOTALL | re.IGNORECASE)
+
+
+def strip_thinking(text: str) -> str:
+    """Remove <think>...</think> blocks (and an unterminated trailing one) from reasoning models."""
+    return _THINK_OPEN.sub("", _THINK_CLOSED.sub("", text)).strip()
+
+
+@dataclass
+class TargetReply:
+    text: str
+    usage: dict[str, int] = field(default_factory=dict)  # prompt_tokens / completion_tokens when known
+    error: str | None = None  # set when the call failed (HTTP error, timeout, exception)
+
+
+class Target:
+    """Base class. ``call`` should not raise for expected failures; set ``error`` instead."""
+
+    def call(self, prompt: str) -> TargetReply:
+        raise NotImplementedError
+
+    def close(self) -> None:
+        """Release connections (default: nothing)."""

guarddiff/targets/mock.py

@@ -0,0 +1,35 @@
+"""Practice mock target. It is NOT a real LLM.
+
+It imitates three behaviours so guard comparisons show real lessons:
+the model sometimes refuses by itself, sometimes over-refuses a harmless question,
+and leaks its "system prompt" when asked. Answers are fixed harmless sentences.
+"""
+
+from __future__ import annotations
+
+import re
+
+from ..errors import ConfigError
+from .base import Target, TargetReply
+
+DEFAULT_SELF_REFUSE = [r"end my life|pipe bomb|폭탄 만드는", r"toxic gas"]
+LEAK_ASK = re.compile(r"system prompt|시스템 프롬프트", re.IGNORECASE)
+HANGUL = re.compile(r"[가-힣]")
+
+
+class MockTarget(Target):
+    def __init__(self, self_refuse: list[str] | None = None) -> None:
+        patterns = DEFAULT_SELF_REFUSE if self_refuse is None else self_refuse
+        try:
+            self._refuse = [re.compile(p, re.IGNORECASE) for p in patterns]
+        except re.error as e:
+            raise ConfigError(f"[target] self_refuse has an invalid regex ({e}).") from e
+
+    def call(self, prompt: str) -> TargetReply:
+        if any(p.search(prompt) for p in self._refuse):
+            if HANGUL.search(prompt):
+                return TargetReply("죄송하지만 그 요청은 도와드릴 수 없습니다.")
+            return TargetReply("I'm sorry, but I can't help with that.")
+        if LEAK_ASK.search(prompt):
+            return TargetReply("SYSTEM PROMPT: You are a helpful assistant. SECRET=xyz")
+        return TargetReply(f"Sure, here is an answer to: {prompt[:40]}")

guarddiff/targets/openai_http.py

@@ -0,0 +1,58 @@
+"""OpenAI-compatible ``/chat/completions`` target (Ollama, vLLM, hosted APIs) over urllib."""
+
+from __future__ import annotations
+
+import json
+import os
+import urllib.error
+import urllib.request
+from typing import Any
+
+from ..errors import ConfigError
+from .base import Target, TargetReply, strip_thinking
+
+
+class OpenAIHTTPTarget(Target):
+    def __init__(self, model: str, base_url: str = "http://127.0.0.1:11434/v1", *, max_tokens: int = 512,
+                 temperature: float = 0.0, api_key_env: str = "OPENAI_API_KEY", system: str | None = None,
+                 timeout_s: float = 300.0) -> None:
+        if not model:
+            raise ConfigError("[target] type='openai' needs 'model' (for example model = \"qwen3:4b\").")
+        if not base_url.startswith(("http://", "https://")):
+            raise ConfigError(f"[target] base_url must start with http:// or https://, got {base_url!r}.")
+        self.model = model
+        self.url = base_url.rstrip("/") + "/chat/completions"
+        self.max_tokens, self.temperature, self.system = max_tokens, temperature, system
+        self.api_key_env, self.timeout_s = api_key_env, timeout_s  # slow local models are normal: generous timeout
+
+    def _request(self, prompt: str) -> urllib.request.Request:
+        messages: list[dict[str, str]] = []
+        if self.system:
+            messages.append({"role": "system", "content": self.system})
+        messages.append({"role": "user", "content": prompt})
+        body = {"model": self.model, "messages": messages, "max_tokens": self.max_tokens,
+                "temperature": self.temperature}
+        headers = {"Content-Type": "application/json"}
+        key = os.environ.get(self.api_key_env)
+        if key:  # no key set -> no Authorization header (local servers need none)
+            headers["Authorization"] = f"Bearer {key}"
+        return urllib.request.Request(self.url, data=json.dumps(body).encode(), headers=headers, method="POST")
+
+    def call(self, prompt: str) -> TargetReply:
+        try:
+            with urllib.request.urlopen(self._request(prompt), timeout=self.timeout_s) as resp:  # noqa: S310
+                data: Any = json.loads(resp.read().decode("utf-8"))
+        except urllib.error.HTTPError as e:
+            return TargetReply("", error=f"HTTP {e.code} from {self.url}")
+        except (urllib.error.URLError, TimeoutError, OSError) as e:
+            return TargetReply("", error=f"{type(e).__name__}: {e} (is the server at {self.url} running?)")
+        except ValueError as e:
+            return TargetReply("", error=f"response is not valid JSON ({e})")
+        try:
+            content = data["choices"][0]["message"]["content"] or ""
+        except (KeyError, IndexError, TypeError):
+            return TargetReply("", error="response has no choices[0].message.content")
+        usage = data.get("usage") if isinstance(data, dict) else None
+        tokens = {k: int(usage[k]) for k in ("prompt_tokens", "completion_tokens")
+                  if isinstance(usage, dict) and isinstance(usage.get(k), int)}
+        return TargetReply(strip_thinking(content), usage=tokens)

guarddiff/targets/python_fn.py

@@ -0,0 +1,18 @@
+"""User-supplied target: ``fn(prompt: str) -> str`` referenced as "module:function"."""
+
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from .base import Target, TargetReply, strip_thinking
+
+
+class PythonTarget(Target):
+    def __init__(self, fn: Callable[[str], Any]) -> None:
+        self._fn = fn
+
+    def call(self, prompt: str) -> TargetReply:
+        result = self._fn(prompt)
+        if not isinstance(result, str):
+            raise TypeError(f"target function must return str, got {type(result).__name__}")
+        return TargetReply(strip_thinking(result))  # reasoning text is not part of the answer