brain-wiki 0.1.0.dev0__py3-none-any.whl

brainwiki/write/_episode.py

@@ -0,0 +1,195 @@
+"""`write_episode()`: the one function every episode writer goes through (design doc 05 §2.1)."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from brainwiki.config import brain_state_dir
+from brainwiki.errors import UsageError
+from brainwiki.format import (
+    CAPTURED,
+    INPUTS,
+    ORIGINS,
+    EpisodeContent,
+    episode_path,
+    now_iso,
+    render_episode,
+)
+from brainwiki.ids import new_ulid
+from brainwiki.index import written
+from brainwiki.lifecycle import declare_format
+from brainwiki.store import sha256_bytes
+from brainwiki.write._secrets import scrub
+
+if TYPE_CHECKING:
+    from brainwiki.engine import Engine
+
+_MAX_EPISODE_BYTES = 64 * 1024
+_TURNS = re.compile(r"^[0-9]+-[0-9]+$")
+_SESSION = re.compile(r"^[A-Za-z0-9._:-]{1,200}$")  # host session ids; also glob-safe
+_PLAIN = re.compile(r"^[^\r\n\x00]{1,200}$")
+
+
+@dataclass
+class EpisodeResult:
+    id: str
+    path: str
+    deduped: bool = False
+    redacted: list[str] = field(default_factory=list)
+    truncated: bool = False
+
+    def to_json(self) -> dict[str, object]:
+        return {
+            "id": self.id,
+            "path": self.path,
+            "deduped": self.deduped,
+            "redacted": self.redacted,
+            "truncated": self.truncated,
+        }
+
+
+def _check(name: str, value: str | None, pattern: re.Pattern[str] = _PLAIN) -> None:
+    if value is not None and not pattern.match(value):
+        what = (
+            "letters, digits, and . _ : - (at most 200)"
+            if pattern is _SESSION
+            else "a single line of at most 200 characters"
+        )
+        raise UsageError(f"{name} must be {what}")
+
+
+def _cap(text: str, limit: int) -> tuple[str, bool]:
+    data = text.encode("utf-8")
+    if len(data) <= limit:
+        return text, False
+    cut = data[:limit].decode("utf-8", errors="ignore")
+    cut = cut[: cut.rfind("\n")] if "\n" in cut[-2000:] else cut
+    return (
+        cut.rstrip() + f"\n\n[truncated: {len(data) - len(cut.encode('utf-8'))} bytes cut]\n",
+        True,
+    )
+
+
+def _dedupe_marker(engine: Engine, key: str) -> str:
+    digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:32]
+    directory = brain_state_dir(engine.brain.key) / "dedupe"
+    directory.mkdir(parents=True, exist_ok=True)
+    return str(directory / digest)
+
+
+def _existing(engine: Engine, key: str) -> tuple[str, str] | None:
+    """An episode already written with this dedupe key: (id, path)."""
+    marker = _dedupe_marker(engine, key)
+    try:
+        with open(marker, encoding="utf-8") as fh:
+            ulid, _, rel = fh.read().partition(" ")
+        if ulid and engine.store.exists(rel.strip()):
+            return ulid, rel.strip()
+    except OSError:
+        pass
+    engine.scan()
+    row = engine.db.execute(
+        """SELECT fm_id, path FROM nodes WHERE kind IN ('episode', 'archived_episode')
+           AND json_extract(props, '$.dedupe') = ? LIMIT 1""",
+        (key,),
+    ).fetchone()
+    return (row["fm_id"], row["path"]) if row else None
+
+
+def _claim_marker(engine: Engine, key: str, ulid: str, rel: str) -> bool:
+    """Atomically claim a dedupe key on this device. False when another writer holds it."""
+    marker = _dedupe_marker(engine, key)
+    try:
+        fd = os.open(marker, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    except FileExistsError:
+        return False
+    with os.fdopen(fd, "w", encoding="utf-8") as fh:
+        fh.write(f"{ulid} {rel}")
+    return True
+
+
+def write_episode(
+    engine: Engine,
+    content: EpisodeContent,
+    *,
+    origin: str,
+    inputs: str,
+    captured: str,
+    agent: str | None = None,
+    session: str | None = None,
+    origin_turns: str | None = None,
+    device: str | None = None,
+    topics: list[str] | None = None,
+    dedupe: str | None = None,
+    source: str | None = None,
+) -> EpisodeResult:
+    """Write one immutable episode. Provenance comes from the caller's code path, never from text."""
+    if origin not in ORIGINS or inputs not in INPUTS or captured not in CAPTURED:
+        raise UsageError("invalid provenance values")
+    for name, value in (
+        ("agent", agent),
+        ("device", device),
+        ("dedupe", dedupe),
+    ):
+        _check(name, value)
+    _check("session", session, _SESSION)
+    _check("origin_turns", origin_turns, _TURNS)
+    if not (content.summary.strip() or content.claims):
+        raise UsageError("an episode needs a summary or at least one claim")
+    manifest = engine.manifest
+    if dedupe:
+        found = _existing(engine, dedupe)
+        if found:
+            return EpisodeResult(found[0], found[1], deduped=True)
+
+    hints = [t.strip() for t in topics or [] if t and t.strip()][:5]
+
+    redacted: list[str] = []
+
+    def clean(text: str) -> str:
+        cleaned, kinds = scrub(text)
+        redacted.extend(kinds)
+        return cleaned
+
+    scrubbed = EpisodeContent(
+        summary=clean(content.summary),
+        claims=[clean(c) for c in content.claims],
+        negative_findings=[clean(c) for c in content.negative_findings],
+        open_questions=[clean(c) for c in content.open_questions],
+    )
+    ulid = new_ulid()
+    rel = episode_path(manifest.layout.episodes_dir, ulid)
+    fields = {
+        "id": ulid,
+        "type": "episode",
+        "agent": agent,
+        "session": session,
+        "origin_turns": origin_turns,
+        "device": device or engine.user.device_name,
+        "origin": origin,
+        "inputs": inputs,
+        "captured": captured,
+        "created": now_iso(),
+        "topics": hints,
+        "dedupe": dedupe,
+        "source": source,
+    }
+    rendered = render_episode(fields, scrubbed)
+    head, sep, body = rendered.partition("\n---\n")
+    body, truncated = _cap(body, _MAX_EPISODE_BYTES - len(head) - len(sep))
+    text = head + sep + body
+    if dedupe and not _claim_marker(engine, dedupe, ulid, rel):
+        found = _existing(engine, dedupe)
+        if found:
+            return EpisodeResult(found[0], found[1], deduped=True)
+    if engine.store.exists(rel):
+        raise UsageError(f"{rel} already exists")
+    declare_format(engine)
+    engine.store.write_text(rel, text)
+    written.mark(engine.brain.key, ulid, sha256_bytes(text.encode("utf-8")))
+    engine.scan(paths=[rel])
+    return EpisodeResult(ulid, rel, redacted=sorted(set(redacted)), truncated=truncated)

brainwiki/write/_move.py

@@ -0,0 +1,181 @@
+"""`brain mv`: move a note, or a topic folder with its notes, and rewrite the links that break.
+
+Links by file name keep working when only folders change; links that name a path, and relative
+Markdown links into or out of what moved, are rewritten. Topics are known by their slug, so
+moving a topic folder re-parents it without breaking topic policies (design doc 02 §2).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import posixpath
+import re
+from typing import TYPE_CHECKING
+from urllib.parse import quote
+
+from brainwiki.errors import Conflict, NotFound, UsageError
+from brainwiki.index import norm_key
+from brainwiki.runs import Check, Run, RunRecord
+
+if TYPE_CHECKING:
+    from brainwiki.engine import Engine
+
+
+def _stem(path: str) -> str:
+    name = path.rsplit("/", 1)[-1]
+    return name[:-3] if name.lower().endswith(".md") else name
+
+
+def _bare(path: str) -> str:
+    return path[:-3] if path.lower().endswith(".md") else path
+
+
+def plan_move(engine: Engine, src: str, dst: str) -> Run:
+    engine.scan(force=True)
+    src, dst = src.rstrip("/"), dst.rstrip("/")
+    layout = engine.manifest.layout
+    managed = (layout.inbox_dir, layout.archive_dir, layout.reports_dir)
+    if any(p == m or p.startswith(m + "/") for p in (src, dst) for m in managed):
+        raise UsageError(
+            "mv moves notes, hubs, and topic folders; inbox, archive, and reports are managed by brain-wiki"
+        )
+    if engine.store.abspath(src).is_dir():
+        return _plan_folder(engine, src, dst)
+    if not dst.lower().endswith(".md"):
+        raise UsageError("the destination must be a .md path (or a folder, to move a topic)")
+    node = engine.db.execute("SELECT id, kind FROM nodes WHERE path=?", (src,)).fetchone()
+    if node is None or node["kind"] not in ("note", "hub"):
+        raise NotFound(f"{src} is not a note in this brain")
+    if engine.store.exists(dst):
+        raise Conflict(f"{dst} already exists")
+    engine.store.abspath(dst)  # refuses paths outside the brain
+    run = Run(engine, "mv", summary=f"mv {src} → {dst}")
+    run.move(src, dst)
+    _rewrite_links(engine, run, {src: dst})
+    return run
+
+
+def _plan_folder(engine: Engine, src: str, dst: str) -> Run:
+    """Move every file under `src` to the same place under `dst`: a topic and what it holds."""
+    if dst == src or dst.startswith(src + "/"):
+        raise UsageError(f"cannot move {src} into itself")
+    if engine.store.abspath(dst).exists():  # abspath also refuses paths outside the brain
+        raise Conflict(f"{dst} already exists")
+    root = engine.store.abspath(src)
+    files = sorted(
+        posixpath.join(src, os.path.relpath(os.path.join(folder, name), root).replace(os.sep, "/"))
+        for folder, _, names in os.walk(root)
+        for name in names
+    )
+    if not files:
+        raise NotFound(f"{src} holds no files")
+    run = Run(engine, "mv", summary=f"mv {src}/ → {dst}/")
+    mapping = {path: dst + path[len(src) :] for path in files}
+    for old, new in mapping.items():
+        run.move(old, new)
+    _rewrite_links(engine, run, mapping)
+    return run
+
+
+def _rewrite_links(engine: Engine, run: Run, mapping: dict[str, str]) -> None:
+    """Rewrite links that named a moved note by path (or by a file name that changed), and
+    relative Markdown links whose two ends no longer sit where they did."""
+    conn = engine.db
+    moved: dict[str, tuple[str, str]] = {}  # node id -> (old path, new path)
+    for chunk in _chunks(list(mapping)):
+        for row in conn.execute(
+            f"SELECT id, path FROM nodes WHERE kind IN ('note', 'hub') AND path IN ({','.join('?' * len(chunk))})",  # noqa: S608 - placeholders only
+            chunk,
+        ):
+            moved[row["id"]] = (row["path"], mapping[row["path"]])
+    if not moved:
+        return
+    link_text: dict[str, tuple[str, bool]] = {}  # node id -> (new link, links by name still work)
+    for node_id, (old, new) in moved.items():
+        others = conn.execute(
+            "SELECT COUNT(*) FROM names WHERE kind='stem' AND key=? AND node_id != ?",
+            (norm_key(_stem(new)), node_id),
+        ).fetchone()[0]
+        link_text[node_id] = (
+            _stem(new) if others == 0 else _bare(new),
+            _stem(old) == _stem(new) and others == 0,
+        )
+
+    # (line, raw target, written form, new wikilink text or None, new target path for md links)
+    edits: dict[str, list[tuple[int, str, str, str | None, str | None]]] = {}
+    for chunk in _chunks(list(moved)):
+        marks = ",".join("?" * len(chunk))
+        for edge in conn.execute(  # links into what moved
+            f"""SELECT e.line, e.raw_target, e.written, e.other, n.path FROM edges e
+                JOIN nodes n ON n.id = e.declared_in
+                WHERE e.other IN ({marks}) AND e.declared_in != e.other AND e.line > 0""",  # noqa: S608
+            chunk,
+        ):
+            old, new = moved[edge["other"]]
+            key = norm_key(edge["raw_target"])
+            by_path = key == norm_key(old) or ("/" in key and norm_key(old).endswith("/" + key))
+            by_stem = key == norm_key(_stem(old))
+            link, name_ok = link_text[edge["other"]]
+            wiki = link if by_path or (by_stem and not name_ok) else None
+            edits.setdefault(edge["path"], []).append(
+                (edge["line"], edge["raw_target"], edge["written"], wiki, new)
+            )
+        for edge in conn.execute(  # relative Markdown links out of what moved
+            f"""SELECT e.line, e.raw_target, e.other, t.path AS target, n.path FROM edges e
+                JOIN nodes n ON n.id = e.declared_in JOIN nodes t ON t.id = e.other
+                WHERE e.declared_in IN ({marks}) AND e.written = 'md' AND e.line > 0""",  # noqa: S608
+            chunk,
+        ):
+            if edge["other"] not in moved:
+                edits.setdefault(edge["path"], []).append(
+                    (edge["line"], edge["raw_target"], "md", None, edge["target"])
+                )
+
+    for path, items in edits.items():
+        where = mapping.get(path, path)  # a file that moved is edited at its new place
+        text = run.read(where)
+        if text is None:
+            continue
+        newline = "\r\n" if "\r\n" in text else "\n"
+        lines = text.split(newline)
+        for line_no, raw, written, wiki, target in items:
+            if line_no - 1 >= len(lines):
+                continue
+            line = lines[line_no - 1]
+            if wiki is not None:
+                line = re.sub(
+                    r"\[\[\s*" + re.escape(raw) + r"(?=\s*(?:\||#|\]\]))", "[[" + wiki, line
+                )
+            if target is not None and (written == "md" or "](" in line):
+                href_new = posixpath.relpath(target, posixpath.dirname(where) or ".")
+                for href in {raw, quote(raw)}:
+                    line = re.sub(
+                        r"\]\(" + re.escape(href) + r"(#[^)]*)?\)",
+                        lambda m, h=href_new: "](" + quote(h) + (m.group(1) or "") + ")",
+                        line,
+                    )
+            lines[line_no - 1] = line
+        joined = newline.join(lines)
+        if joined != text:
+            run.write(where, joined)
+
+
+def _chunks(items: list[str], size: int = 500) -> list[list[str]]:
+    return [items[i : i + size] for i in range(0, len(items), size)]
+
+
+def move_note(engine: Engine, src: str, dst: str, *, check: Check | None = None) -> RunRecord:
+    run = plan_move(engine, src, dst)
+    rewritten = [e.path for e in run.entries if e.moved_to is None]
+    moved = sum(1 for e in run.entries if e.moved_to is not None)
+    head = f"- {src} → {dst}" + (f" ({moved} files)" if moved > 1 else "")
+    record = run.commit(
+        check=check, log_lines=[head] + [f"- rewrote links in {p}" for p in rewritten]
+    )
+    root = engine.store.abspath(src.rstrip("/"))
+    if root.is_dir():  # a folder move leaves its emptied directories behind
+        for folder, _, _ in os.walk(root, topdown=False):
+            with contextlib.suppress(OSError):  # not empty: something besides notes lives there
+                os.rmdir(folder)
+    return record

brainwiki/write/_rules.py

@@ -0,0 +1,204 @@
+"""Rules: the strictest write path (design doc 05 §6). Consolidation never calls this module."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any
+
+from brainwiki.errors import BudgetExceeded, NotFound, UsageError
+from brainwiki.format import (
+    RULES_HEADER,
+    format_rule,
+    format_value,
+    now_iso,
+    parse_rules,
+    split,
+    split_meta,
+)
+from brainwiki.ids import is_prefixed, rule_id
+from brainwiki.runs import Run, RunRecord
+
+if TYPE_CHECKING:
+    from brainwiki.engine import Engine
+
+_RULES = "rules.md"
+_MAX_RULE_CHARS = 300
+_MAX_QUOTE_CHARS = 500
+
+
+def list_rules(engine: Engine) -> list[dict[str, Any]]:
+    try:
+        text = engine.store.read_text(_RULES)
+    except OSError:
+        return []
+    rules, _ = parse_rules(text)
+    return [
+        {
+            "id": r.id,
+            "text": r.text,
+            "quote": r.meta.get("quote"),
+            "at": r.meta.get("at"),
+            "origin": r.meta.get("origin"),
+            "via": r.meta.get("via"),
+            "line": r.line,
+        }
+        for r in rules
+    ]
+
+
+def _validate(text: str, quote: str) -> tuple[str, str]:
+    text, quote = " ".join(text.split()), " ".join(quote.split())
+    if not text or len(text) > _MAX_RULE_CHARS:
+        raise UsageError(f"a rule must be 1 to {_MAX_RULE_CHARS} characters")
+    if not quote or len(quote) > _MAX_QUOTE_CHARS:
+        raise UsageError(f"the quote must be 1 to {_MAX_QUOTE_CHARS} characters")
+    return text, quote
+
+
+def _stage_rule(run: Run, text: str, quote: str, replaces: str | None, via: str | None) -> str:
+    engine = run.engine
+    current = run.read(_RULES) or RULES_HEADER
+    new_id = rule_id()
+    meta = {
+        "id": new_id,
+        "quote": quote,
+        "at": datetime.now(UTC).strftime("%Y-%m-%d"),
+        "origin": "owner",
+    }
+    if via:
+        meta["via"] = via
+    lines = current.split("\n")
+    if replaces:
+        index = next(
+            (
+                i
+                for i, line in enumerate(lines)
+                if f"id:{replaces}" in line and line.lstrip().startswith("-")
+            ),
+            None,
+        )
+        if index is None:
+            raise NotFound(f"no rule {replaces} in rules.md")
+        old_line = lines.pop(index)
+        body, old_meta = split_meta(old_line.lstrip()[1:].strip())
+        old_meta = dict(old_meta or {})
+        old_meta.update(status="superseded", by=new_id)
+        archive = f"{engine.manifest.layout.archive_dir}/rules.md"
+        archived = run.read(archive) or "# Archived rules\n"
+        run.write(archive, archived.rstrip("\n") + "\n" + format_rule(body, old_meta) + "\n")
+    while lines and lines[-1] == "":
+        lines.pop()
+    lines.append(format_rule(text, meta))
+    updated = "\n".join(lines) + "\n"
+    budgets = engine.manifest.budgets
+    if (
+        len(updated.splitlines()) > budgets.rules_max_lines
+        or len(updated) > budgets.rules_max_chars
+    ):
+        longest = sorted(parse_rules(updated)[0], key=lambda r: len(r.text), reverse=True)[:3]
+        raise BudgetExceeded(
+            f"rules.md would exceed its budget ({budgets.rules_max_lines} lines, "
+            f"{budgets.rules_max_chars} characters); longest rules: "
+            + "; ".join(f"{r.id}: {r.text[:60]}" for r in longest)
+        )
+    rules, malformed = parse_rules(updated)
+    if malformed or not any(r.id == new_id for r in rules):
+        raise UsageError("the rule could not be written in a parseable form")
+    run.write(_RULES, updated)
+    return new_id
+
+
+def add_rule(
+    engine: Engine,
+    text: str,
+    *,
+    quote: str | None = None,
+    replaces: str | None = None,
+    via: str | None = None,
+    agent: str | None = None,
+) -> tuple[str, RunRecord]:
+    """Add (or replace) a rule as the user. Callers are responsible for user confirmation."""
+    text, quote = _validate(text, quote or text)
+    if replaces and not is_prefixed(replaces, "r-"):
+        raise UsageError("replaces must be a rule id (r-…)")
+    run = Run(engine, "rule", agent=agent, summary=f"rule: {text[:60]}")
+    new_id = _stage_rule(run, text, quote, replaces, via)
+    log = [f"- added {new_id}: {text}"] + ([f"- replaced {replaces}"] if replaces else [])
+    return new_id, run.commit(log_lines=log)
+
+
+# -- proposals: rules awaiting the user's confirmation --------------------
+def _proposal_path(engine: Engine, proposal_id: str) -> str:
+    return f"{engine.manifest.layout.proposals_dir}/rule-{proposal_id}.md"
+
+
+def propose_rule(
+    engine: Engine, text: str, quote: str, *, replaces: str | None = None, agent: str | None = None
+) -> tuple[str, str]:
+    """Write a rule proposal for `brain rule accept`. Returns (proposal id, path)."""
+    text, quote = _validate(text, quote)
+    proposal_id = rule_id()
+    rel = _proposal_path(engine, proposal_id)
+    fields = {
+        "id": proposal_id,
+        "type": "rule-proposal",
+        "agent": agent,
+        "created": now_iso(),
+        "replaces": replaces,
+        "quote": quote,
+    }
+    head = ["---"] + [f"{k}: {format_value(v)}" for k, v in fields.items() if v] + ["---"]
+    body = [
+        f"# Proposed rule: {text}",
+        "",
+        f"> {quote}",
+        "",
+        f"Accept with `brain rule accept {proposal_id}`, or reject with `brain rule reject {proposal_id}`.",
+        "",
+    ]
+    engine.store.write_text(rel, "\n".join(head + body))
+    return proposal_id, rel
+
+
+def pending_rule_proposals(engine: Engine) -> list[dict[str, Any]]:
+    directory = engine.store.abspath(engine.manifest.layout.proposals_dir)
+    out = []
+    for path in sorted(directory.glob("rule-*.md")) if directory.is_dir() else []:
+        fm = split(path.read_text(encoding="utf-8"))
+        title = next((ln for ln in fm.body.splitlines() if ln.startswith("# Proposed rule: ")), "")
+        out.append(
+            {
+                "id": fm.data.get("id") or path.stem[len("rule-") :],
+                "text": title[len("# Proposed rule: ") :],
+                "quote": fm.data.get("quote"),
+                "agent": fm.data.get("agent"),
+                "replaces": fm.data.get("replaces"),
+            }
+        )
+    return out
+
+
+def _proposal(engine: Engine, proposal_id: str) -> dict[str, Any]:
+    for proposal in pending_rule_proposals(engine):
+        if proposal["id"] == proposal_id:
+            return proposal
+    raise NotFound(f"no rule proposal {proposal_id}")
+
+
+def accept_rule(engine: Engine, proposal_id: str) -> tuple[str, RunRecord]:
+    proposal = _proposal(engine, proposal_id)
+    text, quote = _validate(proposal["text"], str(proposal["quote"] or proposal["text"]))
+    run = Run(engine, "rule", summary=f"rule: {text[:60]}")
+    new_id = _stage_rule(run, text, quote, proposal.get("replaces"), "proposal")
+    run.delete(_proposal_path(engine, proposal_id))
+    log = [
+        f"- accepted proposal {proposal_id} from {proposal.get('agent') or 'an agent'} as {new_id}: {text}"
+    ]
+    return new_id, run.commit(log_lines=log)
+
+
+def reject_rule(engine: Engine, proposal_id: str) -> RunRecord:
+    _proposal(engine, proposal_id)
+    run = Run(engine, "rule-reject", summary=f"rejected rule proposal {proposal_id}")
+    run.delete(_proposal_path(engine, proposal_id))
+    return run.commit()

brainwiki/write/_secrets.py

@@ -0,0 +1,108 @@
+"""Secret detection and redaction (design doc 10 §6). Pattern-based; misses unusual formats."""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from dataclasses import dataclass
+
+# (name, pattern). Fixed-prefix formats first; the generic assignment rule runs last.
+_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
+    (
+        "private-key",
+        re.compile(
+            r"-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY-----[\s\S]*?(?:-----END (?:[A-Z0-9]+ )*PRIVATE KEY-----|\Z)"
+        ),
+    ),
+    ("aws-access-key", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b")),
+    ("google-api-key", re.compile(r"\bAIza[0-9A-Za-z_\-]{35}\b")),
+    (
+        "github-token",
+        re.compile(r"\b(?:gh[pousr]_[0-9A-Za-z]{36,}|github_pat_[0-9A-Za-z_]{40,})\b"),
+    ),
+    ("slack-token", re.compile(r"\bxox[abposr]-[0-9A-Za-z-]{10,}\b")),
+    ("slack-webhook", re.compile(r"https://hooks\.slack\.com/services/[A-Za-z0-9/]{20,}")),
+    ("stripe-key", re.compile(r"\b(?:sk|rk)_(?:live|test)_[0-9A-Za-z]{16,}\b")),
+    ("anthropic-key", re.compile(r"\bsk-ant-[0-9A-Za-z_\-]{20,}\b")),
+    ("openai-key", re.compile(r"\bsk-(?:proj-|svcacct-|admin-)?[0-9A-Za-z_\-]{20,}\b")),
+    ("jwt", re.compile(r"\beyJ[0-9A-Za-z_\-]{8,}\.eyJ[0-9A-Za-z_\-]{8,}\.[0-9A-Za-z_\-]{8,}\b")),
+]
+_ASSIGNMENT = re.compile(
+    r"(?i)\b(?P<key>[a-z0-9_.\-]*(?:password|passwd|secret|token|api[_\-]?key|access[_\-]?key)[a-z0-9_.\-]*)"
+    r"(?P<sep>\s*[:=]\s*|\s+is\s+)"
+    r"(?P<quote>[\"']?)(?P<value>[^\s\"'`,;]{12,})(?P=quote)"
+)
+_REDACTED = re.compile(r"\[REDACTED:[a-z-]+\]")
+# Values that name or compute a secret rather than hold one: code (calls, templates, optional
+# chaining), environment lookups, dotted references, ALL_CAPS variable names, and secret-manager
+# references such as `github-pat:latest`.
+_REFERENCE = re.compile(
+    r"[()${}]|\?\.|^(?:process\.env|os\.environ|env)\b|^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)+$"
+    r"|^[A-Z][A-Z0-9_]*$|:latest$"
+)
+
+
+@dataclass
+class SecretHit:
+    kind: str
+    start: int
+    end: int
+    line: int
+
+
+def _entropy(value: str) -> float:
+    counts = Counter(value)
+    return -sum(c / len(value) * math.log2(c / len(value)) for c in counts.values())
+
+
+def _high_entropy(value: str) -> bool:
+    if _REDACTED.fullmatch(value) or value.lower() in ("redacted", "changeme", "placeholder"):
+        return False
+    if value.startswith(("[[", "[", "(", "<", "{")) or "](" in value:
+        return False  # a link or a placeholder, not a credential
+    if _REFERENCE.search(value):
+        return False
+    classes = sum(
+        bool(re.search(p, value)) for p in (r"[a-z]", r"[A-Z]", r"[0-9]", r"[^A-Za-z0-9]")
+    )
+    return classes >= 2 and _entropy(value) >= 3.5
+
+
+def _keylike(body: str) -> bool:
+    """Generated keys mix upper case, lower case, and digits; words and slugs do not."""
+    return all(re.search(p, body) for p in (r"[a-z]", r"[A-Z]", r"[0-9]"))
+
+
+def find_secrets(text: str) -> list[SecretHit]:
+    spans: list[tuple[int, int, str]] = []
+
+    def free(start: int, end: int) -> bool:
+        return all(end <= s or start >= e for s, e, _ in spans)
+
+    for kind, pattern in _PATTERNS:
+        for match in pattern.finditer(text):
+            if kind == "openai-key" and not _keylike(match.group(0)[3:]):
+                continue  # `sk-` starts many words and URL slugs ("sk-learning-path-…")
+            if free(match.start(), match.end()):
+                spans.append((match.start(), match.end(), kind))
+    for match in _ASSIGNMENT.finditer(text):
+        start, end = match.span("value")
+        if free(start, end) and _high_entropy(match.group("value")):
+            spans.append((start, end, "secret-assignment"))
+    spans.sort()
+    return [SecretHit(kind, s, e, text.count("\n", 0, s) + 1) for s, e, kind in spans]
+
+
+def scrub(text: str) -> tuple[str, list[str]]:
+    """Replace secrets with `[REDACTED:<kind>]`. Returns (text, kinds found)."""
+    hits = find_secrets(text)
+    if not hits:
+        return text, []
+    out, last = [], 0
+    for hit in hits:
+        out.append(text[last : hit.start])
+        out.append(f"[REDACTED:{hit.kind}]")
+        last = hit.end
+    out.append(text[last:])
+    return "".join(out), [h.kind for h in hits]