token-hunt 0.1.0__py3-none-any.whl

token_hunt/__init__.py

@@ -0,0 +1,3 @@
+"""token-hunt: catch secrets at chat time, not commit time."""
+
+__version__ = "0.1.0"

token_hunt/allowlist.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+import re
+import sys
+from pathlib import Path
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
+from token_hunt.types import Match, ScanContext
+
+# Path patterns that commonly contain realistic fake keys (test fixtures etc.)
+# Mirrors Jordan's existing ~/.gitleaks-config.toml allowlist.
+ALLOWED_PATH_PATTERNS = [
+    re.compile(p) for p in [
+        r'(^|/)node_modules/.*',
+        r'(^|/)vendor/.*',
+        r'(^|/)third_party/.*',
+        r'(^|/)tests?/fixtures?/.*',
+        r'(^|/)tests?/.*\.example$',
+        r'(^|/)examples?/.*',
+        r'(^|/)samples?/.*',
+        r'(^|/)demo/.*',
+        r'.*\.example$',
+        r'.*\.sample$',
+        r'.*\.env\.sample$',
+        r'.*\.env\.example$',
+        r'.*_test\..*',
+        r'.*\.test\..*',
+        r'.*\.spec\..*',
+    ]
+]
+
+
+def is_path_allowed(path: str) -> bool:
+    return any(p.search(path) for p in ALLOWED_PATH_PATTERNS)
+
+
+def load_fingerprint_allowlist() -> set[str]:
+    """Load SHA-256 fingerprints from ~/.token-hunt/allowlist.toml.
+    Missing file = empty allowlist."""
+    home = Path.home()
+    allowlist_file = home / ".token-hunt" / "allowlist.toml"
+    if not allowlist_file.is_file():
+        return set()
+    try:
+        data = tomllib.loads(allowlist_file.read_text(encoding="utf-8"))
+    except tomllib.TOMLDecodeError:
+        return set()
+    return set(data.get("fingerprints", []))
+
+
+def apply_allowlist(matches: list[Match], ctx: ScanContext) -> list[Match]:
+    """Filter out allowlisted matches."""
+    if not matches:
+        return matches
+
+    # Path-based allowlist (only applies to file-backed scans)
+    if ctx.file_path and is_path_allowed(ctx.file_path):
+        return []
+
+    # Fingerprint-based allowlist
+    allowed_fps = load_fingerprint_allowlist()
+    if not allowed_fps:
+        return matches
+
+    return [m for m in matches if m.fingerprint not in allowed_fps]

token_hunt/cli.py

@@ -0,0 +1,194 @@
+"""token-hunt CLI entry point."""
+from __future__ import annotations
+import sys
+
+import click
+
+from token_hunt import __version__
+
+
+@click.group()
+@click.version_option(__version__, "--version")
+def main() -> None:
+    """token-hunt: catch secrets at chat time, not commit time."""
+
+
+@main.command()
+@click.argument("path", required=False, type=click.Path(exists=True))
+@click.option("--stdin", is_flag=True, help="Read text from stdin instead of file.")
+@click.option("--context", default="UserPromptSubmit",
+              help="Detection point (UserPromptSubmit, PreToolUse:Bash, etc.)")
+@click.option("--format", "fmt", default="text", type=click.Choice(["text", "json"]))
+@click.option("--no-log", is_flag=True, help="Don't write to rotations.log.")
+def scan(path, stdin, context, fmt, no_log):
+    """Scan a file or stdin for secret leaks."""
+    import json as _json
+    from pathlib import Path as _Path
+
+    from token_hunt.engine import scan as scan_engine
+    from token_hunt.store import LeakRecord, Store
+    from token_hunt.types import ScanContext
+    from dataclasses import asdict
+
+    if stdin:
+        text = sys.stdin.read()
+        file_path = None
+    elif path:
+        text = _Path(path).read_text(encoding="utf-8", errors="replace")
+        file_path = str(path)
+    else:
+        click.echo("error: provide a PATH or use --stdin", err=True)
+        sys.exit(2)
+
+    ctx = ScanContext(detection_point=context, file_path=file_path)
+    result = scan_engine(text, ctx)
+
+    if not no_log and result.matches:
+        store = Store()
+        for m in result.matches:
+            store.append(LeakRecord(
+                id=None,
+                ts=__import__("datetime").datetime.now(__import__("datetime").timezone.utc)
+                    .isoformat(timespec="seconds"),
+                kind=m.kind,
+                source=m.detection_point,
+                fingerprint=m.fingerprint,
+                rotate_url=m.rotate_url,
+                status="open",
+            ))
+
+    if fmt == "json":
+        click.echo(_json.dumps({
+            "count": result.count,
+            "matches": [asdict(m) for m in result.matches],
+        }, indent=2))
+    else:
+        if not result.matches:
+            click.echo("no leaks detected")
+        else:
+            click.echo(f"⚠️  {result.count} potential leak(s) detected:")
+            for m in result.matches:
+                fp_preview = m.fingerprint[:8] if m.fingerprint else "n/a"
+                click.echo(f"  - {m.kind} (fp={fp_preview}) @ line {m.line}")
+                if m.rotate_url:
+                    click.echo(f"    rotate: {m.rotate_url}")
+
+
+@main.command(name="list")
+@click.option("--all", "show_all", is_flag=True, help="Show closed leaks too.")
+def list_cmd(show_all):
+    """List open leaks in rotations.log."""
+    from token_hunt.store import Store
+
+    store = Store()
+    if show_all:
+        records = []
+        for line in store.path.read_text(encoding="utf-8").splitlines():
+            if line.strip():
+                import json as _j
+                from token_hunt.store import LeakRecord
+                records.append(LeakRecord(**_j.loads(line)))
+    else:
+        records = store.list_open()
+
+    if not records:
+        click.echo("no open leaks" if not show_all else "no leaks recorded")
+        return
+
+    click.echo(f"{'ID':<5} {'KIND':<22} {'SOURCE':<22} {'WHEN':<20} {'STATUS':<10} URL")
+    for r in records:
+        ts_short = r.ts[:19].replace("T", " ")
+        status = r.status.upper()
+        if r.is_overdue:
+            status = "OVERDUE"
+        click.echo(f"{r.id:<5} {r.kind:<22} {r.source:<22} {ts_short:<20} {status:<10} {r.rotate_url}")
+
+
+@main.command()
+@click.argument("leak_id", type=int)
+@click.option("--note", default="", help="Optional rotation note.")
+def close(leak_id, note):
+    """Mark a leak as closed after rotation."""
+    from token_hunt.store import Store
+    store = Store()
+    if store.close(leak_id, note=note):
+        click.echo(f"leak {leak_id} marked closed")
+    else:
+        click.echo(f"error: leak {leak_id} not found", err=True)
+        raise click.exceptions.Exit(1)
+
+
+@main.command()
+@click.argument("path", type=click.Path(exists=True, file_okay=False))
+@click.option("--context", default="PostToolUse:Read")
+@click.option("--exit-after", type=int, default=0,
+              help="Exit after N seconds (0 = run forever). Test helper.")
+def watch(path, context, exit_after):
+    """Watch a directory for new transcript files and scan them."""
+    import time as _time
+    from pathlib import Path as _Path
+
+    from watchdog.events import FileSystemEventHandler
+    from watchdog.observers import Observer
+
+    from token_hunt.engine import scan as scan_engine
+    from token_hunt.store import LeakRecord, Store
+    from token_hunt.types import ScanContext
+    import datetime as _dt
+
+    watched = _Path(path)
+    click.echo(f"watching {watched} for new files...")
+
+    class Handler(FileSystemEventHandler):
+        def on_created(self, event):
+            if event.is_directory:
+                return
+            self._scan_file(event.src_path)
+
+        def on_modified(self, event):
+            if event.is_directory:
+                return
+            self._scan_file(event.src_path)
+
+        def _scan_file(self, file_path):
+            try:
+                text = _Path(file_path).read_text(encoding="utf-8", errors="replace")
+            except OSError:
+                return
+            ctx = ScanContext(detection_point=context, file_path=file_path)
+            result = scan_engine(text, ctx)
+            if result.matches:
+                store = Store()
+                for m in result.matches:
+                    store.append(LeakRecord(
+                        id=None,
+                        ts=_dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds"),
+                        kind=m.kind,
+                        source=m.detection_point,
+                        fingerprint=m.fingerprint,
+                        rotate_url=m.rotate_url,
+                        status="open",
+                    ))
+                click.echo(f"⚠️  {file_path}: {result.count} leak(s) detected")
+            else:
+                click.echo(f"✓ {file_path}: clean")
+
+    observer = Observer()
+    observer.schedule(Handler(), str(watched), recursive=True)
+    observer.start()
+
+    try:
+        if exit_after > 0:
+            _time.sleep(exit_after)
+        else:
+            while True:
+                _time.sleep(1)
+    except KeyboardInterrupt:
+        pass
+    finally:
+        observer.stop()
+        observer.join()
+
+
+if __name__ == "__main__":
+    main()

token_hunt/engine.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from pathlib import Path
+
+from token_hunt import regex_scanner
+from token_hunt.allowlist import apply_allowlist
+from token_hunt.entropy import scan_entropy
+from token_hunt.escape import scan_escape
+from token_hunt.patterns_loader import load_builtin_rules, load_user_rules
+from token_hunt.types import Match, ScanContext
+
+
+@dataclass(frozen=True)
+class ScanResult:
+    matches: list[Match]
+    count: int
+    highest_confidence: float
+
+
+def _load_all_rules(ctx: ScanContext):
+    """Load builtin + machine-local + repo-local rules. Later extends earlier."""
+    rules = list(load_builtin_rules())
+    home = Path.home()
+    machine_local = home / ".token-hunt" / "patterns.local.toml"
+    repo_local = Path(ctx.cwd) / ".token-hunt" / "patterns.local.toml"
+    rules.extend(load_user_rules([machine_local, repo_local]))
+    return rules
+
+
+def scan(text: str, ctx: ScanContext) -> ScanResult:
+    """Run all detection passes against text. Fail-closed: any internal
+    exception propagates to caller (hook layer handles user-facing bypass)."""
+    if not text:
+        return ScanResult(matches=[], count=0, highest_confidence=0.0)
+
+    rules = _load_all_rules(ctx)
+    matches: list[Match] = []
+
+    # Pass 1: regex rules (call via module so tests can monkeypatch)
+    matches.extend(regex_scanner.scan_regex(text, rules, ctx))
+
+    # Pass 2: bash escape hatch (only fires in PreToolUse:Bash)
+    matches.extend(scan_escape(text, ctx))
+
+    # Pass 3: entropy candidates (skipped for Bash)
+    matches.extend(scan_entropy(text, ctx))
+
+    # Apply allowlist filters
+    matches = apply_allowlist(matches, ctx)
+
+    highest = max((m.confidence for m in matches), default=0.0)
+    return ScanResult(matches=matches, count=len(matches), highest_confidence=highest)

token_hunt/entropy.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+import math
+import re
+from collections import Counter
+
+from token_hunt.types import Match, ScanContext
+
+MIN_LENGTH = 20
+MIN_ENTROPY = 3.5  # hex keys are ~3.9; threshold below that keeps english prose out
+
+# Word-boundary-ish token extraction. We look for runs of "secret-shaped" chars.
+TOKEN_RE = re.compile(r'[A-Za-z0-9_+/=-]{20,}')
+
+
+def shannon_entropy(s: str) -> float:
+    """Shannon entropy in bits per character."""
+    if not s:
+        return 0.0
+    counts = Counter(s)
+    length = len(s)
+    return -sum((c / length) * math.log2(c / length) for c in counts.values())
+
+
+def scan_entropy(text: str, ctx: ScanContext) -> list[Match]:
+    """Flag long high-entropy strings that didn't match a regex rule.
+    Skipped for PreToolUse:Bash (commands have legit high-entropy args)."""
+    if ctx.detection_point == "PreToolUse:Bash":
+        return []
+
+    matches: list[Match] = []
+    for line_no, line in enumerate(text.splitlines(), start=1):
+        for m in TOKEN_RE.finditer(line):
+            candidate = m.group(0)
+            if len(candidate) < MIN_LENGTH:
+                continue
+            ent = shannon_entropy(candidate)
+            if ent < MIN_ENTROPY:
+                continue
+            # Confidence scales with entropy above threshold
+            confidence = min(0.4, 0.1 + (ent - MIN_ENTROPY) * 0.05)
+            matches.append(Match(
+                kind="entropy-candidate",
+                value=candidate,
+                fingerprint="",  # don't fingerprint unknowns
+                rotate_url="",
+                detection_point=ctx.detection_point,
+                confidence=confidence,
+                rule_id="entropy",
+                line=line_no,
+                column=m.start() + 1,
+            ))
+    return matches

token_hunt/escape.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+import re
+
+from token_hunt.types import Match, ScanContext
+
+# Patterns where bash evaluates a secret-bearing env var into emitted text.
+# These leak the value even when the literal secret isn't in the input.
+ESCAPE_PATTERNS = [
+    # ${VAR:+...} and ${VAR:-...} — Jordan's bug class. Both branches can run.
+    (re.compile(r'\$\{([A-Z][A-Z0-9_]{2,}):[+-][^\}]*\}'), "brace-eval"),
+    # echo $TOKEN — direct echo of var
+    (re.compile(r'\becho\s+["\']?\$([A-Z][A-Z0-9_]{2,})'), "echo-var"),
+    # printenv TOKEN
+    (re.compile(r'\bprintenv\s+([A-Z][A-Z0-9_]{2,})'), "printenv"),
+    # env | grep X (likely hunting for a secret)
+    (re.compile(r'\benv\b\s*\|\s*grep'), "env-pipe-grep"),
+    # grep $VAR (variable expands before grep runs)
+    (re.compile(r'\bgrep\s+.*\$([A-Z][A-Z0-9_]{2,})'), "grep-var"),
+    # $(VAR) command substitution
+    (re.compile(r'\$\(\s*([A-Z][A-Z0-9_]{2,})\s*\)'), "cmd-substitution"),
+    # `VAR` backtick substitution
+    (re.compile(r'`([A-Z][A-Z0-9_]{2,})`'), "backtick-substitution"),
+    # printf with %s and secret arg
+    (re.compile(r'\bprintf\s+.*%s.*\$([A-Z][A-Z0-9_]{2,})'), "printf-secret"),
+    # awk ENVIRON access
+    (re.compile(r'\bawk\b.*ENVIRON\['), "awk-environ"),
+]
+
+# Legit status-check patterns. If the SAME LINE has one of these, the escape
+# pattern is treated as a safe check (not a leak).
+SAFE_STATUS_PATTERNS = [
+    re.compile(r'\[\s*-n\s'),                # [ -n "$VAR" ]
+    re.compile(r'\[\s*-z\s'),                # [ -z "$VAR" ]
+    re.compile(r'\bwc\s+-c\b'),              # wc -c <<< "$VAR"
+    re.compile(r'\bwc\s+-m\b'),              # wc -m
+    re.compile(r'\[\s*-n\s.*\]\s*&&\s*echo\s+(SET|UNSET|set|unset)'),
+]
+
+# Env var name patterns that mark a var as secret-bearing.
+SECRET_NAME_SUFFIXES = (
+    "_TOKEN", "_KEY", "_SECRET", "_API_KEY", "_PASSWORD", "_PASS",
+    "_DBROOT", "_PAT", "_CREDENTIAL", "_CREDS", "_AUTH", "_ACCESS_KEY",
+)
+SECRET_NAME_PREFIXES = (
+    "ANTHROPIC_", "OPENAI_", "GITHUB_", "ZCODE_", "Z_AI_", "FIGMA_",
+    "LINEAR_", "STRIPE_", "SLACK_", "NPM_", "PYPI_", "HEROKU_",
+    "AWS_", "GCP_", "AZURE_", "DBROOT", "TOKEN", "SECRET",
+)
+
+
+def _is_secret_var_name(name: str) -> bool:
+    upper = name.upper()
+    if any(upper.endswith(s) for s in SECRET_NAME_SUFFIXES):
+        return True
+    if any(upper.startswith(p) for p in SECRET_NAME_PREFIXES):
+        return True
+    return False
+
+
+def _is_legit_status_check(line: str) -> bool:
+    return any(p.search(line) for p in SAFE_STATUS_PATTERNS)
+
+
+def scan_escape(text: str, ctx: ScanContext) -> list[Match]:
+    """Detect bash patterns that evaluate secrets into emitted text.
+    Only fires in PreToolUse:Bash context — escape patterns are bash-specific."""
+    if ctx.detection_point != "PreToolUse:Bash":
+        return []
+
+    matches: list[Match] = []
+    for line_no, line in enumerate(text.splitlines(), start=1):
+        if _is_legit_status_check(line):
+            continue
+        for pattern, pattern_name in ESCAPE_PATTERNS:
+            for m in pattern.finditer(line):
+                var_name = m.group(1) if m.groups() else ""
+                if var_name and not _is_secret_var_name(var_name):
+                    continue
+                kind = f"escape:{var_name}" if var_name else f"escape:{pattern_name}"
+                matches.append(Match(
+                    kind=kind,
+                    value=line.strip(),  # the whole line, not the literal secret
+                    fingerprint="",  # no literal secret to fingerprint
+                    rotate_url="",
+                    detection_point=ctx.detection_point,
+                    confidence=0.85,
+                    rule_id=f"escape:{pattern_name}",
+                    line=line_no,
+                    column=m.start() + 1,
+                ))
+    return matches