traceredact 0.1.0__py3-none-any.whl

traceredact/__init__.py

@@ -0,0 +1,43 @@
+"""traceredact — redact PII & secrets from AI prompts, traces and tool-call args.
+
+Quick start::
+
+    from traceredact import redact
+
+    result = redact({"args": {"email": "a@b.com", "key": "sk-abc123..."}})
+    result.value        # -> {"args": {"email": "[REDACTED:pii]", "key": "[REDACTED:secret]"}}
+    result.findings     # -> [Finding(json_path="args.email", ...), ...]
+    result.has_findings # -> True
+
+Deterministic, no data retained. Configure via a :class:`Policy` (in code or a
+``traceredact.yml`` file).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from traceredact.detectors.base import Finding
+from traceredact.engine import Engine, RedactionResult
+from traceredact.policy import CustomPattern, Policy
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "redact",
+    "Engine",
+    "RedactionResult",
+    "Finding",
+    "Policy",
+    "CustomPattern",
+    "__version__",
+]
+
+
+def redact(value: Any, policy: Policy | None = None) -> RedactionResult:
+    """Redact ``value`` (str, dict, list, or nested mix) and return the result.
+
+    ``policy`` defaults to a sensible built-in policy. The input is never
+    mutated; ``result.value`` is a redacted copy.
+    """
+    return Engine(policy).redact(value)

traceredact/cli.py

@@ -0,0 +1,172 @@
+"""``traceredact`` command-line interface.
+
+Two commands, both CI-gateable (non-zero exit when findings exist):
+
+* ``traceredact scan <path>``  — report findings, do not write anything.
+* ``traceredact redact <file>`` — print/emit the redacted content.
+
+``--format pretty|json`` controls output. JSON is stable for piping into CI.
+"""
+
+from __future__ import annotations
+
+import json
+from enum import StrEnum
+from pathlib import Path
+from typing import Any
+
+import typer
+
+from traceredact import __version__
+from traceredact.engine import Engine, RedactionResult
+from traceredact.policy import Policy
+
+app = typer.Typer(
+    add_completion=False,
+    help="Redact PII & secrets from AI prompts, traces and tool-call arguments.",
+    no_args_is_help=True,
+)
+
+
+class OutputFormat(StrEnum):
+    pretty = "pretty"
+    json = "json"
+
+
+def _load_policy(policy_path: Path | None) -> Policy:
+    if policy_path is not None:
+        return Policy.load(policy_path)
+    # Auto-discover a traceredact.yml next to the cwd, else default.
+    default = Path("traceredact.yml")
+    return Policy.load(default) if default.exists() else Policy.default()
+
+
+def _parse_content(text: str) -> Any:
+    """Parse JSON if it looks like JSON, else treat as raw text."""
+    stripped = text.strip()
+    if stripped[:1] in "{[":
+        try:
+            return json.loads(stripped)
+        except json.JSONDecodeError:
+            return text
+    return text
+
+
+def _iter_files(path: Path) -> list[Path]:
+    if path.is_dir():
+        return sorted(p for p in path.rglob("*") if p.is_file())
+    return [path]
+
+
+def _render_pretty(per_file: list[tuple[Path, RedactionResult]]) -> int:
+    total = 0
+    for fpath, result in per_file:
+        if not result.findings:
+            continue
+        typer.secho(f"\n{fpath}", fg=typer.colors.CYAN, bold=True)
+        typer.echo(f"  {'DETECTOR':<26} {'CONF':>5}  {'PATH':<24} PREVIEW")
+        for f in result.findings:
+            total += 1
+            path = f.json_path or "(root)"
+            typer.echo(f"  {f.detector_id:<26} {f.confidence:>5.2f}  {path:<24} {f.preview}")
+    if total:
+        typer.secho(
+            f"\n{total} finding(s) across {len(per_file)} file(s).",
+            fg=typer.colors.RED,
+            bold=True,
+        )
+    else:
+        typer.secho("No findings.", fg=typer.colors.GREEN)
+    return total
+
+
+def _render_json(per_file: list[tuple[Path, RedactionResult]]) -> int:
+    payload = []
+    total = 0
+    for fpath, result in per_file:
+        findings = [
+            {
+                "detector_id": f.detector_id,
+                "category": f.category,
+                "confidence": f.confidence,
+                "json_path": f.json_path,
+                "span": list(f.span),
+                "preview": f.preview,
+                "replacement": f.replacement,
+            }
+            for f in result.findings
+        ]
+        total += len(findings)
+        payload.append({"file": str(fpath), "findings": findings})
+    typer.echo(json.dumps({"total": total, "files": payload}, indent=2))
+    return total
+
+
+@app.command()
+def scan(
+    path: Path = typer.Argument(..., exists=True, help="File or directory to scan."),
+    fmt: OutputFormat = typer.Option(OutputFormat.pretty, "--format", "-f"),
+    policy_path: Path | None = typer.Option(None, "--policy", "-p", help="traceredact.yml path."),
+) -> None:
+    """Scan a file or directory and report findings. Exits non-zero if any."""
+    policy = _load_policy(policy_path)
+    engine = Engine(policy)
+    per_file: list[tuple[Path, RedactionResult]] = []
+    for fpath in _iter_files(path):
+        try:
+            content = _parse_content(fpath.read_text(errors="replace"))
+        except OSError:
+            continue
+        per_file.append((fpath, engine.redact(content)))
+
+    total = _render_json(per_file) if fmt is OutputFormat.json else _render_pretty(per_file)
+    raise typer.Exit(code=1 if total else 0)
+
+
+@app.command()
+def redact(
+    file: Path = typer.Argument(..., exists=True, help="File to redact."),
+    fmt: OutputFormat = typer.Option(OutputFormat.pretty, "--format", "-f"),
+    policy_path: Path | None = typer.Option(None, "--policy", "-p", help="traceredact.yml path."),
+    output: Path | None = typer.Option(None, "--output", "-o", help="Write redacted content here."),
+) -> None:
+    """Redact a file's content and print (or write) the result. Non-zero exit if findings."""
+    policy = _load_policy(policy_path)
+    engine = Engine(policy)
+    content = _parse_content(file.read_text(errors="replace"))
+    result = engine.redact(content)
+
+    if isinstance(result.value, str):
+        rendered = result.value
+    else:
+        rendered = json.dumps(result.value, indent=2, default=str)
+
+    if fmt is OutputFormat.json:
+        rendered = json.dumps(
+            {"value": result.value, "findings_count": len(result.findings)},
+            indent=2,
+            default=str,
+        )
+
+    if output is not None:
+        output.write_text(rendered)
+        typer.secho(f"Wrote redacted content to {output} ({len(result.findings)} finding(s)).",
+                    fg=typer.colors.GREEN)
+    else:
+        typer.echo(rendered)
+
+    raise typer.Exit(code=1 if result.has_findings else 0)
+
+
+@app.command()
+def version() -> None:
+    """Print the version."""
+    typer.echo(__version__)
+
+
+def main() -> None:  # pragma: no cover - console-script shim
+    app()
+
+
+if __name__ == "__main__":  # pragma: no cover
+    app()

traceredact/detectors/__init__.py

@@ -0,0 +1,17 @@
+"""Detector package: base protocol, secret and PII detectors."""
+
+from __future__ import annotations
+
+from traceredact.detectors.base import Detector, Finding, RuleDetector
+from traceredact.detectors.pii import pii_detectors
+from traceredact.detectors.secrets import EntropyDetector, secret_detectors, shannon_entropy
+
+__all__ = [
+    "Detector",
+    "Finding",
+    "RuleDetector",
+    "EntropyDetector",
+    "shannon_entropy",
+    "secret_detectors",
+    "pii_detectors",
+]

traceredact/detectors/base.py

@@ -0,0 +1,109 @@
+"""Base detector protocol and the core Finding model.
+
+A detector inspects a single *string* and yields zero or more :class:`Finding`
+objects describing spans that should be redacted. Detectors never mutate input
+and never retain data — the engine owns replacement and assembly.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Iterable, Iterator
+
+from pydantic import BaseModel, Field
+
+from traceredact.rules import PatternRule
+
+
+class Finding(BaseModel):
+    """A single redactable hit.
+
+    ``span`` is a ``(start, end)`` half-open index into the *string* that was
+    scanned. ``json_path`` is set by the structured walker to record where in a
+    nested document the scanned string lived (e.g. ``args.email`` or
+    ``messages[0].content``). ``matched`` holds the raw matched text so the
+    engine can build a replacement and so allowlists can be applied; callers who
+    need to avoid handling the secret should prefer :attr:`preview`.
+    """
+
+    model_config = {"frozen": True}
+
+    detector_id: str
+    category: str
+    confidence: float = Field(ge=0.0, le=1.0)
+    span: tuple[int, int]
+    matched: str
+    replacement: str
+    json_path: str | None = None
+
+    @property
+    def preview(self) -> str:
+        """A short, non-reversible preview safe to print in logs/tables."""
+        n = len(self.matched)
+        if n <= 8:
+            return "*" * n
+        return f"{self.matched[:2]}{'*' * (n - 4)}{self.matched[-2:]}"
+
+
+class Detector(ABC):
+    """Base class for all detectors.
+
+    Subclasses implement :meth:`scan` over a string. The engine handles
+    allowlisting, replacement, overlap resolution and structured traversal, so
+    detectors stay small and data-driven.
+    """
+
+    #: Stable identifier, e.g. ``"secrets.openai_key"``.
+    detector_id: str
+    #: Category bucket used in placeholders, e.g. ``"secret"`` / ``"pii"``.
+    category: str
+
+    @abstractmethod
+    def scan(self, text: str) -> Iterable[Finding]:
+        """Yield findings for ``text``. Must not mutate or retain ``text``."""
+        raise NotImplementedError
+
+    def __iter__(self) -> Iterator[Detector]:  # convenience for flat iteration
+        yield self
+
+
+class RuleDetector(Detector):
+    """Runs a list of :class:`~traceredact.rules.PatternRule` over a string.
+
+    Applies each rule's cheap literal prefilter before the (bounded) regex, then
+    its optional validator. The matched span is taken from ``rule.group`` so
+    rules like the env-assignment pattern redact only the *value*, not the key.
+    """
+
+    detector_id = "rules"
+    category = "mixed"
+
+    def __init__(self, rules: Iterable[PatternRule]) -> None:
+        self.rules = tuple(rules)
+
+    def scan(self, text: str) -> Iterable[Finding]:
+        lowered = text.lower()
+        for rule in self.rules:
+            if not rule.prefilter_hit(lowered):
+                continue
+            for m in rule.regex.finditer(text):
+                start, end = m.span(rule.group)
+                if start < 0:  # optional group didn't participate
+                    continue
+                matched = m.group(rule.group)
+                if not matched:
+                    continue
+                confidence = rule.confidence
+                if rule.validator is not None:
+                    adjusted = rule.validator(matched)
+                    if adjusted is None:
+                        continue
+                    confidence = adjusted
+                yield Finding(
+                    detector_id=rule.id,
+                    category=rule.category,
+                    confidence=confidence,
+                    span=(start, end),
+                    matched=matched,
+                    replacement="",  # filled in by the engine from policy
+                )

traceredact/detectors/pii.py

@@ -0,0 +1,15 @@
+"""PII detectors: email, phone, credit card (Luhn), IBAN (mod-97), IP.
+
+All validation lives in :mod:`traceredact.rules` so the detector here is just a
+thin wrapper over the PII rule set — keeping detection data-driven.
+"""
+
+from __future__ import annotations
+
+from traceredact.detectors.base import Detector, RuleDetector
+from traceredact.rules import PII_RULES
+
+
+def pii_detectors() -> list[Detector]:
+    """The default PII detector stack."""
+    return [RuleDetector(PII_RULES)]

traceredact/detectors/secrets.py

@@ -0,0 +1,72 @@
+"""Secret detectors: pattern rules + a Shannon-entropy detector.
+
+The entropy detector catches *unstructured* high-entropy tokens (random API
+keys, base64 blobs) that no named pattern covers. It is deliberately
+conservative — long tokens only, with a tunable threshold — because entropy is
+a fuzzy signal and we'd rather a pattern rule own a known secret class.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections.abc import Iterable
+
+from traceredact.detectors.base import Detector, Finding, RuleDetector
+from traceredact.rules import SECRET_RULES
+
+# Candidate tokens for entropy scoring: runs of base64/hex characters (optional
+# trailing '=' padding). We deliberately exclude '=', '_' and '-' as *internal*
+# connectors so an identifier like ``API_KEY=...`` splits into "API_KEY" and the
+# value, instead of being scored as one merged token. Named/base64url secrets
+# are owned by the pattern rules; entropy is the catch-all fallback.
+_TOKEN_RE = re.compile(r"[A-Za-z0-9+/]{16,512}={0,2}")
+
+
+def shannon_entropy(value: str) -> float:
+    """Bits of Shannon entropy per character."""
+    if not value:
+        return 0.0
+    counts: dict[str, int] = {}
+    for ch in value:
+        counts[ch] = counts.get(ch, 0) + 1
+    n = len(value)
+    return -sum((c / n) * math.log2(c / n) for c in counts.values())
+
+
+class EntropyDetector(Detector):
+    """Flags long, high-entropy tokens as likely secrets."""
+
+    detector_id = "secrets.high_entropy"
+    category = "secret"
+
+    def __init__(self, threshold: float = 4.0, min_len: int = 20) -> None:
+        self.threshold = threshold
+        self.min_len = min_len
+
+    def scan(self, text: str) -> Iterable[Finding]:
+        for m in _TOKEN_RE.finditer(text):
+            token = m.group(0)
+            if len(token) < self.min_len:
+                continue
+            ent = shannon_entropy(token)
+            if ent < self.threshold:
+                continue
+            # Map entropy onto a confidence in [0.5, 0.95].
+            confidence = min(0.95, 0.5 + (ent - self.threshold) / 4.0)
+            yield Finding(
+                detector_id=self.detector_id,
+                category=self.category,
+                confidence=round(confidence, 3),
+                span=m.span(0),
+                matched=token,
+                replacement="",
+            )
+
+
+def secret_detectors(entropy_threshold: float = 4.0, min_entropy_len: int = 20) -> list[Detector]:
+    """The default secret detector stack."""
+    return [
+        RuleDetector(SECRET_RULES),
+        EntropyDetector(threshold=entropy_threshold, min_len=min_entropy_len),
+    ]