ciphergate-cli 0.7.0__py3-none-any.whl

ciphergate/pipeline/inspector.py

@@ -0,0 +1,282 @@
+"""Run setup and per-record analysis (readers -> detectors -> policy engine).
+
+The same ``Analyzer`` is used by inspect, plan, sanitize (both passes) and rescan, so the plan
+predicts exactly the decisions that apply mode executes.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass, replace
+from pathlib import Path
+
+from ciphergate.config import RuntimeSettings
+from ciphergate.constants import STACK_TRACE_DETECTOR, Action, InputFormat, ReasonCode
+from ciphergate.detectors.base import is_ciphergate_token, make_finding
+from ciphergate.detectors.registry import (
+    FIELD_CONTEXT_DETECTORS,
+    DetectorSet,
+    TextBlockState,
+)
+from ciphergate.models.context import ContextInput, EffectiveContext
+from ciphergate.models.finding import Finding, ScanItem
+from ciphergate.pipeline.paths import validate_input_path
+from ciphergate.policy.context import resolve_context
+from ciphergate.policy.engine import Decision, PolicyEngine
+from ciphergate.policy.precedence import LEVEL_HARD_CONSTRAINT
+from ciphergate.policy.validator import Policy
+from ciphergate.readers import (
+    STRUCTURED_LINE_FORMATS,
+    ReaderOptions,
+    Record,
+    detect_format,
+    open_reader,
+)
+from ciphergate.readers.base import Reader, normalize_encoding
+from ciphergate.readers.json_reader import JSONReader
+from ciphergate.readers.line_structure import (
+    LineStructure,
+    region_tokens,
+    snap_span,
+    token_at,
+)
+from ciphergate.readers.logfmt_reader import field_for_span, parse_fields
+from ciphergate.readers.stacktrace_reader import FRAME, TraceState, classify
+
+Evaluated = list[tuple[Finding, Decision]]
+
+
+def _attribute(finding: Finding, key: str) -> Finding:
+    return _attribute_path(finding, key, "/" + key)
+
+
+def _attribute_path(finding: Finding, key: str, path: str) -> Finding:
+    location = replace(finding.location, field_path=path)
+    return replace(finding, field_name=key, location=location)
+
+
+def _with_span(finding: Finding, span: tuple[int, int]) -> Finding:
+    return replace(finding, location=replace(finding.location, span=span))
+
+
+@dataclass(frozen=True)
+class RunSetup:
+    input_path: Path
+    fmt: InputFormat
+    policy: Policy
+    ctx: EffectiveContext
+    runtime: RuntimeSettings
+    reader_options: ReaderOptions
+    detectors: DetectorSet
+    engine: PolicyEngine
+
+    def new_reader(self, path: Path | None = None) -> Reader:
+        return open_reader(path or self.input_path, self.fmt, self.reader_options)
+
+
+def prepare_run(
+    input_path: str | Path,
+    policy: Policy,
+    context: ContextInput | None,
+    runtime: RuntimeSettings | None = None,
+    fmt: InputFormat | None = None,
+) -> RunSetup:
+    runtime = runtime or RuntimeSettings()
+    ctx = resolve_context(policy, context)
+    path = Path(input_path)
+    validate_input_path(path, runtime.max_input_bytes)
+    detected = detect_format(path, fmt)
+    options = ReaderOptions(
+        runtime=runtime,
+        policy_settings=policy.document.settings,
+        encoding=normalize_encoding(runtime.encoding),
+    )
+    return RunSetup(
+        input_path=path,
+        fmt=detected,
+        policy=policy,
+        ctx=ctx,
+        runtime=runtime,
+        reader_options=options,
+        detectors=DetectorSet(policy.document.detectors, policy.document),
+        engine=PolicyEngine(policy),
+    )
+
+
+class Analyzer:
+    def __init__(self, setup: RunSetup, reader: Reader) -> None:
+        self.setup = setup
+        self.reader = reader
+        self.text_state = TextBlockState()
+        self.trace_state = TraceState()
+        self._single_json_document = isinstance(reader, JSONReader)
+
+    def analyze(self, record: Record) -> list[tuple[ScanItem, Evaluated]]:
+        return self._analyze_items(self.reader.iter_items(record))
+
+    def analyze_header(self) -> list[tuple[ScanItem, Evaluated]]:
+        """CSV header cells (key items). Must run before ``records()`` is iterated."""
+        header_items = getattr(self.reader, "header_items", None)
+        if header_items is None:
+            return []
+        return self._analyze_items(header_items())
+
+    def _analyze_items(self, items: Iterable[ScanItem]) -> list[tuple[ScanItem, Evaluated]]:
+        setup = self.setup
+        out: list[tuple[ScanItem, Evaluated]] = []
+        for item in items:
+            if setup.fmt == InputFormat.TEXT:
+                findings = setup.detectors.scan_text_line(item, self.text_state)
+            elif setup.fmt == InputFormat.LOGFMT:
+                findings = self._scan_logfmt_line(item)
+            elif setup.fmt == InputFormat.STACKTRACE:
+                findings = self._scan_stacktrace_line(item)
+            elif setup.fmt in STRUCTURED_LINE_FORMATS:
+                findings = self._scan_structured_line(item, STRUCTURED_LINE_FORMATS[setup.fmt])
+            else:
+                findings = setup.detectors.scan(item)
+            if not findings:
+                continue
+            evaluated = [
+                (f, self._constrain(setup.engine.decide(f, setup.ctx, setup.fmt), item))
+                for f in findings
+            ]
+            out.append((item, evaluated))
+        return out
+
+    def _scan_logfmt_line(self, item: ScanItem) -> list[Finding]:
+        """Text-mode scan of the whole line, plus field context from the key=value parse."""
+        detectors = self.setup.detectors
+        in_key_block = self.text_state.in_private_key_block
+        findings = detectors.scan_text_line(item, self.text_state)
+        fields = None if in_key_block else parse_fields(item.text or "")
+        if not fields:
+            return findings
+        out: list[Finding] = []
+        for finding in findings:
+            span = finding.location.span
+            field = field_for_span(fields, span) if span is not None else None
+            out.append(_attribute(finding, field.key) if field is not None else finding)
+        for field in fields:
+            if field.end <= field.start:
+                continue  # empty value: nothing to protect
+            key_item = ScanItem(
+                item.record_index, item.line, "/" + field.key, field.key, field.value
+            )
+            for finding in detectors.scan_field_context(key_item):
+                location = replace(
+                    finding.location, field_path="/" + field.key, span=(field.start, field.end)
+                )
+                out.append(replace(finding, location=location))
+        return detectors.post_process(out)
+
+    def _scan_stacktrace_line(self, item: ScanItem) -> list[Finding]:
+        """Text-mode scan of the line, plus stack trace context from the lines before it."""
+        in_key_block = self.text_state.in_private_key_block
+        findings = self.setup.detectors.scan_text_line(item, self.text_state)
+        if in_key_block:
+            self.trace_state.reset()
+            return findings
+        trace = classify(item.text or "", self.trace_state)
+        if trace is None:
+            return findings
+        # Per-line findings are kept as they are and only get the trace path. Frame and source
+        # lines also get a finding of their own, so the policy decides what leaves with them.
+        trace_item = replace(item, field_path=trace.pointer)
+        out = [replace(f, location=replace(f.location, field_path=trace.pointer)) for f in findings]
+        if trace.span is not None:
+            out.append(
+                make_finding(
+                    trace_item,
+                    category="proprietary.source",
+                    detector=STACK_TRACE_DETECTOR,
+                    confidence=0.9 if trace.role == FRAME else 0.8,
+                    evidence=("structure_match", f"family:{trace.family}"),
+                    span=trace.span,
+                )
+            )
+        return out
+
+    def _scan_structured_line(self, item: ScanItem, structure: LineStructure) -> list[Finding]:
+        """Text-mode scan of the whole line, plus field context from the line's parsed fields."""
+        detectors = self.setup.detectors
+        in_key_block = self.text_state.in_private_key_block
+        findings = detectors.scan_text_line(item, self.text_state)
+        text = item.text or ""
+        max_depth = self.setup.runtime.max_json_depth
+        regions = [] if in_key_block else structure.parse(text, max_depth)
+        if not regions:
+            return findings
+        tokens = region_tokens(regions)
+        out: list[Finding] = []
+        for finding in findings:
+            span = finding.location.span
+            if span is None:
+                out.append(finding)
+                continue
+            # Never let a replacement cross field syntax; keep the original span if nothing is left.
+            for piece in snap_span(regions, *span) or [span]:
+                moved = finding if piece == span else _with_span(finding, piece)
+                owner = token_at(tokens, *piece)
+                if owner is not None and owner.kind != "key" and owner.key is not None:
+                    moved = _attribute_path(moved, owner.key, owner.pointer)
+                out.append(moved)
+        for token in tokens:
+            if token.kind == "key" or token.key is None or token.end <= token.start:
+                continue
+            value_item = ScanItem(
+                item.record_index, item.line, token.pointer, token.key, token.value
+            )
+            # A key CipherGate wrote (marker/pseudonym) says nothing about its value: field
+            # context from it would flag our own output. Content detectors still run.
+            own_key = is_ciphergate_token(token.key)
+            if own_key and not token.escaped:
+                continue
+            # Escaped strings hide content from the raw-text scan: scan the decoded value too.
+            found = (
+                detectors.scan(value_item)
+                if token.escaped
+                else detectors.scan_field_context(value_item)
+            )
+            if token.hint and not own_key:
+                # A canonical name (e.g. CEF "suser" -> "source_user_name") adds heuristics on
+                # top of the real key; findings keep the real key for policy field_names rules.
+                hint_item = replace(value_item, field_name=token.hint)
+                found += detectors.scan_field_context(hint_item)
+            if own_key:
+                found = [f for f in found if f.detector not in FIELD_CONTEXT_DETECTORS]
+            for finding in found:
+                location = replace(
+                    finding.location, field_path=token.pointer, span=(token.start, token.end)
+                )
+                out.append(replace(finding, field_name=token.key, location=location))
+        for token in tokens:
+            if token.kind == "key" and token.escaped:
+                key_item = ScanItem(item.record_index, item.line, token.pointer, None, token.value)
+                for finding in detectors.scan(key_item):
+                    location = replace(finding.location, span=(token.start, token.end))
+                    out.append(replace(finding, location=location))
+        # Each scan above already applied phone suppression; re-running it on the merged list
+        # would let whole-value spans hide findings, so only de-duplicate.
+        return detectors.dedupe(out)
+
+    def _constrain(self, decision: Decision, item: ScanItem) -> Decision:
+        """Location-dependent applicability (fail closed to review, never silently skip)."""
+        single_doc = self._single_json_document and not getattr(
+            self.reader, "top_level_array", False
+        )
+        header_cell = item.record_index < 0
+        unsupported = (decision.action == Action.DROP_RECORD and (single_doc or header_cell)) or (
+            decision.action == Action.DROP_FIELD
+            and self.setup.fmt in (InputFormat.JSON, InputFormat.JSONL)
+            and item.field_name is None
+            and not item.in_key
+        )
+        if unsupported:
+            return Decision(
+                Action.REVIEW,
+                ReasonCode.UNSUPPORTED_TRANSFORM,
+                LEVEL_HARD_CONSTRAINT,
+                overridden_rule_id=decision.rule_id,
+            )
+        return decision

ciphergate/pipeline/paths.py

@@ -0,0 +1,98 @@
+"""Input/output path safety checks.
+
+* Input must be an existing regular file (symlinks are resolved; devices, FIFOs, sockets and
+  directories are rejected) within ``max_input_bytes``.
+* Output must not resolve to the input or any other protected path (realpath + ``samefile``,
+  case-normalised on Windows), must not be a symlink or directory, and must not already exist
+  unless ``force`` is given.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import stat
+from pathlib import Path
+
+from ciphergate.errors import InputLimitError, OutputWriteError, UnsafePathError
+
+
+def validate_input_path(path: Path, max_bytes: int) -> Path:
+    try:
+        st = os.stat(path)  # follows symlinks: we check what will actually be read
+    except FileNotFoundError:
+        raise UnsafePathError("input file does not exist", code="input_not_found") from None
+    except OSError:
+        raise UnsafePathError("input file cannot be accessed", code="input_unreadable") from None
+    if stat.S_ISDIR(st.st_mode):
+        raise UnsafePathError(
+            "input is a directory; use a *_directory function", code="input_is_directory"
+        )
+    if not stat.S_ISREG(st.st_mode):
+        raise UnsafePathError(
+            "input is not a regular file (devices/FIFOs are refused)", code="input_not_regular"
+        )
+    if st.st_size > max_bytes:
+        raise InputLimitError(
+            f"input exceeds max_input_bytes ({max_bytes})", code="size_limit_exceeded"
+        )
+    return Path(os.path.realpath(path))
+
+
+def _same(a: Path, b: Path) -> bool:
+    ra = os.path.normcase(os.path.realpath(a))
+    rb = os.path.normcase(os.path.realpath(b))
+    if ra == rb:
+        return True
+    try:
+        return a.exists() and b.exists() and os.path.samefile(a, b)
+    except OSError:
+        return False
+
+
+def validate_output_path(
+    path: Path, *, protected: list[Path], force: bool, label: str = "output"
+) -> Path:
+    for other in protected:
+        if _same(path, other):
+            raise UnsafePathError(
+                f"{label} path refers to the input or another protected file",
+                code=f"{label}_aliases_input",
+            )
+    if path.is_symlink():
+        raise UnsafePathError(
+            f"{label} path is a symlink; refusing to write through it", code=f"{label}_is_symlink"
+        )
+    if path.exists():
+        if path.is_dir():
+            raise UnsafePathError(f"{label} path is a directory", code=f"{label}_is_directory")
+        if not stat.S_ISREG(path.stat().st_mode):
+            raise UnsafePathError(
+                f"{label} path is not a regular file", code=f"{label}_not_regular"
+            )
+        if not force:
+            raise OutputWriteError(
+                f"{label} file already exists; use --force to replace it", code=f"{label}_exists"
+            )
+    parent = path.parent if str(path.parent) else Path(".")
+    if parent.exists() and not parent.is_dir():
+        raise UnsafePathError(f"{label} parent is not a directory", code=f"{label}_parent_invalid")
+    return path
+
+
+def file_digest(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(1024 * 1024), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def display_path(path: Path | None, mode: str) -> str | None:
+    if path is None:
+        return None
+    if mode == "redacted":
+        return "<redacted>" + (path.suffix if path.suffix else "")
+    if mode == "full":
+        return str(path)
+    return path.name

ciphergate/pipeline/planner.py

@@ -0,0 +1,192 @@
+"""Pass one: analyse the whole input and aggregate a privacy-safe plan.
+
+The plan is the single source of truth for whether apply mode may run. ``block`` or ``review``
+decisions, run-level gate issues, or unavailable keys all prevent output publication.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass, field
+from typing import Any
+
+from ciphergate.constants import Action, InputFormat
+from ciphergate.errors import InputLimitError
+from ciphergate.models.finding import Finding, ScanItem
+from ciphergate.models.policy import DateShiftParams, PseudonymizeParams
+from ciphergate.models.report import RunStatus
+from ciphergate.pipeline.inspector import Analyzer, RunSetup
+from ciphergate.pipeline.paths import file_digest
+from ciphergate.policy.context import GateIssue, evaluate_gates
+from ciphergate.policy.engine import Decision
+from ciphergate.readers.base import ReadStats
+from ciphergate.reporting.safe_text import safe_field_path
+from ciphergate.transforms.keys import KeyProvider, KeyStatus
+
+REDACTED_KEY = "<redacted-key>"
+
+
+def redact_key_path(path: str) -> str:
+    """Hide the last segment of a path whose key/header itself produced a finding."""
+    if path.startswith("/"):
+        head = path.rsplit("/", 1)[0]
+        return safe_field_path(head) + "/" + REDACTED_KEY if head else "/" + REDACTED_KEY
+    return REDACTED_KEY
+
+
+def finding_item(
+    finding: Finding, decision: Decision, *, redact_path: bool = False
+) -> dict[str, Any]:
+    loc = finding.location.to_dict()
+    if "field_path" in loc:
+        path = loc["field_path"]
+        loc["field_path"] = redact_key_path(path) if redact_path else safe_field_path(path)
+    return {
+        "category": finding.category,
+        "detector": finding.detector,
+        "confidence": finding.confidence,
+        "band": finding.band,
+        "evidence": list(finding.evidence),
+        "value_type": finding.value_type,
+        "location": loc,
+        "decision": decision.to_dict(),
+    }
+
+
+@dataclass
+class PlanAccumulator:
+    max_items: int
+    max_findings: int
+    total: int = 0
+    by_category: Counter[str] = field(default_factory=Counter)
+    by_detector: Counter[str] = field(default_factory=Counter)
+    by_action: Counter[str] = field(default_factory=Counter)
+    by_band: Counter[str] = field(default_factory=Counter)
+    by_reason: Counter[str] = field(default_factory=Counter)
+    by_rule: Counter[str] = field(default_factory=Counter)
+    blocked: int = 0
+    review: int = 0
+    items: list[dict[str, Any]] = field(default_factory=list)
+    truncated: bool = False
+    dropped_columns: dict[int, str] = field(default_factory=dict)
+    records_to_drop: set[int] = field(default_factory=set)
+    malformed_records: int = 0
+    key_envs: set[str] = field(default_factory=set)
+    # CSV columns whose header cell produced a finding: their label is never reported.
+    sensitive_columns: set[int] = field(default_factory=set)
+
+    def add(self, item: ScanItem, finding: Finding, decision: Decision) -> None:
+        self.total += 1
+        if self.total > self.max_findings:
+            raise InputLimitError(
+                f"more than {self.max_findings} findings (max_findings)", code="too_many_findings"
+            )
+        if item.in_key and item.column_index is not None:
+            self.sensitive_columns.add(item.column_index)
+        redact = item.in_key or item.column_index in self.sensitive_columns
+        action = str(decision.action)
+        self.by_category[finding.category] += 1
+        self.by_detector[finding.detector] += 1
+        self.by_action[action] += 1
+        self.by_band[finding.band] += 1
+        self.by_reason[str(decision.reason_code)] += 1
+        if decision.rule_id:
+            self.by_rule[decision.rule_id] += 1
+        if decision.action == Action.BLOCK:
+            self.blocked += 1
+        elif decision.action == Action.REVIEW:
+            self.review += 1
+        elif decision.action == Action.DROP_FIELD and item.column_index is not None:
+            self.dropped_columns[item.column_index] = (
+                REDACTED_KEY if redact else safe_field_path(item.field_path)
+            )
+        elif decision.action == Action.DROP_RECORD:
+            self.records_to_drop.add(item.record_index)
+        if isinstance(decision.params, PseudonymizeParams | DateShiftParams):
+            self.key_envs.add(decision.params.key_env)
+        important = decision.action in (Action.BLOCK, Action.REVIEW)
+        if len(self.items) < self.max_items:
+            self.items.append(finding_item(finding, decision, redact_path=redact))
+        elif important:
+            # Make sure blocking items are visible even when the item list is full.
+            self.items.append(finding_item(finding, decision, redact_path=redact))
+            self.truncated = True
+        else:
+            self.truncated = True
+
+    def summary(self) -> dict[str, Any]:
+        def ordered(counter: Counter[str]) -> dict[str, int]:
+            return dict(sorted(counter.items()))
+
+        return {
+            "findings_total": self.total,
+            "by_category": ordered(self.by_category),
+            "by_detector": ordered(self.by_detector),
+            "by_action": ordered(self.by_action),
+            "by_confidence_band": ordered(self.by_band),
+            "by_reason_code": ordered(self.by_reason),
+            "by_rule": ordered(self.by_rule),
+            "blocked": self.blocked,
+            "review_required": self.review,
+            "records_to_drop": len(self.records_to_drop),
+            "malformed_records_dropped": self.malformed_records,
+        }
+
+
+@dataclass
+class Analysis:
+    setup: RunSetup
+    stats: ReadStats
+    acc: PlanAccumulator
+    gates: list[GateIssue]
+    approvals: list[str]
+    digest: str
+    key_statuses: dict[str, KeyStatus]
+    top_level_array: bool
+    key_sources: dict[str, str] = field(default_factory=dict)
+
+    @property
+    def status(self) -> RunStatus:
+        if self.acc.blocked or any(g.severity == "block" for g in self.gates):
+            return RunStatus.BLOCKED
+        if self.acc.review or any(g.severity == "review" for g in self.gates):
+            return RunStatus.REVIEW_REQUIRED
+        if any(s != KeyStatus.AVAILABLE for s in self.key_statuses.values()):
+            return RunStatus.CONFIGURATION_REQUIRED
+        return RunStatus.READY
+
+
+def analyze_file(setup: RunSetup, keys: KeyProvider) -> Analysis:
+    gates, approvals = evaluate_gates(setup.policy, setup.ctx, setup.fmt)
+    digest = file_digest(setup.input_path)
+    reader = setup.new_reader()
+    analyzer = Analyzer(setup, reader)
+    acc = PlanAccumulator(
+        max_items=setup.runtime.max_report_items, max_findings=setup.runtime.max_findings
+    )
+    for item, evaluated in analyzer.analyze_header():
+        for finding, decision in evaluated:
+            acc.add(item, finding, decision)
+    for record in reader.records():
+        if record.malformed:
+            acc.malformed_records += 1
+            continue
+        for item, evaluated in analyzer.analyze(record):
+            for finding, decision in evaluated:
+                acc.add(item, finding, decision)
+    key_statuses = {name: keys.status(name) for name in sorted(acc.key_envs)}
+    return Analysis(
+        setup=setup,
+        stats=reader.stats,
+        acc=acc,
+        gates=gates,
+        approvals=approvals,
+        digest=digest,
+        key_statuses=key_statuses,
+        top_level_array=bool(getattr(reader, "top_level_array", False)),
+        key_sources={name: keys.source(name) for name in key_statuses},
+    )
+
+
+def is_structured(fmt: InputFormat) -> bool:
+    return fmt != InputFormat.TEXT

ciphergate/pipeline/rescan.py

@@ -0,0 +1,120 @@
+"""Residual scanning of (sanitized) output. Read-only; never mutates the scanned file.
+
+Each residual finding is classified:
+
+* ``allowed`` — the policy decision for it is ``allow``;
+* ``recognized_output`` — the text is a CipherGate marker, a pseudonym token, a configured
+  replacement constant, or already has the shape of the configured generalization;
+* ``unverifiable`` — the policy would date-shift it (a shifted date looks like any date);
+* ``unexpected`` — everything else. Unexpected residuals in ``settings.rescan.fail_on_categories``
+  (default: high-risk categories) are *gating*; in strict mode they fail the rescan.
+
+A clean rescan is a quality gate, not proof that no sensitive data remains.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+from ciphergate.constants import Action
+from ciphergate.detectors.base import is_ciphergate_token
+from ciphergate.models.finding import ScanItem
+from ciphergate.models.policy import GeneralizeParams
+from ciphergate.models.report import RescanSummary
+from ciphergate.pipeline.inspector import Analyzer, Evaluated, RunSetup
+from ciphergate.pipeline.planner import finding_item
+from ciphergate.readers import STRUCTURED_LINE_FORMATS
+from ciphergate.readers.line_structure import SpanCodec, token_at
+from ciphergate.transforms.generalize import matches_generalized_shape
+
+
+def rescan_path(setup: RunSetup, path: Path, *, strict: bool) -> RescanSummary:
+    reader = setup.new_reader(path)
+    analyzer = Analyzer(setup, reader)
+    policy = setup.policy
+    fail_set = policy.rescan_fail_categories(untrusted=setup.ctx.is_untrusted)
+    counts: Counter[str] = Counter(
+        {"allowed": 0, "recognized_output": 0, "unverifiable": 0, "unexpected": 0}
+    )
+    unexpected: Counter[str] = Counter()
+    gating: Counter[str] = Counter()
+    items: list[dict[str, Any]] = []
+    truncated = False
+    sensitive_columns: set[int] = set()
+
+    def analyzed() -> Iterator[tuple[ScanItem, Evaluated]]:
+        yield from analyzer.analyze_header()
+        for record in reader.records():
+            if record.malformed:
+                counts["malformed_records"] += 1
+                continue
+            yield from analyzer.analyze(record)
+
+    for item, evaluated in analyzed():
+        if item.in_key and item.column_index is not None and evaluated:
+            sensitive_columns.add(item.column_index)
+        redact = item.in_key or item.column_index in sensitive_columns
+        text = item.text if item.text is not None else ""
+        # Structured line values are escaped in the line: compare what the transform wrote.
+        structure = STRUCTURED_LINE_FORMATS.get(setup.fmt)
+        codec = (
+            SpanCodec(structure, structure.parse(text, setup.runtime.max_json_depth))
+            if structure is not None
+            else None
+        )
+        for finding, decision in evaluated:
+            span = finding.location.span
+            piece = text[span[0] : span[1]] if span else text
+            whole: str | None = None  # the decoded value of the field the finding sits in
+            if codec is not None and span is not None:
+                decoded = codec.decode(span[0], span[1], piece)
+                piece = decoded if isinstance(decoded, str) else piece
+                owner = token_at(codec.tokens, span[0], span[1])
+                if owner is not None and owner.kind != "key" and isinstance(owner.value, str):
+                    whole = owner.value
+            if decision.action == Action.ALLOW:
+                cls = "allowed"
+            elif (
+                is_ciphergate_token(piece)
+                or piece in policy.replacement_values
+                # A pattern can match part of an escaped marker (e.g. "[REDACTED:x\]" in a
+                # syslog SD value); a field that holds exactly our output is recognised.
+                or (whole is not None and is_ciphergate_token(whole))
+                or (whole is not None and whole in policy.replacement_values)
+                or (
+                    decision.action == Action.GENERALIZE
+                    and isinstance(decision.params, GeneralizeParams)
+                    and matches_generalized_shape(piece, decision.params)
+                )
+            ):
+                cls = "recognized_output"
+            elif decision.action == Action.DATE_SHIFT:
+                cls = "unverifiable"
+            else:
+                cls = "unexpected"
+            counts[cls] += 1
+            if cls != "unexpected":
+                continue
+            unexpected[finding.category] += 1
+            is_gating = finding.category in fail_set
+            if is_gating:
+                gating[finding.category] += 1
+            if len(items) < setup.runtime.max_report_items:
+                entry = finding_item(finding, decision, redact_path=redact)
+                entry["residual"] = "gating" if is_gating else "non_gating"
+                items.append(entry)
+            else:
+                truncated = True
+    return RescanSummary(
+        strict=strict,
+        failed=strict and sum(gating.values()) > 0,
+        fail_on_categories=sorted(fail_set),
+        counts=dict(counts),
+        unexpected_by_category=dict(sorted(unexpected.items())),
+        gating_by_category=dict(sorted(gating.items())),
+        items=items,
+        items_truncated=truncated,
+    )