vigil-codeintel 0.1.0__py3-none-any.whl

vigil_forensic/gate_checks/forensic_clusters/allowlist_writer.py

@@ -0,0 +1,302 @@
+"""Allowlist writer — programmatic Python API to append FP entries.
+
+Sprint D1 (2026-04-23). Previously only the executor could write
+`false_positive_allowlist.json` via file tool calls. This module adds a
+Python writer so the PE supervisor pipeline can persist FP classifications
+without going through a file-edit round-trip.
+
+Contract
+--------
+* Only findings with `applicability="unknown"` are eligible. Any entry whose
+  `finding_snapshot.applicability` differs raises ValueError (fail-loud).
+* Callers on the PE path must set `classifier="pe_supervisor"`. Other
+  classifier values raise ValueError at this entrypoint.
+* Merge by fingerprint: an existing entry with the same fingerprint is
+  overwritten (latest classifier wins) — idempotent.
+* Atomic file write (tempfile.mkstemp + os.replace) — same pattern as
+  `save_allowlist()` in allowlist.py, safe against partial writes and
+  reasonably safe across concurrent sessions on the same drive.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import tempfile
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Literal, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from vigil_forensic._shared import GateFinding
+
+_log = logging.getLogger(__name__)
+
+_ALLOWLIST_PATH_PARTS = (".prompt-engineer", "forensic_gates", "false_positive_allowlist.json")
+
+_ALLOWED_CLASSIFIERS: frozenset[str] = frozenset({"executor", "pe_supervisor", "human"})
+
+# Sprint (2026-04-24): default TTL for PE-classifier safety mechanisms.
+DEFAULT_PE_TTL_DAYS = 30
+
+
+@dataclass(frozen=True)
+class FPAllowlistEntry:
+    """Structured FP allowlist entry for programmatic writes.
+
+    `finding_snapshot` must contain `applicability` — the writer validates
+    it equals "unknown" before persisting. `expires_at=""` means no TTL
+    (matches executor-written entries).
+    """
+    fingerprint: str
+    reason: str
+    classifier: Literal["executor", "pe_supervisor", "human"]
+    classified_at: str
+    session_num: int
+    finding_snapshot: dict
+    evidence_type: str = "design_decision"
+    expires_at: str = ""
+    # Derived from finding_snapshot when the writer expands to disk format,
+    # but callers may override if they want a custom file/line/evidence.
+    check_id: str = ""
+    file: str = ""
+    line: int = 0
+    evidence: str = ""
+    added_by: str = ""
+    added_at: str = ""
+    reviewed_by: str = ""
+    # Sprint (2026-04-24): TTL + code-hash safety mechanisms.
+    # `created_at` defaults to time.time() at construction; `code_hash` is
+    # populated by the factory (build_pe_fp_entry_from_finding) using
+    # SYSTEM.shared_helpers.file_hash.compute_code_hash.
+    created_at: float = 0.0
+    ttl_days: int = DEFAULT_PE_TTL_DAYS
+    code_hash: str = ""
+    extra: dict = field(default_factory=dict)
+
+    def to_disk_dict(self) -> dict[str, object]:
+        """Flatten into the JSON shape used by false_positive_allowlist.json."""
+        snapshot = dict(self.finding_snapshot)
+        check_id = self.check_id or str(snapshot.get("check_id", "") or "")
+        file_path = self.file or str(snapshot.get("file", "") or "")
+        line_no = self.line or int(snapshot.get("line", 0) or 0)
+        evidence = self.evidence or self.reason
+        added_at = self.added_at or self.classified_at
+        added_by = self.added_by or self.classifier
+        created_at = float(self.created_at) if self.created_at > 0.0 else float(time.time())
+        payload: dict[str, object] = {
+            "fingerprint": self.fingerprint,
+            "check_id": check_id,
+            "file": file_path,
+            "line": line_no,
+            "reason": self.reason,
+            "evidence_type": self.evidence_type,
+            "evidence": evidence,
+            "added_by": added_by,
+            "added_at": added_at,
+            "reviewed_by": self.reviewed_by,
+            "expires_at": self.expires_at,
+            "classifier": self.classifier,
+            "classified_at": self.classified_at,
+            "created_at": created_at,
+            "ttl_days": int(self.ttl_days),
+            "code_hash": self.code_hash,
+            "finding_snapshot": snapshot,
+        }
+        for key, value in self.extra.items():
+            payload.setdefault(key, value)
+        return payload
+
+
+def _resolve_path(project_dir: Path) -> Path:
+    path = Path(project_dir)
+    for part in _ALLOWLIST_PATH_PARTS:
+        path = path / part
+    return path
+
+
+def _validate_entry(entry: FPAllowlistEntry) -> None:
+    """Enforce D1 constraints on a single entry."""
+    if entry.classifier not in _ALLOWED_CLASSIFIERS:
+        raise ValueError(
+            f"write_fp_allowlist_entries: unsupported classifier "
+            f"{entry.classifier!r}; allowed: {sorted(_ALLOWED_CLASSIFIERS)}"
+        )
+    if not entry.fingerprint:
+        raise ValueError("write_fp_allowlist_entries: entry has empty fingerprint")
+    if len((entry.reason or "").strip()) < 10:
+        raise ValueError(
+            f"write_fp_allowlist_entries: reason for {entry.fingerprint!r} "
+            f"too short (min 10 chars): {entry.reason!r}"
+        )
+    snapshot_app = str(entry.finding_snapshot.get("applicability", "") or "")
+    if snapshot_app != "unknown":
+        raise ValueError(
+            f"write_fp_allowlist_entries: finding_snapshot.applicability must be "
+            f"'unknown' for fingerprint {entry.fingerprint!r}, got {snapshot_app!r}. "
+            "Only uncertain findings are eligible for programmatic allowlist writes."
+        )
+
+
+def write_fp_allowlist_entries(
+    project_dir: Path,
+    entries: list[FPAllowlistEntry],
+) -> int:
+    """Atomic JSON update. Returns the number of new or updated entries.
+
+    Idempotent: re-writing an entry with the same fingerprint replaces the
+    existing record rather than adding a duplicate. Entries are first
+    validated — the call fails loudly if any entry violates the D1 contract
+    (non-unknown applicability, missing fingerprint, short reason, unknown
+    classifier). Nothing is written if validation fails.
+    """
+    if not entries:
+        return 0
+
+    # Validate ALL entries before touching the file — fail-loud, no partial writes.
+    for entry in entries:
+        _validate_entry(entry)
+
+    allowlist_path = _resolve_path(project_dir)
+    allowlist_path.parent.mkdir(parents=True, exist_ok=True)
+
+    existing: list[dict] = []
+    if allowlist_path.exists():
+        try:
+            raw = allowlist_path.read_text(encoding="utf-8")
+            loaded = json.loads(raw) if raw.strip() else []
+            if isinstance(loaded, list):
+                existing = loaded
+            else:
+                _log.warning(
+                    "allowlist_writer: existing allowlist is not a JSON array at %s "
+                    "(got %s); ignoring existing content.",
+                    allowlist_path, type(loaded).__name__,
+                )
+        except (OSError, json.JSONDecodeError) as exc:
+            _log.warning(
+                "allowlist_writer: could not read %s (%s); "
+                "writing fresh with new entries only.",
+                allowlist_path, exc,
+            )
+
+    by_fp: dict[str, dict] = {}
+    for item in existing:
+        if isinstance(item, dict):
+            fp = str(item.get("fingerprint", "") or "")
+            if fp:
+                by_fp[fp] = item
+
+    changed = 0
+    for entry in entries:
+        disk = entry.to_disk_dict()
+        fp = entry.fingerprint
+        if fp in by_fp and by_fp[fp] == disk:
+            continue
+        by_fp[fp] = disk
+        changed += 1
+        _log.info(
+            "allowlist_writer: upserted fingerprint=%r classifier=%r check_id=%r",
+            fp, entry.classifier, disk.get("check_id"),
+        )
+
+    merged = list(by_fp.values())
+    content = json.dumps(merged, indent=2, ensure_ascii=False) + "\n"
+
+    tmp_fd, tmp_path = tempfile.mkstemp(dir=str(allowlist_path.parent), suffix=".tmp")
+    try:
+        os.write(tmp_fd, content.encode("utf-8"))
+        os.close(tmp_fd)
+        os.replace(tmp_path, str(allowlist_path))
+    except BaseException:
+        try:
+            os.close(tmp_fd)
+        except OSError:
+            pass
+        try:
+            os.unlink(tmp_path)
+        except OSError:
+            pass
+        raise
+
+    _log.info(
+        "allowlist_writer: wrote %d total entries (%d new/updated) to %s",
+        len(merged), changed, allowlist_path,
+    )
+    return changed
+
+
+def build_pe_fp_entry_from_finding(
+    finding: "GateFinding",
+    *,
+    reason: str,
+    session_num: int,
+    evidence: Optional[str] = None,
+    evidence_type: str = "design_decision",
+    expires_at: str = "",
+    now_iso: str = "",
+    project_dir: Optional[Path] = None,
+    ttl_days: int = DEFAULT_PE_TTL_DAYS,
+) -> FPAllowlistEntry:
+    """Convert a GateFinding into a PE-classified FPAllowlistEntry.
+
+    Only admits findings with applicability="unknown" — the writer rejects
+    anything else at validation time, so this factory does the same check
+    eagerly for better error messages.
+
+    Sprint (2026-04-24): stamps `created_at` (epoch seconds) and `code_hash`
+    (SHA-256 of evidence file). When `project_dir` is supplied, the hash is
+    computed against ``project_dir / primary_path``. When omitted, the hash
+    is left empty (read-time hash check skipped, TTL still applies).
+    """
+    if finding.applicability != "unknown":
+        raise ValueError(
+            f"build_pe_fp_entry_from_finding: finding {finding.fingerprint!r} has "
+            f"applicability={finding.applicability!r} — only 'unknown' findings are "
+            "eligible for PE-supervised FP classification."
+        )
+    if not now_iso:
+        now_iso = datetime.now(tz=timezone.utc).isoformat()
+    primary_path = ""
+    primary_line = 0
+    if finding.evidence:
+        primary_path = (finding.evidence[0].path or "").strip()
+        detail = finding.evidence[0].detail or ""
+        for token in detail.split():
+            try:
+                primary_line = int(token)
+                break
+            except ValueError:
+                continue
+    snapshot = {
+        "check_id": finding.check_id,
+        "confidence": finding.confidence,
+        "applicability": finding.applicability,
+        "applicability_reason": finding.applicability_reason,
+        "analysis_mode": finding.analysis_mode,
+        "session_num": session_num,
+        "file": primary_path,
+        "line": primary_line,
+    }
+    code_hash = ""  # standalone: code-hash stamping unavailable
+    return FPAllowlistEntry(
+        fingerprint=finding.fingerprint,
+        reason=reason.strip(),
+        classifier="pe_supervisor",
+        classified_at=now_iso,
+        session_num=session_num,
+        finding_snapshot=snapshot,
+        evidence_type=evidence_type,
+        expires_at=expires_at,
+        check_id=finding.check_id,
+        file=primary_path,
+        line=primary_line,
+        evidence=(evidence or reason).strip(),
+        added_by="pe_supervisor",
+        added_at=now_iso,
+        created_at=float(time.time()),
+        ttl_days=int(ttl_days),
+        code_hash=code_hash,
+    )

vigil_forensic/gate_checks/forensic_clusters/api_protocol.py

@@ -0,0 +1,231 @@
+"""API/protocol surface forensics. Clusters 27, 28b, 29b, 30."""
+from __future__ import annotations
+
+from .core import detect_language
+from ...gate_models import (
+    EvidenceReference,
+    GateCategory,
+    GateFinding,
+    GateImpact,
+    GateSeverity,
+    RepairKind,
+)
+from ..common import build_finding
+import logging
+_log = logging.getLogger(__name__)
+
+
+def assess_embedded_code_syntax(
+    file_path: str,
+    content: str,
+) -> list[GateFinding]:
+    """Cluster 27: Validate syntax of JS/CSS/HTML embedded in string literals."""
+    import re
+
+    if not content.strip():
+        return []
+    if detect_language(file_path) != "python":
+        return []
+
+    findings: list[GateFinding] = []
+    string_vars = re.finditer(
+        r'^(_?[A-Z][A-Z_0-9]*)\s*=\s*(?:f?"""(.*?)"""|f?\'\'\'(.*?)\'\'\')',
+        content,
+        re.MULTILINE | re.DOTALL,
+    )
+    for match in string_vars:
+        var_name = match.group(1)
+        embedded = match.group(2) or match.group(3) or ""
+        if len(embedded) < 50:
+            continue
+        is_js = bool(re.search(r'\bfunction\b|\bvar\b|\bconst\b|\blet\b|\bdocument\b|\bfetch\b|\baddEventListener\b', embedded))
+        is_css = bool(re.search(r'[.#]\w+\s*\{|:\s*\w+;|@media\b', embedded))
+        is_html = bool(re.search(r'<div\b|<span\b|<nav\b|<button\b|class="', embedded))
+        if not (is_js or is_css or is_html):
+            continue
+        line_num = content[:match.start()].count("\n") + 1
+        open_chars = {'(': ')', '{': '}', '[': ']'}
+        close_chars = {v: k for k, v in open_chars.items()}
+        stack: list[str] = []
+        issue: str | None = None
+        for ch in embedded:
+            if ch in open_chars:
+                stack.append(ch)
+            elif ch in close_chars:
+                if not stack:
+                    issue = f"Unmatched closing '{ch}' in embedded code ({var_name})"
+                    break
+                if stack[-1] != close_chars[ch]:
+                    issue = f"Mismatched brackets in embedded code ({var_name}): expected '{open_chars[stack[-1]]}' got '{ch}'"
+                    break
+                stack.pop()
+        if issue is None and stack:
+            issue = f"Unclosed brackets in embedded code ({var_name}): {''.join(stack[-3:])}"
+        if issue:
+            findings.append(build_finding(
+                check_id="embedded_syntax_scan",
+                category=GateCategory.CONTRACT,
+                title=f"[embedded_code_syntax] {file_path}:{line_num}:{var_name}",
+                severity=GateSeverity.MEDIUM,
+                impact=GateImpact.REVISE,
+                summary=issue,
+                recommendation="Fix bracket mismatch in embedded code constant.",
+                evidence=(EvidenceReference(kind="probe", detail=issue, ok=False),),
+                repair_kind=RepairKind.FIX_CONTRACT.value,
+                executor_action=f"Fix embedded code syntax in {var_name} at {file_path}:{line_num}",
+            ))
+    return findings[:10]
+
+
+# DOM / built-in property names that routinely appear on variables named
+# ``data``/``body``/``result`` etc. but are NOT response-shape fields. Matching
+# these as missing backend keys produces 100% FP on UI/DOM code.
+_DOM_AND_BUILTIN_PROPS: frozenset[str] = frozenset({
+    # DOM API surface
+    "appendChild", "removeChild", "replaceChild", "insertBefore", "cloneNode",
+    "className", "classList", "id", "innerHTML", "innerText", "textContent",
+    "outerHTML", "outerText", "value", "checked", "disabled", "selected",
+    "setAttribute", "getAttribute", "removeAttribute", "hasAttribute",
+    "addEventListener", "removeEventListener", "dispatchEvent",
+    "parentNode", "parentElement", "childNodes", "children",
+    "firstChild", "lastChild", "firstElementChild", "lastElementChild",
+    "nextSibling", "previousSibling", "nextElementSibling", "previousElementSibling",
+    "style", "dataset", "tagName", "nodeType", "nodeName", "nodeValue",
+    "offsetTop", "offsetLeft", "offsetWidth", "offsetHeight", "offsetParent",
+    "clientTop", "clientLeft", "clientWidth", "clientHeight",
+    "scrollTop", "scrollLeft", "scrollWidth", "scrollHeight",
+    "focus", "blur", "click", "scrollIntoView", "remove",
+    "querySelector", "querySelectorAll", "getElementsByClassName",
+    "getElementsByTagName", "contains", "matches", "closest",
+    # Standard response / fetch surface — NOT a business field
+    "then", "catch", "finally", "json", "ok", "status", "statusText",
+    "text", "body", "blob", "formData", "arrayBuffer", "headers",
+    "redirected", "type", "url", "clone",
+    # Array / String / Object methods
+    "length", "map", "forEach", "filter", "find", "findIndex", "slice",
+    "splice", "concat", "reverse", "sort", "flat", "flatMap", "includes",
+    "every", "some", "reduce", "reduceRight",
+    "toString", "valueOf", "constructor", "prototype", "hasOwnProperty",
+    "trim", "trimStart", "trimEnd", "split", "join", "replace", "replaceAll",
+    "indexOf", "lastIndexOf", "toLowerCase", "toUpperCase", "substring",
+    "substr", "charAt", "charCodeAt", "startsWith", "endsWith", "padStart",
+    "padEnd", "repeat", "normalize",
+    "push", "pop", "shift", "unshift", "entries", "keys", "values",
+    # JS error propagation
+    "error", "message", "name", "stack", "cause",
+    # Event handler surface
+    "target", "currentTarget", "preventDefault", "stopPropagation",
+})
+
+
+def assess_response_shape_drift(
+    backend_files: dict[str, str],
+    frontend_files: dict[str, str],
+) -> list[GateFinding]:
+    """Cluster 28b: Detect JS reading fields that backend doesn't provide.
+
+    A field is flagged only when (a) it appears on the RHS of a ``.<field>``
+    access on a response-like identifier AND (b) it isn't a DOM/built-in
+    property like ``appendChild`` or ``className``. See ``_DOM_AND_BUILTIN_PROPS``.
+    """
+    import re
+
+    if not backend_files or not frontend_files:
+        return []
+
+    backend_keys: set[str] = set()
+    for path, content in backend_files.items():
+        # Multi-line object literal support: dotall so ``{\n "a": 1\n}`` matches.
+        for m in re.finditer(r'_send_json\s*\([^{]*\{(.*?)\}', content, re.DOTALL):
+            keys = re.findall(r'["\'](\w+)["\']\s*:', m.group(1))
+            backend_keys.update(keys)
+
+    if not backend_keys:
+        return []
+
+    frontend_access: dict[str, list[str]] = {}
+    for path, content in frontend_files.items():
+        for i, line in enumerate(content.splitlines(), 1):
+            for m in re.finditer(r'\b(?:d|resp|data|result|body|payload|status)\.([\w]+)\b', line):
+                field = m.group(1)
+                if field in _DOM_AND_BUILTIN_PROPS:
+                    continue
+                frontend_access.setdefault(field, []).append(f"{path}:{i}")
+
+    if not frontend_access:
+        return []
+
+    findings: list[GateFinding] = []
+    for field, locations in sorted(frontend_access.items()):
+        if field not in backend_keys:
+            findings.append(build_finding(
+                check_id="response_shape_scan",
+                category=GateCategory.DRIFT,
+                title=f"[response_shape_drift] .{field}",
+                severity=GateSeverity.MEDIUM,
+                impact=GateImpact.REVISE,
+                summary=f"Frontend reads '.{field}' but no backend _send_json includes '{field}' key",
+                recommendation=f"Add '{field}' to backend response or remove frontend read.",
+                evidence=(EvidenceReference(kind="probe", detail=f"Frontend reads '.{field}' but no backend _send_json includes '{field}' key", ok=False),),
+                repair_kind=RepairKind.NORMALIZE_SHAPE.value,
+                executor_action=f"Sync response shape for field '{field}'",
+            ))
+        if len(findings) >= 20:
+            break
+    return findings
+
+
+def assess_http_method_consistency(
+    route_methods: dict[str, str],
+    js_fetches: list[tuple[str, str, str]],
+) -> list[GateFinding]:
+    """Cluster 29b: Verify JS fetch methods match registered route methods."""
+    if not route_methods or not js_fetches:
+        return []
+
+    findings: list[GateFinding] = []
+    for url, js_method, source in js_fetches:
+        clean_url = url.split("?")[0]
+        if clean_url in route_methods:
+            expected = route_methods[clean_url]
+            if js_method.upper() != expected.upper():
+                detail = f"JS fetches {clean_url} with {js_method.upper()} but route expects {expected.upper()}"
+                findings.append(build_finding(
+                    check_id="method_match_scan",
+                    category=GateCategory.CONTRACT,
+                    title=f"[http_method_consistency] {clean_url}",
+                    severity=GateSeverity.HIGH,
+                    impact=GateImpact.REVISE,
+                    summary=detail,
+                    recommendation=f"Change JS fetch method for {clean_url} to {expected.upper()}.",
+                    evidence=(EvidenceReference(kind="probe", detail=detail, ok=False),),
+                    repair_kind=RepairKind.FIX_CONTRACT.value,
+                    executor_action=f"Fix HTTP method mismatch for {clean_url}",
+                ))
+    return findings
+
+
+def assess_js_surface_coverage(
+    all_js_constants: list[str],
+    checked_js_constants: list[str],
+) -> list[GateFinding]:
+    """Cluster 30: Verify all JS surface constants are covered by forensics."""
+    if not all_js_constants:
+        return []
+
+    findings: list[GateFinding] = []
+    for name in all_js_constants:
+        if name not in checked_js_constants:
+            findings.append(build_finding(
+                check_id="js_coverage_scan",
+                category=GateCategory.CONTRACT,
+                title=f"[js_surface_coverage] {name}",
+                severity=GateSeverity.MEDIUM,
+                impact=GateImpact.REVISE,
+                summary=f"JS constant '{name}' exists but is not checked by route/contract forensics",
+                recommendation=f"Add '{name}' to the checked_js_constants list in _check_js_surface_coverage.",
+                evidence=(EvidenceReference(kind="probe", detail=f"JS constant '{name}' exists but is not checked by route/contract forensics", ok=False),),
+                repair_kind=RepairKind.ADD_PROOF.value,
+                executor_action=f"Add '{name}' to JS surface coverage checks",
+            ))
+    return findings