differix 0.1.0a1__py3-none-any.whl

tools/security/verification_firewall.py

@@ -0,0 +1,113 @@
+"""
+tools/security/verification_firewall.py
+----------------------------------------
+Agent self-reports are ADVISORY. Only external signals are AUTHORITATIVE.
+
+Problem this solves (August 2026 incident):
+  An agent in a loop hallucinated "test passed" and the pipeline believed it,
+  because `state.get("compiler_output")` was populated by the agent itself.
+  The agent then spent tokens on simulated work instead of real work.
+
+Solution:
+  build_success MUST come from run_universal_build(), never from state.
+  apply_result.ok MUST be verified via fs-hash before/after, not trusted.
+  Anything the agent claims about success must be cross-checked.
+
+This module provides those cross-checks.
+"""
+
+import hashlib
+import os
+from typing import Optional
+
+
+def fs_hash(path: str) -> str:
+    """SHA-256 of file content, or empty string if file doesn't exist."""
+    if not os.path.exists(path):
+        return ""
+    try:
+        with open(path, "rb") as f:
+            return hashlib.sha256(f.read()).hexdigest()
+    except OSError:
+        return ""
+
+
+def verify_file_changed(path: str, before_hash: str) -> tuple[bool, str]:
+    """
+    Verify that the file actually changed after the agent claimed a write.
+
+    Returns:
+      (changed, current_hash)
+    """
+    after_hash = fs_hash(path)
+    changed = (after_hash != before_hash)
+    return changed, after_hash
+
+
+def verify_compiler_output(
+    compiler_output_from_state: str,
+    actual_compiler_output: str,
+) -> dict:
+    """
+    Cross-check what state says vs what the compiler actually said.
+
+    In a healthy run, these match. In a confused-deputy attack, they diverge.
+
+    Returns:
+      {"match": bool, "state_claimed": str, "actual": str, "tampered": bool}
+    """
+    # Normalize whitespace for comparison
+    state_norm = (compiler_output_from_state or "").strip()
+    actual_norm = (actual_compiler_output or "").strip()
+
+    match = state_norm == actual_norm
+    tampered = bool(state_norm) and bool(actual_norm) and not match
+
+    return {
+        "match": match,
+        "state_claimed": state_norm[:200],
+        "actual": actual_norm[:200],
+        "tampered": tampered,
+    }
+
+
+def verify_build_result(
+    claimed_success: bool,
+    actual_success: bool,
+    actual_log: str,
+) -> dict:
+    """
+    Cross-check what state/build_success says vs what run_universal_build returned.
+
+    Returns:
+      {"consistent": bool, "trusted_success": bool, "warnings": [str]}
+    """
+    warnings = []
+
+    if claimed_success != actual_success:
+        warnings.append(
+            f"build_success mismatch: state={claimed_success} "
+            f"but run_universal_build={actual_success}"
+        )
+
+    trusted_success = actual_success  # Only trust run_universal_build
+
+    if claimed_success and not actual_success:
+        warnings.append(
+            "AGENT CLAIMED SUCCESS BUT COMPILER FAILED. "
+            "This is the 'simulated test passed' incident pattern."
+        )
+
+    return {
+        "consistent": claimed_success == actual_success,
+        "trusted_success": trusted_success,
+        "warnings": warnings,
+    }
+
+
+__all__ = [
+    "fs_hash",
+    "verify_file_changed",
+    "verify_compiler_output",
+    "verify_build_result",
+]

tools/security_filter.py

@@ -0,0 +1,213 @@
+"""
+tools/security_filter.py
+------------------------
+Detects dangerous code patterns before file write.
+
+Two categories of patterns:
+
+1. ALWAYS_BLOCKED — remote code execution, arbitrary deserialization,
+   subprocess spawning. Blocked in every mode, including RSI.
+
+2. CLEANUP_ALLOWED — file deletion primitives (unlink, rmtree, rmdir).
+   Blocked in normal mode (agent editing a user project could destroy
+   data). Allowed in RSI mode (framework itself needs these for
+   rollback, temp cleanup, and atomic file operations).
+
+Mode is selected by config.RSI_MODE. The filter never distinguishes
+between new and existing files — a cleanup call is either allowed or
+not, regardless of context. This is simpler and more predictable than
+the previous diff-based approach.
+"""
+
+import ast
+import re
+from typing import List, Tuple
+
+_LINE_COMMENT_RE = re.compile(r'^\s*(#|//)')
+_TRAILING_COMMENT_RE = re.compile(r'\s+(#|//).*$')
+_BLOCK_COMMENT_RE = re.compile(r'/\*.*?\*/', re.DOTALL)
+
+
+# ---------------------------------------------------------------------------
+# Category 1: always blocked. RCE, deserialization, process spawning.
+# ---------------------------------------------------------------------------
+ALWAYS_BLOCKED_PATTERNS = {
+    r'\beval\s*\(': "eval()",
+    r'\bexec\s*\(': "exec()",
+    r'\bcompile\s*\(': "compile() (builds executable code objects)",
+    r'__import__\s*\(': "__import__()",
+    r'os\.popen\s*\(': "os.popen()",
+    r'os\.exec[lv]p?e?\s*\(': "os.exec*() family",
+    r'os\.spawn[lv]p?e?\s*\(': "os.spawn*() family",
+    r'pty\.spawn\s*\(': "pty.spawn()",
+    r'ctypes\.(CDLL|WinDLL|windll|cdll)': "ctypes native code loading",
+    r'pickle\.loads?\s*\(': "pickle.load()/loads() (arbitrary deserialization)",
+    r'marshal\.loads?\s*\(': "marshal.load()/loads()",
+    r'getattr\s*\(\s*(os|subprocess|shutil|pickle|marshal|ctypes)\s*,': "dynamic getattr(module, ...) - possible obfuscated dangerous call",
+    r'os\.system\s*\(': "os.system()",
+    r'subprocess\.(run|Popen|call|check_output|check_call)\s*\(': "subprocess call",
+
+    r'rm\s+-rf': "rm -rf",
+    r'Remove-Item\s+.*-Recurse': "Remove-Item -Recurse",
+    r'del\s+/s': "del /s",
+
+    r'def\s+test_.*inspect': "Inspection test definition (test_inspect)",
+    r'raise\s+Exception\s*\(.*(os\.listdir|glob)\s*\(': "Raising exception to dump workspace files",
+    r'assert\s+False\s*,\s*.*(inspect|workspace)': "Intentional assert failure to dump workspace context",
+    r'sys\.stderr\.write\s*\(.*open\s*\(': "Writing file content to stderr for log leakage",
+    r'FILE_CONTENT_START': "Context dumping string markers detected",
+}
+
+
+# ---------------------------------------------------------------------------
+# Category 2: cleanup primitives. Blocked in normal mode, allowed in RSI.
+# ---------------------------------------------------------------------------
+CLEANUP_PATTERNS = {
+    r'os\.remove\s*\(': "os.remove()",
+    r'os\.unlink\s*\(': "os.unlink()",
+    r'os\.rmdir\s*\(': "os.rmdir()",
+    r'os\.removedirs\s*\(': "os.removedirs()",
+    r'shutil\.rmtree\s*\(': "shutil.rmtree()",
+    r'\.unlink\s*\(': "Path.unlink()",
+    r'shutil\.move\s*\(.*\.{2}[/\\]': "shutil.move with parent directory traversal",
+    r'os\.rename\s*\(.*\.{2}[/\\]': "os.rename with parent directory traversal",
+}
+
+
+def _strip_comments_only(code: str) -> str:
+    """Strip single-line and block comments. Triple-quoted strings intact."""
+    cleaned = _BLOCK_COMMENT_RE.sub('', code)
+    lines = []
+    for line in cleaned.splitlines():
+        if _LINE_COMMENT_RE.match(line):
+            continue
+        lines.append(_TRAILING_COMMENT_RE.sub('', line))
+    return "\n".join(lines)
+
+
+def _regex_scan(code: str, patterns: dict) -> List[str]:
+    cleaned = _strip_comments_only(code)
+    hits = []
+    for pattern, label in patterns.items():
+        if re.search(pattern, cleaned, re.IGNORECASE | re.DOTALL):
+            hits.append(label)
+    return hits
+
+
+_DANGEROUS_CALL_NAMES = {"eval", "exec", "compile", "__import__"}
+
+# AST-level dangerous call chains. Split into the same two categories.
+_ALWAYS_BLOCKED_ATTR_CHAINS = {
+    ("os", "system"): "os.system()",
+    ("os", "popen"): "os.popen()",
+    ("os", "execl"): "os.exec*() family",
+    ("os", "execle"): "os.exec*() family",
+    ("os", "execlp"): "os.exec*() family",
+    ("os", "execlpe"): "os.exec*() family",
+    ("os", "execv"): "os.exec*() family",
+    ("os", "execve"): "os.exec*() family",
+    ("os", "execvp"): "os.exec*() family",
+    ("os", "execvpe"): "os.exec*() family",
+    ("os", "spawnl"): "os.spawn*() family",
+    ("os", "spawnv"): "os.spawn*() family",
+    ("subprocess", "run"): "subprocess call",
+    ("subprocess", "Popen"): "subprocess call",
+    ("subprocess", "call"): "subprocess call",
+    ("subprocess", "check_output"): "subprocess call",
+    ("subprocess", "check_call"): "subprocess call",
+    ("pickle", "load"): "pickle.load()/loads() (arbitrary deserialization)",
+    ("pickle", "loads"): "pickle.load()/loads() (arbitrary deserialization)",
+    ("marshal", "load"): "marshal.load()/loads()",
+    ("marshal", "loads"): "marshal.load()/loads()",
+    ("pty", "spawn"): "pty.spawn()",
+    ("ctypes", "CDLL"): "ctypes native code loading",
+    ("ctypes", "WinDLL"): "ctypes native code loading",
+}
+
+_CLEANUP_ATTR_CHAINS = {
+    ("os", "remove"): "os.remove()",
+    ("os", "unlink"): "os.unlink()",
+    ("os", "rmdir"): "os.rmdir()",
+    ("os", "removedirs"): "os.removedirs()",
+    ("shutil", "rmtree"): "shutil.rmtree()",
+}
+
+_GETATTR_WATCHED_MODULES = {"os", "subprocess", "shutil", "pickle", "marshal", "ctypes"}
+
+
+def _attr_chain(node: ast.AST) -> Tuple[str, ...]:
+    parts = []
+    while isinstance(node, ast.Attribute):
+        parts.append(node.attr)
+        node = node.value
+    if isinstance(node, ast.Name):
+        parts.append(node.id)
+    return tuple(reversed(parts))
+
+
+def _ast_scan(code: str, allow_cleanup: bool) -> List[str]:
+    try:
+        tree = ast.parse(code)
+    except (SyntaxError, ValueError):
+        return []
+
+    hits = []
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call):
+            continue
+        func = node.func
+
+        if isinstance(func, ast.Name):
+            if func.id in _DANGEROUS_CALL_NAMES:
+                hits.append(f"{func.id}() (AST)")
+                continue
+            if func.id == "getattr" and node.args:
+                first_arg = node.args[0]
+                if isinstance(first_arg, ast.Name) and first_arg.id in _GETATTR_WATCHED_MODULES:
+                    hits.append(f"dynamic getattr({first_arg.id}, ...) - possible obfuscated call (AST)")
+                    continue
+
+        if isinstance(func, ast.Attribute):
+            chain = _attr_chain(func)
+            if len(chain) >= 2:
+                key = (chain[-2], chain[-1])
+                if key in _ALWAYS_BLOCKED_ATTR_CHAINS:
+                    hits.append(f"{_ALWAYS_BLOCKED_ATTR_CHAINS[key]} (AST)")
+                elif not allow_cleanup and key in _CLEANUP_ATTR_CHAINS:
+                    hits.append(f"{_CLEANUP_ATTR_CHAINS[key]} (AST)")
+
+    return hits
+
+
+def find_dangerous_code(code: str, allow_cleanup: bool = False) -> list:
+    """
+    Detect dangerous code patterns.
+
+    Args:
+        code: source text to scan.
+        allow_cleanup: if True, file deletion primitives (unlink, rmtree,
+            rmdir, remove) are permitted. Used in RSI mode where the
+            framework itself contains legitimate cleanup code.
+
+    Returns a deduplicated list of human-readable labels.
+    """
+    if not isinstance(code, str) or not code.strip():
+        return []
+
+    hits = []
+    hits.extend(_regex_scan(code, ALWAYS_BLOCKED_PATTERNS))
+    if not allow_cleanup:
+        hits.extend(_regex_scan(code, CLEANUP_PATTERNS))
+    hits.extend(_ast_scan(code, allow_cleanup))
+
+    seen = set()
+    deduped = []
+    for h in hits:
+        if h not in seen:
+            seen.add(h)
+            deduped.append(h)
+    return deduped
+
+
+def is_safe(code: str, allow_cleanup: bool = False) -> bool:
+    return len(find_dangerous_code(code, allow_cleanup=allow_cleanup)) == 0

tools/session_adr.py

@@ -0,0 +1,311 @@
+"""
+tools/session_adr.py
+---------------------
+Compressed session history (Architecture Decision Records) for VegeX.
+
+Storage layout:
+  .vegex/session_adr.jsonl         append-only, one JSON object per line
+  .vegex/session_adr.compact.json  derived, written periodically
+
+Raw JSONL is the source of truth. Compact JSON is regenerated from the raw
+when the raw file exceeds RAW_COMPACT_THRESHOLD lines, or on explicit call.
+Deleting the compact file is safe — it will be rebuilt on next read.
+
+Public API:
+  - ADREntry (dataclass)
+  - SessionADR(root).append(entry)
+  - SessionADR(root).get_recent_summary(limit=5, token_budget=300) -> str
+  - SessionADR(root).get_recent_entries(limit=20) -> List[ADREntry]
+  - SessionADR(root).compact(keep_raw=200)
+  - SessionADR(root).stats()
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import tempfile
+import time
+import uuid
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional
+
+logger = logging.getLogger(__name__)
+
+# Constants
+RAW_COMPACT_THRESHOLD = 400   # compact when JSONL exceeds this many lines
+DEFAULT_KEEP_RAW = 200
+SCHEMA_VERSION = "1.0.0"
+
+# Rough token estimator. Deliberately conservative (4 chars/token) with an
+# additional 10% safety margin applied at format time. Real tokenizers vary
+# by ±20% across providers — this is for budgeting, not billing.
+_CHARS_PER_TOKEN = 4
+_BUDGET_SAFETY_MARGIN = 0.9
+
+
+# ---------------------------------------------------------------------------
+# ADR entry
+# ---------------------------------------------------------------------------
+@dataclass
+class ADREntry:
+    session_id: str
+    cycle: int
+    timestamp: str  # ISO-8601 UTC
+    action: str     # "research" | "edit" | "review" | "refactor" | "bugfix" | "promote"
+    outcome: str    # "accepted" | "rejected" | "reverted" | "info"
+    files_touched: List[str] = field(default_factory=list)
+    contract_changes: List[str] = field(default_factory=list)
+    decisions: List[str] = field(default_factory=list)
+    duration_ms: int = 0
+    extra: Dict[str, Any] = field(default_factory=dict)
+
+    def to_json_line(self) -> str:
+        return json.dumps(asdict(self), ensure_ascii=False)
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> "ADREntry":
+        return cls(
+            session_id=str(d.get("session_id", "")),
+            cycle=int(d.get("cycle", 0)),
+            timestamp=str(d.get("timestamp", "")),
+            action=str(d.get("action", "unknown")),
+            outcome=str(d.get("outcome", "info")),
+            files_touched=list(d.get("files_touched", []) or []),
+            contract_changes=list(d.get("contract_changes", []) or []),
+            decisions=list(d.get("decisions", []) or []),
+            duration_ms=int(d.get("duration_ms", 0) or 0),
+            extra=dict(d.get("extra", {}) or {}),
+        )
+
+
+# ---------------------------------------------------------------------------
+# Manager
+# ---------------------------------------------------------------------------
+class SessionADR:
+    def __init__(self, project_root: str) -> None:
+        self.root = Path(project_root).resolve()
+        self.dir = self.root / ".vegex"
+        self.dir.mkdir(parents=True, exist_ok=True)
+        self.raw_path = self.dir / "session_adr.jsonl"
+        self.compact_path = self.dir / "session_adr.compact.json"
+
+    # ------------------------------------------------------------------ #
+    # Append
+    # ------------------------------------------------------------------ #
+    def append(self, entry: ADREntry) -> None:
+        """Append one entry. Uses O_APPEND for atomicity on POSIX."""
+        line = entry.to_json_line() + "\n"
+        try:
+            fd = os.open(self.raw_path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o644)
+            try:
+                os.write(fd, line.encode("utf-8"))
+            finally:
+                os.close(fd)
+        except Exception as exc:
+            logger.warning("ADR append failed: %s", exc)
+            return
+
+        # Opportunistic compaction
+        try:
+            if self._line_count() >= RAW_COMPACT_THRESHOLD:
+                self.compact()
+        except Exception as exc:
+            logger.warning("ADR compact check failed: %s", exc)
+
+    def _line_count(self) -> int:
+        try:
+            with open(self.raw_path, "rb") as f:
+                return sum(1 for _ in f)
+        except FileNotFoundError:
+            return 0
+
+    # ------------------------------------------------------------------ #
+    # Read
+    # ------------------------------------------------------------------ #
+    def get_recent_entries(self, limit: int = 20) -> List[ADREntry]:
+        """Read the tail of the raw JSONL. Corrupt lines are skipped."""
+        if not self.raw_path.exists():
+            return []
+        try:
+            with open(self.raw_path, "r", encoding="utf-8") as f:
+                lines = f.readlines()
+        except Exception as exc:
+            logger.warning("ADR read failed: %s", exc)
+            return []
+
+        entries: List[ADREntry] = []
+        for line in lines[-limit:]:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entries.append(ADREntry.from_dict(json.loads(line)))
+            except Exception:
+                continue
+        return entries
+
+    def get_recent_summary(
+        self,
+        limit: int = 5,
+        token_budget: int = 300,
+    ) -> str:
+        """
+        Render the last `limit` entries as a compact LLM-injectable string.
+
+        Returns empty string if there are no entries.
+        """
+        entries = self.get_recent_entries(limit=limit)
+        if not entries:
+            return ""
+
+        char_budget = int(token_budget * _CHARS_PER_TOKEN * _BUDGET_SAFETY_MARGIN)
+
+        # Format newest-first, drop entries when we exceed budget
+        rendered: List[str] = ["## RECENT SESSION DECISIONS", ""]
+        for e in reversed(entries):
+            block = self._format_entry(e)
+            if sum(len(l) + 1 for l in rendered) + len(block) > char_budget:
+                break
+            rendered.append(block)
+
+        if len(rendered) <= 2:
+            return ""  # nothing fit
+        return "\n".join(rendered)
+
+    @staticmethod
+    def _format_entry(e: ADREntry) -> str:
+        ts_short = e.timestamp[:16].replace("T", " ") if e.timestamp else "?"
+        lines = [f"[cycle {e.cycle} | {ts_short}] {e.action} → {e.outcome}"]
+        if e.files_touched:
+            lines.append(f"  files: {', '.join(e.files_touched[:4])}")
+        if e.contract_changes:
+            lines.append(f"  contract: {'; '.join(e.contract_changes[:3])}")
+        if e.decisions:
+            for d in e.decisions[:2]:
+                short = d if len(d) <= 140 else d[:137] + "..."
+                lines.append(f"  - {short}")
+        return "\n".join(lines) + "\n"
+
+    # ------------------------------------------------------------------ #
+    # Compaction
+    # ------------------------------------------------------------------ #
+    def compact(self, keep_raw: int = DEFAULT_KEEP_RAW) -> Dict[str, Any]:
+        """
+        Prune the raw JSONL to the last `keep_raw` entries, and write a
+        compact summary of every entry ever seen. Idempotent.
+        """
+        if not self.raw_path.exists():
+            return {"status": "no_raw"}
+
+        try:
+            with open(self.raw_path, "r", encoding="utf-8") as f:
+                all_lines = [l for l in f if l.strip()]
+        except Exception as exc:
+            return {"status": "read_error", "error": str(exc)}
+
+        total = len(all_lines)
+        kept_lines = all_lines[-keep_raw:] if total > keep_raw else all_lines
+
+        # All entries (for compact json)
+        all_entries: List[ADREntry] = []
+        for line in all_lines:
+            try:
+                all_entries.append(ADREntry.from_dict(json.loads(line)))
+            except Exception:
+                continue
+
+        # Pruned raw
+        try:
+            fd, tmp = tempfile.mkstemp(dir=str(self.dir), suffix=".jsonl.tmp")
+            try:
+                with os.fdopen(fd, "w", encoding="utf-8") as f:
+                    f.writelines(kept_lines)
+                os.replace(tmp, self.raw_path)
+            except Exception:
+                try:
+                    os.unlink(tmp)
+                except OSError:
+                    pass
+                raise
+        except Exception as exc:
+            return {"status": "write_raw_error", "error": str(exc)}
+
+        # Compact file
+        compact_data = {
+            "schema_version": SCHEMA_VERSION,
+            "compacted_at": _utc_iso(),
+            "total_entries_ever": total,
+            "kept_in_raw": len(kept_lines),
+            "entries": [asdict(e) for e in all_entries],
+        }
+        try:
+            fd, tmp = tempfile.mkstemp(dir=str(self.dir), suffix=".json.tmp")
+            try:
+                with os.fdopen(fd, "w", encoding="utf-8") as f:
+                    json.dump(compact_data, f, ensure_ascii=False, indent=2)
+                os.replace(tmp, self.compact_path)
+            except Exception:
+                try:
+                    os.unlink(tmp)
+                except OSError:
+                    pass
+                raise
+        except Exception as exc:
+            return {"status": "write_compact_error", "error": str(exc)}
+
+        return {
+            "status": "ok",
+            "total_entries": total,
+            "kept_in_raw": len(kept_lines),
+            "pruned": total - len(kept_lines),
+        }
+
+    # ------------------------------------------------------------------ #
+    # Stats / lifecycle
+    # ------------------------------------------------------------------ #
+    def stats(self) -> Dict[str, Any]:
+        return {
+            "raw_path": str(self.raw_path),
+            "raw_exists": self.raw_path.exists(),
+            "raw_lines": self._line_count(),
+            "compact_exists": self.compact_path.exists(),
+            "compact_threshold": RAW_COMPACT_THRESHOLD,
+        }
+
+
+# ---------------------------------------------------------------------------
+# Entry builder (convenience)
+# ---------------------------------------------------------------------------
+def build_entry(
+    session_id: str,
+    cycle: int,
+    action: str,
+    outcome: str,
+    files_touched: Optional[Iterable[str]] = None,
+    contract_changes: Optional[Iterable[str]] = None,
+    decisions: Optional[Iterable[str]] = None,
+    duration_ms: int = 0,
+    extra: Optional[Dict[str, Any]] = None,
+) -> ADREntry:
+    return ADREntry(
+        session_id=session_id or str(uuid.uuid4()),
+        cycle=int(cycle),
+        timestamp=_utc_iso(),
+        action=action,
+        outcome=outcome,
+        files_touched=list(files_touched or []),
+        contract_changes=list(contract_changes or []),
+        decisions=list(decisions or []),
+        duration_ms=int(duration_ms),
+        extra=dict(extra or {}),
+    )
+
+
+def _utc_iso() -> str:
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+
+__all__ = ["ADREntry", "SessionADR", "build_entry"]