patchi 0.7.5__py3-none-any.whl

patchi/core/scan_bus.py

@@ -0,0 +1,156 @@
+"""
+ScanBus §MASTER_ROADMAP Scan Bus — FileCorpus parse-once + shards + FindingBus.
+
+Shards FileInfos for parallel agents without re-walking filesystem.
+FindingBus is thread-safe queue for findings (replaces per-agent list merging).
+QueueRunner batches agent classes across shard workers sharing one corpus.
+
+File-sharding per agent activates only once agents honor `AgentInput.scope`
+(4 do today: sast, security_config, unit_test, live_v2) — until then every
+agent receives the full input and the bus is the single merge point.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import queue
+import threading
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from patchi.core.brain.file_corpus import FileCorpus
+from patchi.core.brain.scanner import FileInfo
+
+
+@dataclass
+class ScanShard:
+    idx: int
+    files: list[FileInfo]
+    total_shards: int
+
+
+class ScanBus:
+    """Shard-aware FileCorpus wrapper. Build once, reuse across agents."""
+
+    def __init__(self, root: Path, shard_count: int = 4):
+        self.root = root
+        self.corpus = FileCorpus(root)
+        self.shard_count = max(1, shard_count)
+        self._shards: list[ScanShard] | None = None
+
+    def build(self) -> None:
+        # Force corpus build
+        _ = self.corpus.entries
+
+    def shards(self, file_infos: list[FileInfo] | None = None) -> list[ScanShard]:
+        if file_infos is None:
+            # Build from corpus if available, else scan
+            from patchi.core.brain.scanner import FileScanner
+            file_infos = FileScanner(self.root, corpus=self.corpus).scan()
+        n = self.shard_count
+        # shard by hash of path for even distribution
+        buckets: list[list[FileInfo]] = [[] for _ in range(n)]
+        # Deterministic sharding via MD5 (Python's hash() is randomized across processes)
+        for fi in file_infos:
+            digest = hashlib.md5(fi.path.encode()).digest()
+            shard_id = int.from_bytes(digest[:4], "little") % n
+            buckets[shard_id].append(fi)
+        return [ScanShard(idx=i, files=buckets[i], total_shards=n) for i in range(n)]
+
+    def shard_paths(self, file_infos: list[FileInfo] | None = None) -> list[list[str]]:
+        """Relative path lists per shard — future per-agent scope input."""
+        return [[fi.path for fi in s.files] for s in self.shards(file_infos)]
+
+
+@dataclass
+class FindingBus:
+    """Thread-safe bus for findings. Agents push, coordinator drains."""
+
+    _q: queue.Queue = field(default_factory=queue.Queue)
+    _lock: threading.Lock = field(default_factory=threading.Lock)
+    _count: int = 0
+
+    def publish(self, finding: Any) -> None:
+        self._q.put(finding)
+        with self._lock:
+            self._count += 1
+
+    def drain(self) -> list[Any]:
+        out = []
+        while not self._q.empty():
+            try:
+                out.append(self._q.get_nowait())
+            except queue.Empty:
+                break
+        return out
+
+    @property
+    def count(self) -> int:
+        with self._lock:
+            return self._count
+
+
+class QueueRunner:
+    """Shard-worked parallel runner over agent classes.
+
+    One ScanBus (single FileCorpus build) is shared by all workers. Agent
+    classes are fanned out over `shard_count` threads with the same per-agent
+    timeout semantics as the coordinator's thread pool. Every finished
+    result's findings are auto-published to the FindingBus — agents never
+    touch the bus directly — so `drain()` is the single merge point.
+    """
+
+    def __init__(self, bus: ScanBus, finding_bus: FindingBus | None = None):
+        self.scan_bus = bus
+        self.finding_bus = finding_bus or FindingBus()
+        self.scan_bus.build()
+
+    @property
+    def shards(self) -> list[ScanShard]:
+        return self.scan_bus.shards()
+
+    def run(
+        self,
+        agent_classes: list,
+        run_one: Callable[[Any], Any],
+        on_done: Callable[[Any], None],
+        default_timeout: int = 120,
+    ) -> list:
+        """Run every agent class, publish findings, call on_done per result."""
+        from concurrent.futures import Future
+
+        from patchi.core.agents.base import AgentGroup, AgentResult, AgentStatus
+
+        results: list = []
+        futures: dict[Future, str] = {}
+        timeout_map: dict[Future, int] = {}
+        with ThreadPoolExecutor(max_workers=self.scan_bus.shard_count) as ex:
+            for cls in agent_classes:
+                fut = ex.submit(self._run_and_publish, run_one, cls)
+                futures[fut] = getattr(cls, "name", str(cls))
+                timeout_map[fut] = getattr(cls, "timeout", default_timeout)
+            for fut in as_completed(futures):
+                try:
+                    result = fut.result(timeout=timeout_map.get(fut, default_timeout))
+                except Exception as e:
+                    result = AgentResult(
+                        agent_name=futures[fut],
+                        agent_group=AgentGroup.SCANNER,
+                        status=AgentStatus.FAILED,
+                        errors=[f"QueueRunner caught: {e}"],
+                    )
+                on_done(result)
+                results.append(result)
+        return results
+
+    def _run_and_publish(self, run_one: Callable[[Any], Any], cls: Any) -> Any:
+        result = run_one(cls)
+        for f in getattr(result, "findings", []):
+            self.finding_bus.publish(f)
+        return result
+
+    def drain(self) -> list[Any]:
+        return self.finding_bus.drain()

patchi/core/security/__init__.py

@@ -0,0 +1 @@
+

patchi/core/security/ai_validator.py

@@ -0,0 +1,462 @@
+"""
+AI Validator — the core harness engine that eliminates false positives.
+
+Instead of asking "is this a vulnerability?" (which produces noise), the
+validator asks three chained questions:
+
+1. CODE READING: "Read the surrounding function. What does this code actually do?"
+2. DATA FLOW:    "Where does the tainted input come from and where does it go?"
+3. EXPLOITABILITY: "Given the app context, is an attacker able to reach this?"
+
+Each question builds on the previous answer, creating a reasoning chain that
+produces a structured verdict with confidence and evidence.
+
+The output is deterministic enough for the ConfidenceGate to score, but rich
+enough for human review when needed.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+_log = logging.getLogger("patchi.security.ai_validator")
+
+# ── Result types ─────────────────────────────────────────────────────────────
+
+
+@dataclass
+class ReasoningStep:
+    """One step in the chain-of-reasoning."""
+
+    question: str
+    analysis: str
+    confidence: float  # 0.0 - 1.0
+
+    def to_dict(self) -> dict:
+        return {
+            "question": self.question,
+            "analysis": self.analysis,
+            "confidence": self.confidence,
+        }
+
+
+@dataclass
+class ValidationResult:
+    """Full validation result with chain-of-reasoning."""
+
+    finding_id: str
+    is_true_positive: bool
+    confidence: float  # 0.0 - 1.0
+    reasoning: list[ReasoningStep] = field(default_factory=list)
+    explanation: str = ""
+    fix_suggestion: str = ""
+    test_suggestion: str = ""
+    severity_adjustment: str = ""  # "increase", "decrease", ""
+    tokens_used: int = 0
+    model: str = ""
+
+    @property
+    def summary(self) -> str:
+        verdict = "TRUE POSITIVE" if self.is_true_positive else "FALSE POSITIVE"
+        return f"{verdict} (confidence {self.confidence:.0%})"
+
+    def to_dict(self) -> dict:
+        return {
+            "finding_id": self.finding_id,
+            "is_true_positive": self.is_true_positive,
+            "confidence": self.confidence,
+            "reasoning": [s.to_dict() for s in self.reasoning],
+            "explanation": self.explanation,
+            "fix_suggestion": self.fix_suggestion,
+            "test_suggestion": self.test_suggestion,
+            "severity_adjustment": self.severity_adjustment,
+            "tokens_used": self.tokens_used,
+            "model": self.model,
+        }
+
+
+# ── Heuristic pre-filters (no AI needed) ─────────────────────────────────────
+
+_FP_PATTERNS: list[tuple[str, str, float]] = [
+    # (file pattern, finding type, confidence_penalty)
+    (".lock", "dependency", 0.8),
+    ("package-lock.json", "dependency", 0.9),
+    ("yarn.lock", "dependency", 0.9),
+    ("Pipfile.lock", "dependency", 0.9),
+    (".min.js", "code_quality", 0.7),
+    (".min.css", "code_quality", 0.7),
+    ("__pycache__", "code_quality", 0.9),
+    (".generated.", "code_quality", 0.8),
+    (".pb.", "code_quality", 0.7),
+    ("_pb2.py", "code_quality", 0.8),
+    (".proto", "code_quality", 0.6),
+]
+
+_TEST_PATTERNS = [
+    "test_",
+    "_test.",
+    "spec_",
+    "_spec.",
+    "tests/",
+    "test/",
+    "__tests__/",
+    "fixture",
+    "mock_",
+    "fake_",
+    "conftest",
+]
+
+_GENERATED_MARKERS = [
+    "generated by",
+    "auto-generated",
+    "auto generated",
+    "do not edit",
+    "do not modify",
+    "autogenerated",
+    "@generated",
+    "# noqa",
+    "// generated",
+    "code-generated",
+    "machine-generated",
+]
+
+
+def heuristic_pre_filter(finding: Any) -> tuple[str, float] | None:
+    """Quick heuristic check — returns (reason, penalty) if likely FP, None otherwise.
+
+    This runs BEFORE the AI validator to skip obviously non-exploitable
+    findings, saving tokens and improving signal.
+    """
+    ffile = getattr(finding, "file", "") or ""
+    ftype = getattr(finding, "type", "") or ""
+    snippet = getattr(finding, "code_snippet", "") or ""
+
+    # 1. File pattern FP
+    ffile_lower = ffile.lower()
+    for pattern, ftype_match, penalty in _FP_PATTERNS:
+        if pattern in ffile_lower and (not ftype_match or ftype_match in ftype.lower()):
+            return f"File pattern match: {pattern}", penalty
+
+    # 2. Test file FP
+    if any(tp in ffile_lower for tp in _TEST_PATTERNS):
+        # Test files with injection findings are often intentional test payloads
+        if any(kw in ftype.lower() for kw in ["injection", "xss", "command", "traversal"]):
+            return "Test file with injection pattern (likely test payload)", 0.6
+
+    # 3. Generated code FP
+    snippet_lower = snippet.lower()
+    if any(marker in snippet_lower for marker in _GENERATED_MARKERS):
+        return "Generated code marker detected", 0.7
+
+    # 4. TODO/FIXME with security mention = not a real vuln yet
+    if "todo" in snippet_lower and any(
+        kw in ftype.lower() for kw in ["security", "auth", "crypto"]
+    ):
+        return "TODO/FIXME marker — not implemented yet", 0.5
+
+    return None
+
+
+# ── AI Validator ──────────────────────────────────────────────────────────────
+
+_SYSTEM_PROMPT = """\
+You are Patchi's security harness validator. You receive a security finding with its full file context and must determine if it is a TRUE POSITIVE or FALSE POSITIVE.
+
+You must produce a chain-of-reasoning in 3 steps:
+
+STEP 1 — CODE READING:
+Read the function/method containing the finding. What does the code actually do? Is the pattern suspicious or benign in context?
+
+STEP 2 — DATA FLOW:
+Trace the data flow. Where does user-controlled input come from? Is there any sanitization between source and sink? Is the path realistic for an attacker?
+
+STEP 3 — EXPLOITABILITY:
+Given the application context (routes, middleware, auth), can an attacker actually reach this code? What is the real-world impact?
+
+After the 3 steps, output a structured JSON verdict:
+{
+  "is_true_positive": bool,
+  "confidence": float (0.0-1.0),
+  "reasoning": [
+    {"question": "...", "analysis": "...", "confidence": float},
+    {"question": "...", "analysis": "...", "confidence": float},
+    {"question": "...", "analysis": "...", "confidence": float}
+  ],
+  "explanation": "one-sentence verdict summary",
+  "fix_suggestion": "concrete fix code or empty string",
+  "test_suggestion": "test case to verify fix or empty string",
+  "severity_adjustment": "increase" | "decrease" | ""
+}
+
+Rules:
+- Be CONSERVATIVE: if you are not sure, mark as true positive with lower confidence
+- Never dismiss a real vulnerability to reduce noise
+- Always explain WHY it is/isn't exploitable
+- Generate minimal, targeted fixes (not architectural rewrites)
+- The fix must preserve existing functionality
+"""
+
+_CACHE_TTL = 86400  # 24h
+
+
+class AIValidator:
+    """AI-powered true/false positive validator with chain-of-reasoning.
+
+    Usage::
+
+        validator = AIValidator(root)
+        result = validator.validate(finding, file_content="...")
+        if result.confidence >= 0.8 and not result.is_true_positive:
+            # High-confidence false positive — safe to discard
+            pass
+    """
+
+    def __init__(self, root: Path, config: dict | None = None):
+        self.root = root
+        self.config = config or {}
+        self._cache: dict[str, tuple[float, ValidationResult]] = {}
+        self._stats = {"validated": 0, "cached": 0, "heuristic_skip": 0}
+
+    def validate(
+        self,
+        finding: Any,
+        file_content: str | None = None,
+        route_context: dict | None = None,
+        use_heuristics: bool = True,
+    ) -> ValidationResult:
+        """Validate a single finding.
+
+        Args:
+            finding: A Finding or GatedFinding with .file, .line, .type, .message,
+                     .code_snippet, .severity, .cwe attributes.
+            file_content: Full source file content (read if not provided).
+            route_context: Dict with 'routes', 'middleware', 'auth' for context.
+            use_heuristics: Run heuristic pre-filter before AI.
+
+        Returns:
+            ValidationResult with chain-of-reasoning.
+        """
+        fid = self._finding_id(finding)
+        self._stats["validated"] += 1
+
+        # Check cache
+        if fid in self._cache:
+            ts, cached = self._cache[fid]
+            import time
+
+            if time.time() - ts < _CACHE_TTL:
+                self._stats["cached"] += 1
+                return cached
+
+        # Heuristic pre-filter
+        if use_heuristics:
+            hp = heuristic_pre_filter(finding)
+            if hp:
+                reason, penalty = hp
+                sev = getattr(finding, "severity", None)
+                sev_val = sev.value if hasattr(sev, "value") else str(sev)
+                # Only auto-discard if severity is low AND penalty is high
+                if penalty >= 0.7 and sev_val in ("info", "low"):
+                    result = ValidationResult(
+                        finding_id=fid,
+                        is_true_positive=False,
+                        confidence=max(0.1, 1.0 - penalty),
+                        explanation=f"Heuristic filter: {reason}",
+                    )
+                    self._cache[fid] = (self._now(), result)
+                    self._stats["heuristic_skip"] += 1
+                    return result
+
+        # Read file content if not provided
+        if file_content is None:
+            file_content = self._read_file(getattr(finding, "file", ""))
+
+        # Build the 3-step prompt
+        prompt = self._build_prompt(finding, file_content, route_context)
+
+        # Call AI
+        result = self._call_ai(fid, prompt)
+        self._cache[fid] = (self._now(), result)
+        return result
+
+    def validate_batch(
+        self,
+        findings: list,
+        use_heuristics: bool = True,
+    ) -> list[ValidationResult]:
+        """Validate a batch of findings, using cache and heuristics."""
+        return [self.validate(f, use_heuristics=use_heuristics) for f in findings]
+
+    def get_stats(self) -> dict:
+        return dict(self._stats)
+
+    # ── Internal ──────────────────────────────────────────────────────────────
+
+    def _finding_id(self, finding: Any) -> str:
+        raw = (
+            f"{getattr(finding, 'file', '')}:"
+            f"{getattr(finding, 'line', 0)}:"
+            f"{getattr(finding, 'type', '')}:"
+            f"{getattr(finding, 'message', '')[:80]}"
+        )
+        return hashlib.sha256(raw.encode()).hexdigest()[:16]
+
+    def _read_file(self, filepath: str) -> str:
+        if not filepath:
+            return ""
+        try:
+            full = self.root / filepath
+            if full.is_file():
+                return full.read_text(encoding="utf-8", errors="replace")[:8000]
+        except Exception as _exc:
+            _log.warning('_read_file failed: %s', _exc)
+        return ""
+
+    def _build_prompt(
+        self,
+        finding: Any,
+        file_content: str,
+        route_context: dict | None,
+    ) -> str:
+        ffile = getattr(finding, "file", "")
+        fline = getattr(finding, "line", 0)
+        ftype = getattr(finding, "type", "")
+        message = getattr(finding, "message", "")
+        snippet = getattr(finding, "code_snippet", "")
+        severity = getattr(finding, "severity", "")
+        sev_val = severity.value if hasattr(severity, "value") else str(severity)
+        cwe = getattr(finding, "cwe", "")
+        confirmed_by = getattr(finding, "confirmed_by", [])
+
+        # Extract the function around the finding line
+        context_block = self._extract_context_block(file_content, fline)
+
+        parts = [
+            f"FINDING: {ftype} at {ffile}:{fline}",
+            f"Severity: {sev_val}",
+            f"CWE: {cwe}" if cwe else "",
+            f"Message: {message}",
+            f"Confirmed by: {', '.join(confirmed_by) if confirmed_by else 'N/A'}",
+            "",
+            "--- CODE SNIPPET (from finding) ---",
+            snippet if snippet else "(no snippet available)",
+            "",
+            f"--- SURROUNDING FUNCTION (lines {max(1, fline - 20)}-{fline + 20}) ---",
+            context_block if context_block else "(could not extract context)",
+        ]
+
+        if route_context:
+            routes = route_context.get("routes", [])
+            auth = route_context.get("auth", "")
+            if routes:
+                parts.append(f"\n--- ROUTE CONTEXT ---\nAuth: {auth}")
+                for r in routes[:5]:
+                    if isinstance(r, dict):
+                        parts.append(f"  {r.get('method', '?')} {r.get('path', '?')}")
+
+        parts.append("\nAnalyze this finding through the 3-step reasoning chain.")
+        return "\n".join(p for p in parts if p)
+
+    def _extract_context_block(self, file_content: str, line: int) -> str:
+        """Extract the function/block surrounding the finding line."""
+        if not file_content or line <= 0:
+            return ""
+        lines = file_content.splitlines()
+        if line > len(lines):
+            return ""
+
+        # Find function start (go up to find `def` or `function` or `class`)
+        start = max(0, line - 30)
+        for i in range(line - 1, max(0, line - 40), -1):
+            line_text = lines[i].strip()
+            if (
+                line_text.startswith("def ")
+                or line_text.startswith("function ")
+                or line_text.startswith("class ")
+            ):
+                start = i
+                break
+            if line_text.startswith("@") and i + 1 < len(lines):
+                start = i
+                break
+
+        # Find function end (go down to find next def/class at same indent)
+        end = min(len(lines), line + 30)
+        base_indent = (
+            len(lines[line - 1]) - len(lines[line - 1].lstrip()) if line <= len(lines) else 0
+        )
+        for i in range(line, min(len(lines), line + 50)):
+            cur_line = lines[i]
+            if cur_line.strip() and not cur_line.strip().startswith("#"):
+                indent = len(cur_line) - len(cur_line.lstrip())
+                if indent <= base_indent and (
+                    cur_line.strip().startswith("def ")
+                    or cur_line.strip().startswith("class ")
+                    or cur_line.strip().startswith("async def ")
+                ):
+                    end = i
+                    break
+
+        # Format with line numbers
+        block_lines = lines[start:end]
+        formatted = []
+        for i, line_text in enumerate(block_lines, start=start + 1):
+            marker = " >>>" if i == line else "    "
+            formatted.append(f"{i:4d}{marker} {line_text}")
+        return "\n".join(formatted)
+
+    def _call_ai(self, fid: str, prompt: str) -> ValidationResult:
+        """Make the AI call and parse the structured response."""
+        try:
+            from patchi.core.ai.client import call_ai_structured
+
+            response = call_ai_structured(
+                prompt=prompt,
+                system=_SYSTEM_PROMPT,
+                output_model=dict,
+            )
+
+            if response and isinstance(response, dict):
+                reasoning_steps = []
+                for step in response.get("reasoning", []):
+                    if isinstance(step, dict):
+                        reasoning_steps.append(
+                            ReasoningStep(
+                                question=step.get("question", ""),
+                                analysis=step.get("analysis", ""),
+                                confidence=float(step.get("confidence", 0.5)),
+                            )
+                        )
+
+                return ValidationResult(
+                    finding_id=fid,
+                    is_true_positive=bool(response.get("is_true_positive", True)),
+                    confidence=float(response.get("confidence", 0.5)),
+                    reasoning=reasoning_steps,
+                    explanation=response.get("explanation", ""),
+                    fix_suggestion=response.get("fix_suggestion", ""),
+                    test_suggestion=response.get("test_suggestion", ""),
+                    severity_adjustment=response.get("severity_adjustment", ""),
+                    tokens_used=int(response.get("tokens_used", 0)),
+                    model=response.get("model", "unknown"),
+                )
+        except Exception as e:
+            _log.debug("AIValidator._call_ai failed: %s", e)
+
+        # Offline fallback — assume true positive with low confidence
+        return ValidationResult(
+            finding_id=fid,
+            is_true_positive=True,
+            confidence=0.3,
+            explanation="AI unavailable — assumed true positive for safety",
+        )
+
+    @staticmethod
+    def _now() -> float:
+        import time
+
+        return time.time()