codevanta 0.1.0__py3-none-any.whl

codevanta/analysis/decompilers/base.py

@@ -0,0 +1,278 @@
+"""Backend-neutral primitives shared by every decompiler provider (phase 3).
+
+The package :mod:`codevanta.analysis.decompilers` lets CODEVANTA use a *real*
+decompiler when one happens to be installed and its own heuristic renderer
+otherwise. This module holds the parts that must never depend on an optional
+third-party package:
+
+* :class:`BackendAvailability` -- the answer to "can this backend run here, and
+  if not, why not". A probe never raises; a missing dependency becomes an
+  ``available=False`` record with a human-readable :attr:`BackendAvailability.reason`.
+* :class:`BackendResult` -- the backend-neutral rendering handed back to the CLI,
+  the MCP server and the report: ``text``, which backend produced it, the source
+  ``language``, the ``warnings`` collected on the way and the :class:`Evidence`
+  that justifies it. :meth:`BackendResult.to_dict` is JSON-safe.
+* :class:`DecompilerBackend` -- the structural protocol every provider
+  implements (``name``, ``requires``, ``availability()``, ``decompile(...)``).
+* :class:`NativeBackend` -- the fallback provider. It forwards to
+  :func:`codevanta.analysis.decompile.decompile_function` and can therefore
+  always answer, because that renderer only needs the required ``capstone``
+  dependency and the loaded binary metadata.
+* :func:`require_budget` -- the single validation path for
+  ``max_instructions`` / ``max_blocks``, so every backend rejects the same
+  out-of-range requests with the same :class:`AnalysisError` messages as the
+  native renderer.
+
+Heavy modules are imported lazily: this file imports only the standard library
+plus in-project modules that are already required to run CODEVANTA, and the
+optional backends (:mod:`codevanta.analysis.decompilers.angr_backend`,
+:mod:`codevanta.analysis.decompilers.ghidra_backend`) import their dependency
+inside ``availability()`` / ``decompile()`` only.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import Any, Protocol, runtime_checkable
+
+from codevanta.analysis.decompile import (
+    DEFAULT_MAX_BLOCKS,
+    DEFAULT_MAX_INSTRUCTIONS,
+    MAX_MAX_BLOCKS,
+    MAX_MAX_INSTRUCTIONS,
+)
+from codevanta.analysis.decompile import (
+    decompile_function as render_pseudo_c,
+)
+from codevanta.core.loader import BinaryInfo
+from codevanta.core.report import AnalysisError
+from codevanta.evidence.models import Evidence
+
+__all__ = [
+    "BackendAvailability",
+    "BackendResult",
+    "DecompilerBackend",
+    "NativeBackend",
+    "require_budget",
+]
+
+
+#: Source language of every rendering this layer produces. All three backends
+#: emit C (the native one emits *heuristic* pseudo-C), so the tag is shared.
+@dataclass(frozen=True)
+class BackendAvailability:
+    """Whether one decompiler backend can run in this environment.
+
+    Attributes:
+        name: Backend identifier, one of ``ghidra``, ``angr`` or ``native``.
+        available: ``True`` only when :meth:`DecompilerBackend.decompile` can be
+            expected to run without installing anything else.
+        reason: Human-readable explanation. Always non-empty: for an available
+            backend it states what was detected, for an unavailable one it names
+            the missing piece and, where possible, how to install it.
+        requires: Packages or installations the backend needs, in the order they
+            are probed. Empty for the built-in backend.
+    """
+
+    name: str
+    available: bool
+    reason: str
+    requires: tuple[str, ...]
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the probe for the JSON report."""
+        return {
+            "name": self.name,
+            "available": self.available,
+            "reason": self.reason,
+            "requires": list(self.requires),
+        }
+
+
+@dataclass(frozen=True)
+class BackendResult:
+    """One decompilation rendered by a backend.
+
+    Attributes:
+        text: The rendering itself. Never empty: a backend that produced nothing
+            still returns an explanatory ``/* ... */`` comment plus a warning.
+        backend: Name of the backend that produced ``text`` (not the one that was
+            requested, which matters for ``auto``).
+        language: Source language tag of ``text``; always :data:`LANGUAGE` today.
+        warnings: Everything the backend wants the caller to know: budget
+            overruns it could not prevent, fallbacks, empty renderings.
+        evidence: How the text was produced, including which backend and which
+            budgets were in effect.
+    """
+
+    text: str
+    backend: str
+    language: str
+    warnings: tuple[str, ...]
+    evidence: Evidence
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the rendering for the JSON report (JSON-safe values only)."""
+        return {
+            "text": self.text,
+            "backend": self.backend,
+            "language": self.language,
+            "warnings": list(self.warnings),
+            "evidence": self.evidence.to_dict(),
+        }
+
+
+@runtime_checkable
+class DecompilerBackend(Protocol):
+    """Structural interface every provider implements.
+
+    The protocol is ``runtime_checkable`` so callers can assert that a registry
+    entry really exposes ``name``, ``requires``, ``availability`` and
+    ``decompile`` (attribute presence only; signatures are checked statically).
+
+    Attributes:
+        name: Short backend identifier used by :func:`get_backend`.
+        requires: Hard requirements, reported verbatim by
+            :class:`BackendAvailability`.
+    """
+
+    name: str
+    requires: tuple[str, ...]
+
+    def availability(self) -> BackendAvailability:
+        """Probe the environment. Must never raise and never install anything."""
+        ...
+
+    def decompile(
+        self,
+        path: str | Path,
+        info: BinaryInfo,
+        address: int,
+        max_instructions: int = ...,
+        max_blocks: int = ...,
+    ) -> BackendResult:
+        """Render the function at ``address``.
+
+        Raises:
+            AnalysisError: When the backend is unavailable, when the request is
+                out of range, or when the backend itself fails. Never raises
+                ``ImportError`` for a missing optional dependency.
+        """
+        ...
+
+
+LANGUAGE = "c"
+
+#: Prefix of a warning line inside the native renderer's header comment block.
+_HEADER_WARNING_PREFIX = " * warning: "
+
+#: Prefix of the native renderer's "more warnings follow" summary line.
+_HEADER_WARNING_OVERFLOW = "(+"
+
+
+def require_budget(name: str, value: int, maximum: int) -> int:
+    """Validate one budget argument against its hard limit.
+
+    Mirrors the native renderer's validation so the provider layer and the
+    renderer reject the same requests.
+
+    Args:
+        name: Argument name, used in the error message.
+        value: Requested budget.
+        maximum: Largest accepted value.
+
+    Returns:
+        ``value``, unchanged, for convenient inlining.
+
+    Raises:
+        AnalysisError: When ``value`` is not an ``int`` or is out of range.
+    """
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise AnalysisError(f"{name} must be an int, not {type(value).__name__}")
+    if value < 1:
+        raise AnalysisError(f"{name} must be >= 1, got {value}")
+    if value > maximum:
+        raise AnalysisError(f"{name} too large ({value}); limit is {maximum}")
+    return value
+
+
+def _header_warnings(body: str) -> tuple[str, ...]:
+    """Lift the native renderer's ``warning:`` header lines out of the body."""
+    warnings: list[str] = []
+    for line in body.splitlines():
+        if line == " */":
+            break
+        if not line.startswith(_HEADER_WARNING_PREFIX):
+            continue
+        text = line[len(_HEADER_WARNING_PREFIX) :].strip()
+        if text and not text.startswith(_HEADER_WARNING_OVERFLOW):
+            warnings.append(text)
+    return tuple(warnings)
+
+
+class NativeBackend:
+    """The built-in heuristic pseudo-C renderer, always available.
+
+    The rendering itself lives in :mod:`codevanta.analysis.decompile`; this
+    provider only adapts its :class:`~codevanta.analysis.decompile.PseudoFunction`
+    to the backend-neutral :class:`BackendResult` (``body`` becomes ``text``) and
+    re-publishes the renderer's header warnings as structured warnings.
+    """
+
+    name: str = "native"
+    requires: tuple[str, ...] = ()
+
+    def availability(self) -> BackendAvailability:
+        """Report the built-in renderer; it needs no optional dependency."""
+        return BackendAvailability(
+            name=self.name,
+            available=True,
+            reason=(
+                "built-in mnemonic-level pseudo-C renderer; needs no optional "
+                "dependency and always runs"
+            ),
+            requires=self.requires,
+        )
+
+    def decompile(
+        self,
+        path: str | Path,
+        info: BinaryInfo,
+        address: int,
+        max_instructions: int = DEFAULT_MAX_INSTRUCTIONS,
+        max_blocks: int = DEFAULT_MAX_BLOCKS,
+    ) -> BackendResult:
+        """Render ``address`` with the native heuristic renderer.
+
+        Raises:
+            AnalysisError: When the address is not executable code, when a budget
+                is out of range, or when the region decodes to nothing.
+        """
+        instruction_budget = require_budget(
+            "max_instructions", max_instructions, MAX_MAX_INSTRUCTIONS
+        )
+        block_budget = require_budget("max_blocks", max_blocks, MAX_MAX_BLOCKS)
+        rendered = render_pseudo_c(
+            path,
+            info,
+            address,
+            max_instructions=instruction_budget,
+            max_blocks=block_budget,
+        )
+        evidence = replace(
+            rendered.evidence,
+            extra={
+                **rendered.evidence.extra,
+                "backend": self.name,
+                "max_instructions": instruction_budget,
+                "max_blocks": block_budget,
+            },
+        )
+        return BackendResult(
+            text=rendered.body,
+            backend=self.name,
+            language=LANGUAGE,
+            warnings=_header_warnings(rendered.body),
+            evidence=evidence,
+        )

codevanta/analysis/decompilers/ghidra_backend.py

@@ -0,0 +1,316 @@
+"""Ghidra decompiler backend via ``pyghidra`` (optional, imported lazily).
+
+Ghidra is the strongest free decompiler, but it needs two independent pieces: the
+``pyghidra`` Python bridge *and* an actual Ghidra installation. Both are probed --
+``pyghidra`` by importing it, the installation through the ``GHIDRA_INSTALL_DIR``
+environment variable -- and the backend only reports ``available=True`` when both
+are present. Nothing is ever downloaded, extracted or launched by the probe: a
+machine without Ghidra simply gets ``available=False`` and a reason saying which
+half is missing.
+
+Rendering uses pyghidra's headless entry point, ``pyghidra.open_program``: it
+starts the JVM, imports and analyses the file into an in-memory project and hands
+back a ``FlatProgramAPI``. Inside that context ``ghidra.app.decompiler`` is
+importable through JPype, so :class:`DecompInterface` decompiles the function at
+the requested address and returns C text.
+
+Budgeting: Ghidra's decompiler has no instruction or block budget either. The
+instruction budget is projected onto the decompiler *timeout* (roughly one
+simulated second per 200 instructions, clamped to 5..300 s) and the resulting
+basic-block count is compared with ``max_blocks`` afterwards, which can only be
+reported as a warning. Both budgets travel into the evidence record.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+from pathlib import Path
+from typing import Any
+
+from codevanta.analysis.decompile import (
+    DEFAULT_MAX_BLOCKS,
+    DEFAULT_MAX_INSTRUCTIONS,
+    MAX_MAX_BLOCKS,
+    MAX_MAX_INSTRUCTIONS,
+)
+from codevanta.core.loader import BinaryInfo
+from codevanta.core.report import AnalysisError
+from codevanta.evidence.confidence import Confidence
+from codevanta.evidence.models import Evidence
+
+from .base import (
+    LANGUAGE,
+    BackendAvailability,
+    BackendResult,
+    require_budget,
+)
+
+__all__ = ["GhidraBackend", "INSTALL_ENV", "INSTALL_HINT", "NAME"]
+
+#: Backend identifier.
+NAME = "ghidra"
+
+#: Environment variable that points at a Ghidra installation.
+INSTALL_ENV = "GHIDRA_INSTALL_DIR"
+
+#: What a user has to install for this backend to become available.
+INSTALL_HINT = (
+    f"pip install pyghidra and download Ghidra, then set {INSTALL_ENV} to the "
+    "installation directory"
+)
+
+#: Evidence source label.
+_SOURCE = "ghidra.decompiler"
+
+#: Evidence method label.
+_METHOD = "pyghidra.headless.DecompInterface"
+
+#: Entries that mark a directory as a Ghidra installation.
+_INSTALL_MARKERS = ("support", "ghidraRun", "ghidraRun.bat")
+
+#: Smallest and largest decompiler timeout, in seconds.
+_MIN_TIMEOUT = 5
+_MAX_TIMEOUT = 300
+
+#: Instructions per projected timeout second.
+_INSTRUCTIONS_PER_SECOND = 200
+
+#: Comment emitted when Ghidra's decompiler produced no code.
+_EMPTY_TEXT = "/* ghidra produced no pseudocode for 0x{address:x} */"
+
+
+def _unavailable(availability: BackendAvailability) -> AnalysisError:
+    """Build the error raised when a caller asked for an unavailable backend."""
+    return AnalysisError(
+        f"decompiler backend {availability.name!r} is unavailable: {availability.reason}; "
+        "use backend='native' or backend='auto' instead"
+    )
+
+
+def _timeout_seconds(instruction_budget: int) -> int:
+    """Project an instruction budget onto a decompiler timeout."""
+    seconds = instruction_budget // _INSTRUCTIONS_PER_SECOND
+    return max(_MIN_TIMEOUT, min(_MAX_TIMEOUT, seconds))
+
+
+def _installation_problem(install_dir: str) -> str | None:
+    """Return why ``install_dir`` is not usable, or ``None`` when it looks right."""
+    if not install_dir:
+        return f"no Ghidra installation is configured ({INSTALL_ENV} is not set)"
+    root = Path(install_dir)
+    if not root.is_dir():
+        return f"{INSTALL_ENV}={install_dir!r} is not a directory"
+    if not any((root / marker).exists() for marker in _INSTALL_MARKERS):
+        return (
+            f"{INSTALL_ENV}={install_dir!r} does not look like a Ghidra installation "
+            "(no support/ directory and no ghidraRun launcher)"
+        )
+    return None
+
+
+class GhidraBackend:
+    """Decompilation through a local Ghidra installation driven by pyghidra.
+
+    Attributes:
+        name: ``"ghidra"``.
+        requires: ``("pyghidra", "ghidra")``.
+    """
+
+    name: str = NAME
+    requires: tuple[str, ...] = ("pyghidra", "ghidra")
+
+    def availability(self) -> BackendAvailability:
+        """Probe ``pyghidra`` and :data:`INSTALL_ENV`; never raises, never launches."""
+        problems: list[str] = []
+        version = "unknown"
+        try:
+            import pyghidra
+
+            version = str(getattr(pyghidra, "__version__", "unknown"))
+        except ImportError as exc:
+            problems.append(
+                f"the 'pyghidra' package is not installed ({type(exc).__name__}: {exc})"
+            )
+        except Exception as exc:  # a broken install must not break discovery
+            problems.append(f"importing 'pyghidra' failed ({type(exc).__name__}: {exc})")
+        problem = _installation_problem(os.environ.get(INSTALL_ENV, "").strip())
+        if problem is not None:
+            problems.append(problem)
+        if problems:
+            return BackendAvailability(
+                name=self.name,
+                available=False,
+                reason="; ".join(problems) + f"; {INSTALL_HINT}",
+                requires=self.requires,
+            )
+        return BackendAvailability(
+            name=self.name,
+            available=True,
+            reason=(
+                f"pyghidra {version} is importable and {INSTALL_ENV} points at a Ghidra "
+                "installation; the headless decompiler can run"
+            ),
+            requires=self.requires,
+        )
+
+    def decompile(
+        self,
+        path: str | Path,
+        info: BinaryInfo,
+        address: int,
+        max_instructions: int = DEFAULT_MAX_INSTRUCTIONS,
+        max_blocks: int = DEFAULT_MAX_BLOCKS,
+    ) -> BackendResult:
+        """Decompile ``address`` with the local Ghidra installation.
+
+        Args:
+            path: Binary to import into a temporary Ghidra project (read-only use
+                of the file; the target is analysed, never executed).
+            info: Loaded metadata; Ghidra imports the file itself, so only the
+                path is used. Kept for a uniform backend signature.
+            address: Virtual address of the function to decompile.
+            max_instructions: Projected onto the decompiler timeout; Ghidra has no
+                instruction budget.
+            max_blocks: Compared with the function's block count afterwards; an
+                overrun becomes a warning, not an error.
+
+        Returns:
+            The rendering plus any warnings Ghidra could not prevent.
+
+        Raises:
+            AnalysisError: When pyghidra or the Ghidra installation is missing,
+                when a budget is out of range, when no function exists at
+                ``address``, or when the decompiler reports a failure.
+        """
+        instruction_budget = require_budget(
+            "max_instructions", max_instructions, MAX_MAX_INSTRUCTIONS
+        )
+        block_budget = require_budget("max_blocks", max_blocks, MAX_MAX_BLOCKS)
+        availability = self.availability()
+        if not availability.available:
+            raise _unavailable(availability)
+
+        import pyghidra
+
+        opener = getattr(pyghidra, "open_program", None)
+        if not callable(opener):
+            raise AnalysisError(
+                "the installed pyghidra exposes no open_program() headless entry point; "
+                f"{INSTALL_HINT}"
+            )
+        timeout = _timeout_seconds(instruction_budget)
+        _ensure_started(pyghidra)
+        try:
+            program_context = opener(str(path), analyze=True)
+        except TypeError:
+            program_context = opener(str(path))
+
+        try:
+            with program_context as flat_api:
+                text, block_count = _decompile_in_program(flat_api, address, timeout)
+        except AnalysisError:
+            raise
+        except Exception as exc:  # JPype surfaces Java failures as generic errors
+            raise AnalysisError(
+                f"ghidra could not decompile 0x{address:x}: {type(exc).__name__}: {exc}"
+            ) from exc
+
+        warnings: list[str] = []
+        if not text.strip():
+            warnings.append(
+                f"ghidra returned no decompilation for 0x{address:x}: the decompiler "
+                "completed without C text, so the rendering is a placeholder comment"
+            )
+            text = _EMPTY_TEXT.format(address=address)
+        if block_count > block_budget:
+            warnings.append(
+                f"ghidra decompiled {block_count} block(s), above max_blocks={block_budget}; "
+                "Ghidra exposes no block budget, so the text is complete rather than truncated"
+            )
+
+        evidence = Evidence(
+            source=_SOURCE,
+            method=_METHOD,
+            confidence=Confidence.MEDIUM,
+            address=address,
+            detail=(
+                f"ghidra decompiled the function at 0x{address:x} ({block_count} block(s)) "
+                f"headlessly through pyghidra with a {timeout}s timeout projected from "
+                f"max_instructions={instruction_budget}; Ghidra has no instruction or block "
+                f"budget, so max_blocks={block_budget} is checked afterwards only"
+            ),
+            extra={
+                "backend": self.name,
+                "install_dir": os.environ.get(INSTALL_ENV, ""),
+                "timeout_seconds": timeout,
+                "max_instructions": instruction_budget,
+                "max_blocks": block_budget,
+                "block_count": block_count,
+            },
+        )
+        return BackendResult(
+            text=text,
+            backend=self.name,
+            language=LANGUAGE,
+            warnings=tuple(warnings),
+            evidence=evidence,
+        )
+
+
+def _ensure_started(pyghidra: Any) -> None:
+    """Start the JVM the way pyghidra expects, ignoring "already started"."""
+    start = getattr(pyghidra, "start", None)
+    if not callable(start):
+        return
+    # pyghidra raises when the JVM is already up, which is exactly the state we
+    # want; any other failure resurfaces from open_program() below.
+    with contextlib.suppress(Exception):
+        start()
+
+
+def _decompile_in_program(
+    flat_api: Any,
+    address: int,
+    timeout_seconds: int,
+) -> tuple[str, int]:
+    """Decompile one function inside an open pyghidra program context."""
+    from ghidra.app.decompiler import DecompileOptions, DecompInterface
+
+    program = flat_api.getCurrentProgram()
+    target = flat_api.toAddr(address)
+    function = flat_api.getFunctionAt(target)
+    if function is None:
+        function = flat_api.getFunctionContaining(target)
+    if function is None:
+        raise AnalysisError(
+            f"ghidra found no function at 0x{address:x}; decompilation needs a defined function"
+        )
+    # Ghidra's Function API has no ``getBasicBlocks``; the body of a function is a
+    # set of address ranges, and one range is the closest portable stand-in for a
+    # block count. ``getBody`` is present across the versions pyghidra ships, and
+    # any failure here must not sink the decompilation itself, so it degrades to 0
+    # (reported as "unknown") instead of raising.
+    block_count = 0
+    try:
+        body = function.getBody()
+        ranges = body.getAddressRanges() if body is not None else None
+        block_count = len(list(ranges)) if ranges is not None else 0
+    except Exception:  # pragma: no cover - defensive across Ghidra versions
+        block_count = 0
+    decompiler = DecompInterface()
+    try:
+        decompiler.setOptions(DecompileOptions())
+        decompiler.openProgram(program)
+        # A ``None`` task monitor is accepted and is what headless scripting uses.
+        result = decompiler.decompileFunction(function, timeout_seconds, None)
+        if result is None:
+            raise AnalysisError(f"ghidra returned no decompiler result for 0x{address:x}")
+        if not result.decompileCompleted():
+            message = str(result.getErrorMessage() or "no error message reported")
+            raise AnalysisError(f"ghidra could not decompile 0x{address:x}: {message}")
+        code = result.getDecompiledFunction()
+        text = "" if code is None else str(code.getC() or "")
+    finally:
+        decompiler.dispose()
+    return text, block_count