codevanta 0.1.0__py3-none-any.whl

codevanta/core/__init__.py

@@ -0,0 +1,31 @@
+"""CODEVANTA analysis core: loader, disassembler, strings, functions."""
+
+from .disassembler import Disassembler, DisassemblyError, DisassemblyResult
+from .functions import FunctionCandidate, discover_functions
+from .loader import (
+    BinaryInfo,
+    BinaryLoader,
+    BinaryLoadError,
+    ExportInfo,
+    ImportInfo,
+    SectionInfo,
+    SymbolInfo,
+)
+from .strings import StringExtractor, StringResult
+
+__all__ = [
+    "BinaryInfo",
+    "BinaryLoader",
+    "BinaryLoadError",
+    "Disassembler",
+    "DisassemblyError",
+    "DisassemblyResult",
+    "ExportInfo",
+    "FunctionCandidate",
+    "ImportInfo",
+    "SectionInfo",
+    "StringExtractor",
+    "StringResult",
+    "SymbolInfo",
+    "discover_functions",
+]

codevanta/core/disassembler.py

@@ -0,0 +1,121 @@
+"""Capstone-backed disassembly for supported architectures."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import capstone
+
+from codevanta.evidence.confidence import Confidence
+from codevanta.evidence.models import Evidence
+from codevanta.ir.instruction import IRInstruction
+
+
+class DisassemblyError(Exception):
+    """Raised when disassembly cannot be performed."""
+
+
+@dataclass(frozen=True)
+class DisassemblyResult:
+    """Outcome of a single disassembly request."""
+
+    instructions: list[IRInstruction]
+    base_address: int
+    consumed_bytes: int
+    evidence: Evidence
+    truncated: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "base_address": f"0x{self.base_address:x}",
+            "consumed_bytes": self.consumed_bytes,
+            "truncated": self.truncated,
+            "instruction_count": len(self.instructions),
+            "instructions": [i.to_dict() for i in self.instructions],
+            "evidence": self.evidence.to_dict(),
+        }
+
+
+_ARCH_MAP: dict[tuple[str, int], tuple[int, int]] = {
+    ("x86", 32): (capstone.CS_ARCH_X86, capstone.CS_MODE_32),
+    ("x86", 64): (capstone.CS_ARCH_X86, capstone.CS_MODE_64),
+    ("x86_64", 64): (capstone.CS_ARCH_X86, capstone.CS_MODE_64),
+    ("arm", 32): (capstone.CS_ARCH_ARM, capstone.CS_MODE_ARM),
+    ("aarch64", 64): (capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM),
+    ("arm64", 64): (capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM),
+    ("mips", 32): (capstone.CS_ARCH_MIPS, capstone.CS_MODE_MIPS32),
+    ("mips", 64): (capstone.CS_ARCH_MIPS, capstone.CS_MODE_MIPS64),
+}
+
+
+class Disassembler:
+    """Disassembles machine code with Capstone."""
+
+    MAX_BYTES = 16 * 1024 * 1024
+    MAX_INSTRUCTIONS = 1_000_000
+
+    def __init__(self, architecture: str, bitness: int, endianness: str = "little") -> None:
+        self.architecture = architecture
+        self.bitness = bitness
+        self.endianness = endianness
+        arch_mode = _ARCH_MAP.get((architecture.lower(), bitness))
+        if arch_mode is None:
+            raise DisassemblyError(f"Unsupported architecture: {architecture} ({bitness}-bit)")
+        cs_arch, cs_mode = arch_mode
+        if endianness.lower() == "big":
+            if cs_arch in (capstone.CS_ARCH_ARM, capstone.CS_ARCH_MIPS):
+                cs_mode |= capstone.CS_MODE_BIG_ENDIAN
+            elif cs_arch == capstone.CS_ARCH_X86:
+                raise DisassemblyError("x86 is little-endian only")
+            elif cs_arch == capstone.CS_ARCH_ARM64:
+                raise DisassemblyError("big-endian AArch64 is not supported")
+        self._cs = capstone.Cs(cs_arch, cs_mode)
+        self._cs.detail = False
+
+    def disassemble(
+        self,
+        data: bytes,
+        base_address: int,
+        max_instructions: int = 0,
+        max_bytes: int = 0,
+    ) -> DisassemblyResult:
+        """Decode ``data`` starting at ``base_address``."""
+        if not data:
+            raise DisassemblyError("No bytes supplied for disassembly")
+        if base_address < 0:
+            raise DisassemblyError("base_address must be non-negative")
+        byte_limit = min(len(data), max_bytes if max_bytes > 0 else self.MAX_BYTES)
+        instruction_limit = max_instructions if max_instructions > 0 else self.MAX_INSTRUCTIONS
+        window = data[:byte_limit]
+        instructions: list[IRInstruction] = []
+        consumed = 0
+        truncated = byte_limit < len(data)
+        for insn in self._cs.disasm(window, base_address):
+            instructions.append(
+                IRInstruction(
+                    address=insn.address,
+                    size=insn.size,
+                    mnemonic=insn.mnemonic,
+                    operands=insn.op_str,
+                    raw_bytes=bytes(insn.bytes),
+                )
+            )
+            consumed += insn.size
+            if len(instructions) >= instruction_limit:
+                truncated = truncated or consumed < len(window)
+                break
+        evidence = Evidence(
+            source="capstone",
+            method="linear_disassembly",
+            confidence=Confidence.CERTAIN,
+            address=base_address,
+            detail=f"{len(instructions)} instruction(s) decoded",
+        )
+        return DisassemblyResult(
+            instructions=instructions,
+            base_address=base_address,
+            consumed_bytes=consumed,
+            evidence=evidence,
+            truncated=truncated,
+        )

codevanta/core/entropy.py

@@ -0,0 +1,237 @@
+"""Per-section Shannon entropy for packing triage.
+
+Entropy itself is a plain measurement over file bytes (confidence ``certain``);
+the *packed/compressed* interpretation is a heuristic and is labelled as such
+(``medium`` confidence, threshold stated in the evidence detail). Sections
+without file bytes (e.g. ``S_ZEROFILL``) are reported with ``entropy=None``
+rather than a fabricated value. Sampling is capped per section because entropy
+converges quickly; the sample size actually used is part of every result.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from codevanta.core.loader import BinaryInfo
+from codevanta.core.report import AnalysisError, read_code
+from codevanta.evidence.confidence import Confidence
+from codevanta.evidence.models import Evidence
+
+__all__ = [
+    "DEFAULT_MAX_SAMPLE_BYTES",
+    "DEFAULT_MIN_SAMPLE_BYTES",
+    "PACKED_HINT_THRESHOLD",
+    "EntropyReport",
+    "SectionEntropy",
+    "analyze_entropy",
+]
+
+#: Entropy (bits/byte) at or above which a section is flagged as possibly
+#: packed or compressed. 7.2 is a widely used rule of thumb, not a law.
+PACKED_HINT_THRESHOLD = 7.2
+
+#: Default cap on bytes read per section; entropy converges well before this.
+DEFAULT_MAX_SAMPLE_BYTES = 65_536
+
+#: Below this sample size the entropy is still reported but too noisy to mean
+#: much, so the packed hint is suppressed and a caveat is attached instead.
+DEFAULT_MIN_SAMPLE_BYTES = 256
+
+#: Hard ceiling accepted for ``max_sample_bytes`` (bounded-resource rule).
+MAX_SAMPLE_LIMIT = 4 * 1024 * 1024
+
+_MAX_ENTROPY = 8.0
+
+
+@dataclass(frozen=True)
+class SectionEntropy:
+    """Entropy of one section, sampled from its file bytes."""
+
+    name: str
+    virtual_address: int
+    size: int
+    entropy: float | None
+    sampled_bytes: int
+    is_executable: bool
+    packed_hint: bool
+    evidence: Evidence
+
+    def to_dict(self) -> dict[str, Any]:
+        data: dict[str, Any] = {
+            "name": self.name,
+            "virtual_address": f"0x{self.virtual_address:x}",
+            "size": self.size,
+            "entropy": round(self.entropy, 4) if self.entropy is not None else None,
+            "sampled_bytes": self.sampled_bytes,
+            "executable": self.is_executable,
+            "packed_hint": self.packed_hint,
+            "evidence": self.evidence.to_dict(),
+        }
+        return data
+
+
+@dataclass(frozen=True)
+class EntropyReport:
+    """Entropy triage across every section of one binary."""
+
+    sections: list[SectionEntropy]
+    threshold: float
+    warnings: tuple[str, ...]
+    evidence: Evidence
+
+    @property
+    def packed_candidates(self) -> list[SectionEntropy]:
+        """Sections whose entropy reached the packed-compression threshold."""
+        return [section for section in self.sections if section.packed_hint]
+
+    def to_dict(self) -> dict[str, Any]:
+        packed = self.packed_candidates
+        return {
+            "section_count": len(self.sections),
+            "threshold": self.threshold,
+            "packed_candidate_count": len(packed),
+            "packed_candidates": [section.name for section in packed],
+            "sections": [section.to_dict() for section in self.sections],
+            "warnings": list(self.warnings),
+            "evidence": self.evidence.to_dict(),
+        }
+
+
+def shannon_entropy(data: bytes) -> float:
+    """Return the Shannon entropy of ``data`` in bits per byte (0..8)."""
+    if not data:
+        return 0.0
+    counts = [0] * 256
+    for byte in data:
+        counts[byte] += 1
+    total = len(data)
+    entropy = 0.0
+    for count in counts:
+        if count:
+            probability = count / total
+            entropy -= probability * math.log2(probability)
+    return min(entropy, _MAX_ENTROPY)
+
+
+def analyze_entropy(
+    path: str | Path,
+    info: BinaryInfo,
+    *,
+    threshold: float = PACKED_HINT_THRESHOLD,
+    max_sample_bytes: int = DEFAULT_MAX_SAMPLE_BYTES,
+    min_sample_bytes: int = DEFAULT_MIN_SAMPLE_BYTES,
+) -> EntropyReport:
+    """Compute per-section entropy and flag possible packing.
+
+    Args:
+        path: File the metadata in ``info`` was loaded from (read-only).
+        info: Normalized binary metadata.
+        threshold: Entropy at or above which ``packed_hint`` is set (0 < t <= 8).
+        max_sample_bytes: Bytes read per section at most (1 .. 4 MiB).
+        min_sample_bytes: Sample size below which the packed hint is suppressed.
+
+    Returns:
+        An :class:`EntropyReport` with one entry per section, sorted by
+        entropy (descending), sections without file bytes carrying ``None``.
+
+    Raises:
+        AnalysisError: If any limit is out of range.
+    """
+    if not 0.0 < threshold <= _MAX_ENTROPY:
+        raise AnalysisError(f"threshold must be in (0, {_MAX_ENTROPY}]")
+    if max_sample_bytes < 1:
+        raise AnalysisError("max_sample_bytes must be >= 1")
+    if max_sample_bytes > MAX_SAMPLE_LIMIT:
+        raise AnalysisError(f"max_sample_bytes too large (limit {MAX_SAMPLE_LIMIT})")
+    if min_sample_bytes < 1:
+        raise AnalysisError("min_sample_bytes must be >= 1")
+
+    warnings: list[str] = []
+    results: list[SectionEntropy] = []
+    for section in info.sections:
+        entropy: float | None = None
+        sampled = 0
+        # File-backed means: PE/Mach-O raw bytes, or an ELF section with a real
+        # file offset (ELF loader metadata carries raw_size=0 by design).
+        if section.raw_size > 0 or section.offset > 0:
+            try:
+                data = read_code(path, info, section.virtual_address, max_sample_bytes)
+            except (AnalysisError, OSError) as exc:
+                warnings.append(
+                    f"section {section.name!r}: cannot read bytes ({exc}); entropy unknown"
+                )
+            else:
+                sampled = len(data)
+                if sampled:
+                    entropy = shannon_entropy(data)
+                else:
+                    warnings.append(
+                        f"section {section.name!r}: no file bytes; entropy unknown"
+                    )
+        else:
+            warnings.append(
+                f"section {section.name!r}: not file-backed (size {section.size}); "
+                "entropy unknown"
+            )
+
+        packed_hint = entropy is not None and entropy >= threshold
+        detail_suffix = ""
+        if entropy is None:
+            detail = "entropy not computable: no file bytes"
+            confidence = Confidence.LOW
+        else:
+            confidence = Confidence.CERTAIN
+            if sampled < min_sample_bytes:
+                packed_hint = False
+                detail_suffix = "; sample too small for a packing judgment"
+                detail = (
+                    f"entropy {entropy:.3f} bits/byte from {sampled} sampled byte(s)"
+                    + detail_suffix
+                )
+            else:
+                detail = (
+                    f"entropy {entropy:.3f} bits/byte from {sampled} sampled byte(s); "
+                    f"packed hint at >={threshold} is a heuristic, not proof"
+                    if packed_hint
+                    else f"entropy {entropy:.3f} bits/byte from {sampled} sampled byte(s)"
+                )
+        results.append(
+            SectionEntropy(
+                name=section.name,
+                virtual_address=section.virtual_address,
+                size=section.size,
+                entropy=entropy,
+                sampled_bytes=sampled,
+                is_executable=section.is_executable,
+                packed_hint=packed_hint,
+                evidence=Evidence(
+                    source="entropy.section",
+                    method="shannon_entropy",
+                    confidence=confidence,
+                    address=section.virtual_address,
+                    detail=detail,
+                ),
+            )
+        )
+
+    results.sort(key=lambda item: (-1 if item.entropy is None else -item.entropy, item.name))
+    packed_names = [item.name for item in results if item.packed_hint]
+    report_evidence = Evidence(
+        source="entropy.triage",
+        method="per_section_shannon_entropy",
+        confidence=Confidence.CERTAIN,
+        detail=(
+            f"{len(results)} section(s) measured; threshold {threshold}; "
+            f"{len(packed_names)} packed candidate(s)"
+            + (f" ({', '.join(packed_names)})" if packed_names else "")
+        ),
+    )
+    return EntropyReport(
+        sections=results,
+        threshold=threshold,
+        warnings=tuple(warnings),
+        evidence=report_evidence,
+    )

codevanta/core/functions.py

@@ -0,0 +1,176 @@
+"""Function candidate discovery with explicit provenance."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from codevanta.evidence.confidence import Confidence
+from codevanta.evidence.models import Evidence
+
+from .loader import BinaryInfo
+
+
+@dataclass(frozen=True)
+class FunctionCandidate:
+    """A function candidate and the evidence that identified it."""
+
+    address: int
+    name: str | None
+    size: int | None
+    source: str
+    confidence: Confidence
+    evidence: Evidence
+    aliases: tuple[str, ...] = field(default_factory=tuple)
+
+    def to_dict(self) -> dict[str, Any]:
+        data: dict[str, Any] = {
+            "address": f"0x{self.address:x}",
+            "source": self.source,
+            "confidence": self.confidence.value,
+            "evidence": self.evidence.to_dict(),
+        }
+        if self.name:
+            data["name"] = self.name
+        if self.size is not None:
+            data["size"] = self.size
+        if self.aliases:
+            data["aliases"] = list(self.aliases)
+        return data
+
+
+@dataclass
+class _Draft:
+    address: int
+    name: str | None
+    size: int | None
+    source: str
+    confidence: Confidence
+    evidence: Evidence
+    aliases: list[str] = field(default_factory=list)
+
+
+#: Evidence source for exported functions, by binary format.
+_EXPORT_SOURCES: dict[str, str] = {
+    "PE": "pe.export_table",
+    "ELF": "elf.dynsym",
+    "MachO": "macho.exported_symbols",
+}
+
+#: Evidence source for symbol-table functions, by binary format.
+_SYMBOL_SOURCES: dict[str, str] = {
+    "PE": "pe.coff_symbols",
+    "ELF": "elf.symtab",
+    "MachO": "macho.symbol_table",
+    "DEX": "dex.method_ids",
+}
+
+#: Discovery confidence for symbol-table candidates, by binary format. Mach-O
+#: symbol entries do not encode kind, so function-ness is inferred from
+#: executable-section membership and deserves slightly lower confidence.
+_SYMBOL_CONFIDENCE: dict[str, Confidence] = {
+    "PE": Confidence.HIGH,
+    "ELF": Confidence.HIGH,
+    "MachO": Confidence.MEDIUM,
+}
+
+
+def discover_functions(info: BinaryInfo) -> list[FunctionCandidate]:
+    """Discover function candidates from structural evidence only.
+
+    Phase 1 reports exported functions, function symbols, and the entry point.
+    Speculative heuristics are intentionally excluded; every candidate states
+    where it came from and how much confidence it deserves.
+    """
+    drafts: dict[int, _Draft] = {}
+
+    def add(
+        address: int | None,
+        name: str | None,
+        size: int | None,
+        source: str,
+        confidence: Confidence,
+        method: str,
+        detail: str,
+    ) -> None:
+        if not address or address <= 0:
+            return
+        clean_name = name or None
+        evidence = Evidence(
+            source=source,
+            method=method,
+            confidence=confidence,
+            address=address,
+            detail=detail,
+        )
+        existing = drafts.get(address)
+        if existing is None:
+            drafts[address] = _Draft(
+                address=address,
+                name=clean_name,
+                size=size,
+                source=source,
+                confidence=confidence,
+                evidence=evidence,
+            )
+            return
+        if confidence.score > existing.confidence.score:
+            if existing.name and existing.name != clean_name:
+                existing.aliases.append(existing.name)
+            existing.name = clean_name
+            existing.size = size if size is not None else existing.size
+            existing.source = source
+            existing.confidence = confidence
+            existing.evidence = evidence
+        elif clean_name and clean_name != existing.name and clean_name not in existing.aliases:
+            existing.aliases.append(clean_name)
+
+    for export in info.exports:
+        add(
+            export.address,
+            export.name,
+            None,
+            _EXPORT_SOURCES.get(info.format, "binary.exports"),
+            Confidence.HIGH,
+            "exported_function",
+            "exported function entry",
+        )
+
+    for symbol in info.symbols:
+        if not symbol.is_function:
+            continue
+        add(
+            symbol.address,
+            symbol.name,
+            symbol.size or None,
+            _SYMBOL_SOURCES.get(info.format, "binary.symbol_table"),
+            _SYMBOL_CONFIDENCE.get(info.format, Confidence.HIGH),
+            "function_symbol",
+            "function symbol from the symbol table",
+        )
+
+    if info.entry_point:
+        add(
+            info.entry_point,
+            None,
+            None,
+            "binary.entry_point",
+            Confidence.MEDIUM,
+            "entry_point",
+            "program entry point; a code candidate, not a proven function boundary",
+        )
+
+    candidates = [
+        FunctionCandidate(
+            address=draft.address,
+            name=draft.name,
+            size=draft.size,
+            source=draft.source,
+            confidence=draft.confidence,
+            evidence=draft.evidence,
+            aliases=tuple(dict.fromkeys(draft.aliases)),
+        )
+        for draft in drafts.values()
+    ]
+    candidates.sort(key=lambda c: c.address)
+    return candidates