oswatcher-plugins 0.14.0__py3-none-any.whl

plugins/plugins/linux_symbols.py

@@ -0,0 +1,271 @@
+"""Linux kernel symbol extraction plugin using DWARF debug info from Ubuntu ddebs."""
+
+from __future__ import annotations
+
+import tempfile
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import requests
+from attrs import define, field
+from neogit.model.neo import Commit
+
+from plugins.plugins.linux_symbols_service import (
+    detect_ubuntu_codename,
+    extract_vmlinux_from_ddeb,
+    parse_kernel_version_parts,
+    parse_symbols_for_neo4j,
+    resolve_ddeb_url_from_packages,
+    run_dwarf2json,
+)
+from plugins.plugins.symbols import DataTypeMerkleNode, StructMerkleNode, StructNode, SymbolsMerkleVisitor
+from plugins.plugins.symbols_repository import SymbolsRepository
+from plugins.syscalls.filesystem import KernelInfo, find_kernel_versions, get_boot_directory
+from plugins.types import AbstractPlugin, UniqueConstraint
+
+
+def lief_arch_to_ddeb_arch(lief_arch: str) -> str:
+    """Convert lief architecture string to ddeb architecture name.
+
+    Handles both ELF machine types (e.g., "ARCH.x86_64") and
+    PE machine types from EFI boot stubs (e.g., "MACHINE_TYPES.AMD64").
+
+    Args:
+        lief_arch: Architecture string from lief
+
+    Returns:
+        Architecture for ddeb URL like "amd64" or "arm64"
+    """
+    arch_lower = lief_arch.lower()
+    if "x86_64" in arch_lower or "amd64" in arch_lower:
+        return "amd64"
+    if "aarch64" in arch_lower or "arm64" in arch_lower:
+        return "arm64"
+    if "arm" in arch_lower:
+        return "armhf"
+    if "i386" in arch_lower or "i686" in arch_lower:
+        return "i386"
+    raise ValueError(f"Unrecognized architecture: {lief_arch!r}")
+
+
+@define(auto_attribs=True)
+class LinuxSymbolsPlugin(AbstractPlugin):
+    """Plugin to extract Linux kernel symbols and struct definitions from DWARF debug info.
+
+    This plugin:
+    1. Finds Linux kernels in /boot directory
+    2. Downloads debug symbols from ddebs.ubuntu.com
+    3. Parses DWARF debug info using dwarf2json (Volatility Foundation)
+    4. Stores symbols and struct definitions in Neo4j
+    """
+
+    _repository: Optional[SymbolsRepository] = field(init=False, default=None)
+
+    # Request timeout for downloading ddeb packages (in seconds)
+    DOWNLOAD_TIMEOUT = 300
+
+    @property
+    def repository(self) -> SymbolsRepository:
+        """Lazy-initialize repository."""
+        if self._repository is None:
+            self._repository = SymbolsRepository(self.neogit)
+        return self._repository
+
+    def constraints_data(self) -> List[UniqueConstraint]:
+        """Return constraints for symbol-related nodes.
+
+        Reuses same constraints as SymbolsPlugin since we use the same node types.
+        """
+        return [
+            UniqueConstraint(label="Symbol", property_list=["hash"]),
+            UniqueConstraint(label="Struct", property_list=["hash"]),
+            UniqueConstraint(label="StructField", property_list=["hash"]),
+            UniqueConstraint(label="DataType", property_list=["hash"]),
+        ]
+
+    def run(self, commit: Commit):
+        """Execute Linux symbol extraction for a commit.
+
+        Args:
+            commit: The Commit node to analyze
+        """
+        self.logger.info(f"Running Linux symbols plugin for commit {commit.hash}")
+
+        # Get the root filesystem tree
+        try:
+            root_tree = commit.filesystem[0]
+        except IndexError:
+            self.logger.warning(f"No filesystem found for commit {commit.hash}")
+            return
+
+        # Navigate to /boot directory
+        boot_tree = get_boot_directory(root_tree)
+        if not boot_tree:
+            self.logger.info("No /boot directory found, skipping Linux symbol extraction")
+            return
+
+        codename = detect_ubuntu_codename(root_tree)
+        if codename:
+            self.logger.info(f"Detected Ubuntu codename: {codename}")
+        else:
+            self.logger.warning("Could not detect Ubuntu codename from /etc/os-release")
+
+        # Find kernel versions from vmlinuz files
+        kernel_info_list = find_kernel_versions(boot_tree, self)
+        if not kernel_info_list:
+            self.logger.info("No kernel files found in /boot")
+            return
+
+        self.logger.info(
+            f"Found {len(kernel_info_list)} kernel(s): "
+            + ", ".join(f"{k.filename} ({k.architecture})" for k in kernel_info_list)
+        )
+
+        # Process each kernel
+        for kernel_info in kernel_info_list:
+            try:
+                self._process_kernel(kernel_info, codename)
+            except Exception:
+                self.logger.exception(f"Failed to process kernel {kernel_info.filename}")
+
+        self.logger.info(f"Linux symbol extraction complete for commit {commit.hash}")
+
+    def _process_kernel(self, kernel_info: KernelInfo, codename: Optional[str]):
+        """Process a single kernel: download debug symbols and extract info.
+
+        Args:
+            kernel_info: Information about the kernel to process
+        """
+        self.logger.info(f"Processing kernel: {kernel_info.filename}")
+
+        # Parse version components from filename
+        try:
+            version, build, flavor = parse_kernel_version_parts(kernel_info.filename)
+        except ValueError as e:
+            self.logger.warning(f"Cannot parse kernel version from {kernel_info.filename}: {e}")
+            return
+
+        # Convert architecture
+        ddeb_arch = lief_arch_to_ddeb_arch(kernel_info.architecture)
+
+        # Resolve exact ddeb URL from repository metadata.
+        ddeb_url = resolve_ddeb_url_from_packages(version, build, flavor, ddeb_arch, codename)
+        if ddeb_url is None:
+            self.logger.error(
+                f"Could not resolve debug package URL for {kernel_info.filename} "
+                f"(codename={codename}, arch={ddeb_arch})"
+            )
+            return
+
+        self.logger.info(f"Debug package URL: {ddeb_url}")
+
+        # Download and process ddeb
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmpdir_path = Path(tmpdir)
+
+            # Download ddeb
+            ddeb_path = tmpdir_path / "debug.ddeb"
+            try:
+                self._download_ddeb(ddeb_url, ddeb_path)
+            except Exception as e:
+                self.logger.error(f"Failed to download ddeb from {ddeb_url}: {e}")
+                return
+
+            # Extract vmlinux
+            try:
+                vmlinux_path = extract_vmlinux_from_ddeb(ddeb_path, tmpdir_path)
+                self.logger.info(f"Extracted vmlinux: {vmlinux_path}")
+            except Exception as e:
+                self.logger.error(f"Failed to extract vmlinux: {e}")
+                return
+
+            # Parse everything in one dwarf2json call
+            try:
+                self.logger.info("Running dwarf2json...")
+                dwarf_data = run_dwarf2json(vmlinux_path)
+            except Exception as e:
+                self.logger.error(f"Failed to run dwarf2json: {e}")
+                return
+
+            # Insert symbols
+            symbols_dict = dwarf_data.get("symbols", {})
+            self.logger.info(f"Found {len(symbols_dict)} symbols")
+            param_list = parse_symbols_for_neo4j(symbols_dict)
+            self.logger.info(f"Inserting {len(param_list)} symbols into Neo4j")
+            self.repository.insert_symbols(kernel_info.blob_hash, param_list)
+
+            # Insert enums
+            enums = dwarf_data.get("enums", {})
+            self.logger.info(f"Found {len(enums)} enums")
+            count_enum = self._insert_user_types(kernel_info.blob_hash, enums)
+
+            # Insert structs/unions
+            types = dwarf_data.get("user_types", {})
+            self.logger.info(f"Found {len(types)} struct/union types")
+            count_types = self._insert_user_types(kernel_info.blob_hash, types)
+
+            self.logger.info(f"Inserted {len(param_list)} symbols, {count_enum} enums, {count_types} types")
+
+    def _download_ddeb(self, url: str, output_path: Path):
+        """Download ddeb package from URL.
+
+        Args:
+            url: URL to download from
+            output_path: Path to save the downloaded file
+        """
+        self.logger.info(f"Downloading: {url}")
+
+        response = requests.get(url, timeout=self.DOWNLOAD_TIMEOUT, stream=True)
+        response.raise_for_status()
+
+        total_size = int(response.headers.get("content-length", 0))
+        downloaded = 0
+
+        with open(output_path, "wb") as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                f.write(chunk)
+                downloaded += len(chunk)
+                if total_size > 0:
+                    percent = (downloaded / total_size) * 100
+                    if downloaded % (10 * 1024 * 1024) < 8192:  # Log every ~10MB
+                        mb_downloaded = downloaded / 1024 / 1024
+                        self.logger.debug(f"Downloaded {mb_downloaded:.1f}MB ({percent:.1f}%)")  # noqa: E231
+
+        mb_downloaded = downloaded / 1024 / 1024
+        self.logger.info(f"Downloaded {mb_downloaded:.1f}MB to {output_path}")  # noqa: E231
+
+    def _insert_user_types(self, blob_hash: str, types_dict: Dict) -> int:
+        """Insert user types (structs/unions/enums) into Neo4j using Merkle visitor.
+
+        This method reuses the same logic as SymbolsPlugin.parse_users_types().
+
+        Args:
+            blob_hash: Hash of the kernel blob
+            types_dict: Dictionary of type definitions in volatility3 format
+
+        Returns:
+            Number of types processed
+        """
+        with SymbolsMerkleVisitor(thread=True) as visitor:
+            for struct_name, struct_data in sorted(types_dict.items()):
+                self.logger.debug(f"Processing type: {struct_name}")
+                struct_node = StructNode(name=struct_name, struct_data=struct_data)
+                visitor.run_visit(struct_node)
+
+                for node in visitor.as_gen():
+                    merkle_node = node.return_value
+                    if isinstance(merkle_node, DataTypeMerkleNode):
+                        self.repository.insert_data_type(merkle_node)
+                    if isinstance(merkle_node, StructMerkleNode):
+                        unwind_param = [
+                            {
+                                "hash": child_node.hash,
+                                "name": child_name,
+                                "offset": child_node.offset,
+                                "data_type": child_node.data_type,
+                            }
+                            for child_name, child_node in merkle_node.children.items()
+                        ]
+                        self.repository.insert_struct(blob_hash, merkle_node, unwind_param)
+
+        return len(types_dict)

plugins/plugins/linux_symbols_service.py

@@ -0,0 +1,373 @@
+"""Pure functions for Linux kernel symbol extraction.
+
+This module provides pure functions for:
+- Constructing ddebs.ubuntu.com URLs for debug packages
+- Parsing kernel version strings
+- Extracting vmlinux from ddeb archives
+- Parsing DWARF debug info via dwarf2json (Volatility Foundation Go binary)
+"""
+
+from __future__ import annotations
+
+import gzip
+import hashlib
+import json
+import lzma
+import re
+import subprocess
+import tarfile
+import tempfile
+from pathlib import Path, PurePath
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
+
+import requests
+
+if TYPE_CHECKING:
+    from neogit.model.merkle import Tree
+
+
+# Pattern: vmlinuz-{major}.{minor}.{patch}-{build}-{flavor}
+# Example: vmlinuz-6.8.0-45-generic -> ("6.8.0", "45", "generic")
+KERNEL_VERSION_PARTS_PATTERN = re.compile(r"^vmlinuz-(\d+\.\d+\.\d+)-(\d+)-(.+)$")
+
+
+def parse_kernel_version_parts(filename: str) -> Tuple[str, str, str]:
+    """Parse vmlinuz filename into version components.
+
+    Args:
+        filename: Kernel filename like "vmlinuz-6.8.0-45-generic"
+
+    Returns:
+        Tuple of (version, build, flavor) e.g., ("6.8.0", "45", "generic")
+
+    Raises:
+        ValueError: If filename format is invalid
+    """
+    match = KERNEL_VERSION_PARTS_PATTERN.match(filename)
+    if not match:
+        raise ValueError(f"Invalid kernel filename format: {filename}")
+
+    return match.groups()  # type: ignore
+
+
+def construct_ddeb_url(kernel_version: str, build: str, flavor: str, arch: str) -> str:
+    """Construct ddebs.ubuntu.com URL for debug package.
+
+    The URL pattern is:
+    http://ddebs.ubuntu.com/pool/main/l/linux/linux-image-unsigned-{version}-{build}-{flavor}-dbgsym_{version}-{build}.{build}_{arch}.ddeb
+
+    Args:
+        kernel_version: Kernel version like "6.8.0"
+        build: Build number like "45"
+        flavor: Kernel flavor like "generic"
+        arch: Architecture like "amd64"
+
+    Returns:
+        Full URL to the ddeb package
+
+    Example:
+        >>> construct_ddeb_url("6.8.0", "45", "generic", "amd64")
+        'http://ddebs.ubuntu.com/pool/main/l/linux/linux-image-unsigned-6.8.0-45-generic-dbgsym_6.8.0-45.45_amd64.ddeb'
+    """
+    # Format: linux-image-unsigned-{ver}-{build}-{flavor}-dbgsym_{ver}-{build}.{build}_{arch}.ddeb
+    package_name = f"linux-image-unsigned-{kernel_version}-{build}-{flavor}-dbgsym"
+    version_string = f"{kernel_version}-{build}.{build}"
+    filename = f"{package_name}_{version_string}_{arch}.ddeb"
+
+    base_url = "http://ddebs.ubuntu.com/pool/main/l/linux"
+    return f"{base_url}/{filename}"
+
+
+def _parse_packages_index_for_filename(index_content: str, package_names: List[str], arch: str) -> Optional[str]:
+    """Parse Debian Packages index and return matching ddeb filename."""
+    package_set = set(package_names)
+    fields: Dict[str, str] = {}
+
+    def match_current_stanza() -> Optional[str]:
+        if fields.get("Package") not in package_set:
+            return None
+        if fields.get("Architecture") != arch:
+            return None
+        filename = fields.get("Filename")
+        if filename and filename.endswith(".ddeb"):
+            return filename
+        return None
+
+    for line in index_content.splitlines():
+        if not line.strip():
+            filename = match_current_stanza()
+            if filename:
+                return filename
+            fields = {}
+            continue
+
+        if line.startswith(" "):
+            continue
+
+        key, sep, value = line.partition(":")
+        if not sep:
+            continue
+        fields[key] = value.strip()
+
+    return match_current_stanza()
+
+
+def resolve_ddeb_url_from_packages(
+    kernel_version: str,
+    build: str,
+    flavor: str,
+    arch: str,
+    codename: Optional[str],
+    timeout: int = 30,
+) -> Optional[str]:
+    """Resolve exact ddeb URL by scanning ddebs Packages indexes.
+
+    Only searches for the unsigned dbgsym package, which contains
+    the actual vmlinux with DWARF debug info. The signed variant
+    is a stub with no debug symbols.
+
+    Searches the provided codename's suites first. If no codename is
+    given, falls back to a list of known LTS/current releases.
+    """
+    package_name = f"linux-image-unsigned-{kernel_version}-{build}-{flavor}-dbgsym"
+    base_url = "http://ddebs.ubuntu.com"
+
+    # Determine which codenames to search
+    if codename:
+        codenames = [codename]
+    else:
+        codenames = ["noble", "jammy", "focal"]
+
+    for release in codenames:
+        for suite in [release, f"{release}-updates", f"{release}-proposed"]:
+            for index_name in ["Packages.xz", "Packages.gz"]:
+                index_url = f"{base_url}/dists/{suite}/main/binary-{arch}/{index_name}"
+                try:
+                    response = requests.get(index_url, timeout=timeout)
+                    response.raise_for_status()
+                except requests.RequestException:
+                    continue
+
+                try:
+                    if index_name.endswith(".xz"):
+                        content = lzma.decompress(response.content).decode("utf-8", errors="replace")
+                    elif index_name.endswith(".gz"):
+                        content = gzip.decompress(response.content).decode("utf-8", errors="replace")
+                    else:
+                        content = response.text
+                except (lzma.LZMAError, gzip.BadGzipFile, UnicodeDecodeError):
+                    continue
+
+                filename = _parse_packages_index_for_filename(content, [package_name], arch)
+                if filename:
+                    return f"{base_url}/{filename.lstrip('/')}"
+                # Successfully parsed this index; skip other compression variants
+                break
+
+    # Fallback: scan pool directory listing directly.
+    major_minor = ".".join(kernel_version.split(".")[:2])
+    return _resolve_ddeb_url_from_pool_listing(kernel_version, build, flavor, arch, major_minor, timeout)
+
+
+def _resolve_ddeb_url_from_pool_listing(
+    kernel_version: str,
+    build: str,
+    flavor: str,
+    arch: str,
+    major_minor: str,
+    timeout: int = 30,
+) -> Optional[str]:
+    """Resolve ddeb URL by matching filenames in the pool directory listing.
+
+    Searches both the main linux pool and HWE-specific pool directories.
+    Only looks for unsigned dbgsym packages.
+    """
+    pool_urls = [
+        "http://ddebs.ubuntu.com/pool/main/l/linux/",
+        f"http://ddebs.ubuntu.com/pool/main/l/linux-hwe-{major_minor}/",
+    ]
+    prefix = f"linux-image-unsigned-{kernel_version}-{build}-{flavor}-dbgsym_"
+
+    for pool_url in pool_urls:
+        try:
+            response = requests.get(pool_url, timeout=timeout)
+            response.raise_for_status()
+        except requests.RequestException:
+            continue
+
+        pattern = rf'href="({re.escape(prefix)}[^"]*_{re.escape(arch)}\.ddeb)"'
+        matches = re.findall(pattern, response.text)
+
+        if matches:
+            # Keep deterministic behavior and prefer highest lexical revision.
+            filename = sorted(set(matches))[-1]
+            return f"{pool_url}{filename}"
+
+    return None
+
+
+def detect_ubuntu_codename(root_tree: "Tree") -> Optional[str]:
+    """Extract Ubuntu codename from /etc/os-release in filesystem.
+
+    Args:
+        root_tree: Root filesystem tree
+
+    Returns:
+        Ubuntu codename like "noble", "jammy", or None if not found
+    """
+    try:
+        os_release = root_tree.get_child_at_path(PurePath("/etc/os-release"))
+        if hasattr(os_release, "content"):
+            content = os_release.content.decode("utf-8", errors="replace")
+            for line in content.splitlines():
+                if line.startswith("VERSION_CODENAME="):
+                    return line.split("=", 1)[1].strip().strip('"')
+    except (FileNotFoundError, AttributeError):
+        pass
+    return None
+
+
+def extract_vmlinux_from_ddeb(ddeb_path: Path, output_dir: Path) -> Path:
+    """Extract vmlinux from ddeb archive.
+
+    ddeb structure:
+      - ddeb is an ar archive
+      - Contains data.tar.xz (or data.tar.zst)
+      - vmlinux is at usr/lib/debug/boot/vmlinux-*
+
+    Args:
+        ddeb_path: Path to downloaded ddeb file
+        output_dir: Directory to extract vmlinux to
+
+    Returns:
+        Path to extracted vmlinux file
+
+    Raises:
+        ValueError: If vmlinux not found in archive
+    """
+    # Extract data.tar from ar archive using ar command
+    # ar archives are simple and ar is universally available
+    result = subprocess.run(
+        ["ar", "-t", str(ddeb_path)],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    archive_members = result.stdout.strip().split("\n")
+
+    # Find data.tar.* file
+    data_tar_name = None
+    for member in archive_members:
+        if member.startswith("data.tar"):
+            data_tar_name = member
+            break
+
+    if not data_tar_name:
+        raise ValueError(f"No data.tar found in ddeb: {ddeb_path}")
+
+    # Extract data.tar to temp location
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmpdir_path = Path(tmpdir)
+        subprocess.run(
+            ["ar", "-x", str(ddeb_path), data_tar_name],
+            cwd=tmpdir_path,
+            check=True,
+        )
+
+        data_tar_path = tmpdir_path / data_tar_name
+
+        # tarfile doesn't support zstd until Python 3.14, decompress first
+        if data_tar_name.endswith(".zst"):
+            plain_tar_path = tmpdir_path / "data.tar"
+            subprocess.run(
+                ["zstd", "-d", str(data_tar_path), "-o", str(plain_tar_path)],
+                check=True,
+            )
+            data_tar_path = plain_tar_path
+
+        # Open data.tar (handles .xz, .gz, .bz2 and plain tar)
+        with tarfile.open(data_tar_path, "r:*") as tar:
+            # Find vmlinux file
+            vmlinux_member: Optional[tarfile.TarInfo] = None
+            tar_members: List[tarfile.TarInfo] = tar.getmembers()
+            for tar_entry in tar_members:
+                if tar_entry.name.endswith("/vmlinux") or "/boot/vmlinux-" in tar_entry.name:
+                    if tar_entry.isfile():
+                        vmlinux_member = tar_entry
+                        break
+
+            if vmlinux_member is None:
+                # List all members for debugging
+                members_list = [m.name for m in tar_members][:20]
+                raise ValueError(f"Could not find vmlinux in ddeb. Members: {members_list}")
+
+            # Extract vmlinux
+            vmlinux_member.name = Path(vmlinux_member.name).name  # Flatten path
+            tar.extract(vmlinux_member, path=output_dir)
+            return output_dir / vmlinux_member.name
+
+
+# --- DWARF/Symbol Parsing with dwarf2json ---
+
+
+def run_dwarf2json(vmlinux_path: Path) -> Dict[str, Any]:
+    """Run dwarf2json on vmlinux and return parsed ISF JSON.
+
+    dwarf2json is a Go binary from the Volatility Foundation that extracts
+    DWARF debug info into the Volatility3 Intermediate Symbol Format (ISF).
+
+    Args:
+        vmlinux_path: Path to vmlinux ELF file with debug symbols
+
+    Returns:
+        Dictionary with keys: "symbols", "user_types", "enums", "base_types", "metadata"
+
+    Raises:
+        FileNotFoundError: If dwarf2json binary is not found on PATH
+        RuntimeError: If dwarf2json exits with non-zero status
+    """
+    try:
+        proc = subprocess.Popen(
+            ["dwarf2json", "linux", "--elf", str(vmlinux_path)],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+    except FileNotFoundError:
+        raise FileNotFoundError(
+            "dwarf2json not found on PATH. " "Install from https://github.com/volatilityfoundation/dwarf2json"
+        )
+
+    stdout_bytes, stderr_bytes = proc.communicate()
+
+    if proc.returncode != 0:
+        stderr = stderr_bytes.decode("utf-8", errors="replace")
+        raise RuntimeError(f"dwarf2json failed with exit code {proc.returncode}: {stderr}")
+
+    return json.loads(stdout_bytes)
+
+
+def parse_symbols_for_neo4j(symbols_dict: Dict[str, Dict]) -> List[Dict[str, str]]:
+    """Convert parsed ELF symbols to Neo4j-compatible format.
+
+    Args:
+        symbols_dict: Dictionary from parse_elf_symbols()
+
+    Returns:
+        List of symbol dictionaries with sym_name, address, and hash fields
+    """
+    entries = []
+    for sym, value in sorted(symbols_dict.items()):
+        # Skip internal/compiler symbols
+        if sym.startswith("__"):
+            continue
+
+        address = str(value["address"])
+        entries.append(
+            {
+                "sym_name": sym,
+                "address": address,
+                "hash": hashlib.sha1(address.encode()).hexdigest(),
+            }
+        )
+
+    return entries