oswatcher-plugins 0.14.0__py3-none-any.whl

plugins/plugins/symbols_repository.py

@@ -0,0 +1,142 @@
+"""Repository layer for symbols plugin Neo4j operations."""
+
+from pathlib import PurePath
+from typing import Dict, List, Tuple
+
+from neogit.service.neogit import cypher_query_with_backoff
+
+
+class SymbolsRepository:
+    """Handles Neo4j database operations for symbols plugin."""
+
+    def __init__(self, neogit):
+        """Initialize repository with neogit service.
+
+        Args:
+            neogit: Neogit service instance for database access
+        """
+        self.neogit = neogit
+
+    def query_pe_blobs(self, root_hash: str, mime_type: str) -> List[Tuple[PurePath, str]]:
+        """Query for PE file blobs matching MIME type.
+
+        Args:
+            root_hash: Hash of the root Tree node
+            mime_type: MIME type to filter by
+
+        Returns:
+            List of (file_path, blob_hash) tuples
+        """
+        query = """
+        MATCH path = (r:Tree {hash: $root_hash})-[:HAS_CHILD_TREE|HAS_CHILD_BLOB*]->(b:Blob)
+        WHERE EXISTS {
+                MATCH (b)-[:HAS_MIME_TYPE]->(m:MimeType)
+                WHERE m.mime = $mime_type
+            }
+        RETURN [rel IN relationships(path) | rel.name] AS parts, b.hash
+        """
+        rows, _ = self.neogit.db.cypher_query(query, {"mime_type": mime_type, "root_hash": root_hash})
+        return [(PurePath(*row[0]), row[1]) for row in rows]
+
+    def insert_symbols(self, blob_hash: str, param_list: List[Dict]) -> None:
+        """Insert symbols into Neo4j.
+
+        Args:
+            blob_hash: Hash of the PE file blob
+            param_list: List of dicts with hash, address, and symbol name key
+                (`sym_name` or `name`)
+        """
+        query = """
+        MATCH (b:Blob {hash: $blob_hash})
+        WITH b
+        UNWIND $unwind as p
+        MERGE (s:Symbol {hash: p.hash, address: p.address})
+        MERGE (b)-[:HAS_SYMBOL {name: coalesce(p.sym_name, p.name)}]->(s)
+        """
+        cypher_query_with_backoff(query, {"blob_hash": blob_hash, "unwind": param_list})
+
+    def insert_struct(self, blob_hash: str, struct_node, unwind_param: List[Dict]) -> None:
+        """Insert Windows struct definition into Neo4j.
+
+        Args:
+            blob_hash: Hash of the PE file blob
+            struct_node: StructMerkleNode with hash, size, kind, name
+            unwind_param: List of field dicts with hash, name, offset, data_type
+        """
+        query = """
+        MERGE (s:Struct {hash: $hash, size: $size, kind: $kind})
+        WITH s
+        UNWIND $unwind_param as x
+        MERGE (f:StructField {hash: x.hash, offset: x.offset, data_type: x.data_type})
+        MERGE (s)-[:HAS_FIELD {name: x.name}]->(f)
+        WITH s
+        MATCH (b:Blob {hash: $blob_hash})
+        WITH b, s
+        MERGE (b)-[:HAS_STRUCT {name: $name}]->(s)
+        """
+        cypher_query_with_backoff(
+            query,
+            {
+                "blob_hash": blob_hash,
+                "unwind_param": unwind_param,
+                "hash": struct_node.hash,
+                "name": struct_node.name,
+                "size": struct_node.size,
+                "kind": struct_node.kind.name,
+            },
+        )
+
+    def insert_data_type(self, node) -> None:
+        """Insert Windows data type into Neo4j.
+
+        Args:
+            node: DataTypeMerkleNode with type metadata
+        """
+        query = """
+        MERGE (d:DataType {hash: $hash})  // Ensure 'hash' uniquely identifies 'DataType'
+        ON CREATE SET
+            d.type = CASE WHEN $type IS NOT NULL THEN $type END,
+            d.name = CASE WHEN $name IS NOT NULL THEN $name END,
+            d.array_counter = CASE WHEN $array_counter IS NOT NULL THEN $array_counter END,
+            d.bit_position = CASE WHEN $bit_position IS NOT NULL THEN $bit_position END,
+            d.bit_length = CASE WHEN $bit_length IS NOT NULL THEN $bit_length END
+        ON MATCH SET
+            d.type = CASE WHEN $type IS NOT NULL THEN $type END,
+            d.name = CASE WHEN $name IS NOT NULL THEN $name END,
+            d.array_counter = CASE WHEN $array_counter IS NOT NULL THEN $array_counter END,
+            d.bit_position = CASE WHEN $bit_position IS NOT NULL THEN $bit_position END,
+            d.bit_length = CASE WHEN $bit_length IS NOT NULL THEN $bit_length END
+        WITH d
+        UNWIND $children AS child
+        MERGE (c:DataType {hash: child.hash})  // Assuming 'hash' is unique for child nodes too
+        ON CREATE SET
+            c.type = CASE WHEN child.type IS NOT NULL THEN child.type END,
+            c.name = CASE WHEN child.name IS NOT NULL THEN child.name END,
+            c.array_counter = CASE WHEN child.array_counter IS NOT NULL THEN child.array_counter END,
+            c.bit_position = CASE WHEN child.bit_position IS NOT NULL THEN child.bit_position END,
+            c.bit_length = CASE WHEN child.bit_length IS NOT NULL THEN child.bit_length END
+        MERGE (d)-[:HAS_DATA_TYPE]->(c)
+        """
+        children = [
+            {
+                "hash": x.hash,
+                "type": x.kind.name,
+                "name": x.name,
+                "array_counter": x.array_counter,
+                "bit_position": x.bit_position,
+                "bit_length": x.bit_length,
+            }
+            for hash, x in node.children.items()
+        ]
+        cypher_query_with_backoff(
+            query,
+            {
+                "hash": node.hash,
+                "type": node.kind.name,
+                "name": node.name,
+                "array_counter": node.array_counter,
+                "bit_position": node.bit_position,
+                "bit_length": node.bit_length,
+                "children": children,
+            },
+        )

plugins/plugins/symbols_service.py

@@ -0,0 +1,47 @@
+"""Pure functions for symbols plugin business logic."""
+
+import hashlib
+from pathlib import PurePath
+from typing import Dict, List, Tuple
+
+
+def filter_valid_filenames(
+    blob_results: List[Tuple[PurePath, str]], allowed_filenames: List[str]
+) -> List[Tuple[PurePath, str]]:
+    """Filter blob results to only include allowed filenames.
+
+    This is a pure function with no side effects.
+
+    Args:
+        blob_results: List of (path, blob_hash) tuples
+        allowed_filenames: List of allowed filenames to keep
+
+    Returns:
+        Filtered list of (path, blob_hash) tuples
+    """
+    return [(path, blob_hash) for path, blob_hash in blob_results if path.name in allowed_filenames]
+
+
+def parse_symbols_from_json(symbols_dict: Dict) -> List[Dict[str, str]]:
+    """Parse symbols from PDB JSON, filtering mangled names.
+
+    This is a pure function with no side effects.
+
+    Filters out symbols starting with '?' or '$' (compiler-generated).
+
+    Args:
+        symbols_dict: Dictionary mapping symbol names to symbol data
+
+    Returns:
+        List of symbol dictionaries with name, address, and hash fields
+    """
+    entries = []
+    for sym, value in sorted(symbols_dict.items()):
+        # Skip mangled/compiler symbols
+        if sym.startswith("?") or sym.startswith("$"):
+            continue
+
+        address = str(value["address"])
+        entries.append({"name": sym, "address": address, "hash": hashlib.sha1(address.encode()).hexdigest()})
+
+    return entries

plugins/plugins/syscalls.py

@@ -0,0 +1,180 @@
+"""Linux syscall extraction plugin using kernel filesystem analysis and Git repository."""
+
+from typing import Dict, List
+
+import appdirs
+from attrs import define
+from neogit.model.neo import Commit
+
+from plugins.syscalls.exceptions import KernelVersionNotFoundError, PreKernel2011Error, SyscallFileNotFoundError
+from plugins.syscalls.filesystem import KernelInfo, find_kernel_versions, get_boot_directory
+from plugins.syscalls.kernel_repo_manager import ensure_kernel_repo, get_syscall_files
+from plugins.syscalls.nodes import SyscallsMerkleVisitor, SyscallTableNode
+from plugins.syscalls.syscall_table_parser import parse_syscall_table_line
+from plugins.syscalls.syscalls_h_parser import parse_syscall_signature
+from plugins.types import AbstractPlugin
+
+
+@define(auto_attribs=True)
+class SyscallsPlugin(AbstractPlugin):
+    """Plugin to extract Linux syscall information from kernel files.
+
+    This plugin:
+    1. Navigates to /boot directory using new Tree API
+    2. Finds vmlinuz kernel files
+    3. Parses kernel versions
+    4. Fetches syscall information from Linux kernel Git repository
+    5. Extracts syscall signatures with parameters
+    """
+
+    def run(self, commit: Commit):
+        """Execute syscall extraction for a commit.
+
+        Args:
+            commit: The Commit node to analyze
+        """
+        self.logger.info(f"Running syscall plugin for commit {commit.hash}")
+
+        # Get the root filesystem tree
+        try:
+            root_tree = commit.filesystem[0]
+        except IndexError:
+            self.logger.warning(f"No filesystem found for commit {commit.hash}")
+            return
+
+        # Navigate to /boot directory - using public function
+        boot_tree = get_boot_directory(root_tree)
+        if not boot_tree:
+            self.logger.info("No /boot directory found, skipping syscall extraction")
+            return
+
+        # Find kernel versions from vmlinuz files - using public function
+        kernel_info_list = find_kernel_versions(boot_tree, self)
+        if not kernel_info_list:
+            self.logger.info("No kernel files found in /boot")
+            return
+
+        self.logger.info(
+            f"Found {len(kernel_info_list)} kernel(s): "
+            + ", ".join(f"{k.filename} ({k.architecture})" for k in kernel_info_list)
+        )
+
+        # Extract syscalls from Linux kernel repository
+        syscall_data = self._extract_syscalls_from_repo(kernel_info_list)
+
+        # Transform to Nodes and visit with MerkleVisitor
+        self._transform_and_visit(kernel_info_list, syscall_data)
+
+        self.logger.info(f"Syscall extraction complete for commit {commit.hash}")
+
+    def _extract_syscalls_from_repo(self, kernel_info_list: List[KernelInfo]) -> Dict[str, List[dict]]:
+        """Extract syscall information from Linux kernel Git repository.
+
+        Args:
+            kernel_info_list: List of kernel information to extract
+
+        Returns:
+            Dictionary mapping kernel version to list of syscall data
+        """
+        syscall_data = {}
+        cache_dir = appdirs.user_cache_dir("oswatcher-plugins")
+        self.logger.info(f"Cloning/updating Linux kernel repository to {cache_dir}")
+        repo = ensure_kernel_repo(cache_dir)
+
+        # Extract syscalls for each kernel version
+        for kernel_info in kernel_info_list:
+            version = kernel_info.version
+            try:
+                syscalls = self._extract_version_syscalls(repo, version)
+                if syscalls:
+                    syscall_data[version] = syscalls
+                    self.logger.info(f"Extracted {len(syscalls)} syscalls for {version} ({kernel_info.architecture})")
+            except PreKernel2011Error as e:
+                self.logger.warning(f"Skipping {version}: {e}")
+            except KernelVersionNotFoundError as e:
+                self.logger.warning(f"Version {version} not found in repository: {e}")  # noqa: E713
+            except SyscallFileNotFoundError as e:
+                self.logger.warning(f"Syscall files not found for {version}: {e}")
+            except Exception as e:
+                self.logger.error(f"Failed to extract syscalls for {version}: {e}")
+
+        return syscall_data
+
+    def _extract_version_syscalls(self, repo, version: str) -> List[dict]:
+        """Extract syscall data for a specific kernel version.
+
+        Args:
+            repo: Git repository object
+            version: Kernel version like 'v5.15'
+
+        Returns:
+            List of dictionaries with syscall information
+        """
+        # Get syscall table and header files from repository
+        table_content, header_content = get_syscall_files(repo, version)
+
+        syscalls = []
+
+        # Parse syscall table
+        for line in table_content.splitlines():
+            syscall_index = parse_syscall_table_line(line)
+            if syscall_index:
+                # Get signature from header file
+                entry_name = f"sys_{syscall_index.name}"
+                signature = parse_syscall_signature(header_content, entry_name)
+
+                syscall_info = {
+                    "name": syscall_index.name,
+                    "index": syscall_index.index,
+                    "entry_point": entry_name,
+                }
+
+                if signature:
+                    syscall_info["parameters"] = signature.parameters
+                else:
+                    syscall_info["parameters"] = None
+                    self.logger.debug(f"No signature found for {entry_name}")
+
+                syscalls.append(syscall_info)
+
+        return syscalls
+
+    def _transform_and_visit(self, kernel_info_list: List[KernelInfo], syscall_data: Dict[str, List[dict]]):
+        """Transform syscall data to Nodes and visit with MerkleVisitor.
+
+        Args:
+            kernel_info_list: List of kernel information
+            syscall_data: Dictionary mapping kernel version to syscall list
+        """
+        if not syscall_data:
+            self.logger.info("No syscall data to transform")
+            return
+
+        # Create visitor (threaded for async processing)
+        with SyscallsMerkleVisitor(thread=True) as visitor:
+            # Create SyscallTableNode for each kernel
+            for kernel_info in kernel_info_list:
+                version_data = syscall_data.get(kernel_info.version)
+                if not version_data:
+                    continue
+
+                # Create SyscallTableNode
+                table_node = SyscallTableNode(architecture=kernel_info.architecture, syscalls=version_data)
+
+                # Visit to compute hashes
+                self.logger.info(
+                    f"Creating SyscallTableNode for {kernel_info.version} "
+                    f"({kernel_info.architecture}) with {len(version_data)} syscalls"
+                )
+                visitor.run_visit(table_node)
+
+            # Iterate visited nodes (includes both SyscallMerkleNodes and SyscallTableMerkleNodes)
+            for visited_node in visitor.as_gen():
+                merkle_node = visited_node.return_value
+                # Only log SyscallTableMerkleNodes (skip individual syscall children)
+                if hasattr(merkle_node, "architecture"):
+                    self.logger.info(
+                        f"Created SyscallTableMerkleNode: hash={merkle_node.hash[:8]}... "
+                        f"arch={merkle_node.architecture} children={len(merkle_node.children)}"
+                    )
+                    # TODO: Insert into Neo4j (next phase)

plugins/syscalls/exceptions.py

@@ -0,0 +1,25 @@
+"""Syscall extraction specific exceptions."""
+
+
+class SyscallExtractionError(Exception):
+    """Base exception for syscall extraction errors."""
+
+    pass
+
+
+class KernelVersionNotFoundError(SyscallExtractionError):
+    """Kernel version does not exist in the repository."""
+
+    pass
+
+
+class PreKernel2011Error(SyscallExtractionError):
+    """Kernel version predates 2011 syscall table format (not supported)."""
+
+    pass
+
+
+class SyscallFileNotFoundError(SyscallExtractionError):
+    """Required syscall files not found for this kernel version."""
+
+    pass

plugins/syscalls/filesystem.py

@@ -0,0 +1,121 @@
+"""Filesystem navigation utilities for Linux kernel analysis."""
+
+from dataclasses import dataclass
+from pathlib import PurePath
+from typing import TYPE_CHECKING, List, Optional
+
+import lief
+from neogit.model.merkle import Blob, Tree
+
+if TYPE_CHECKING:
+    from plugins.types import AbstractPlugin
+
+
+@dataclass(frozen=True)
+class KernelInfo:
+    """Information about a kernel found in /boot directory."""
+
+    version: str  # e.g., "v5.15"
+    blob_hash: str  # Neo4j Blob hash
+    filename: str  # e.g., "vmlinuz-5.15.0-91-generic"
+    architecture: str  # e.g., "x86_64", "AARCH64" (from lief enum)
+
+
+def detect_kernel_arch(vmlinuz_path: str) -> str:
+    """Detect kernel architecture using lief parser.
+
+    Handles both raw ELF kernels (vmlinux) and compressed vmlinuz files
+    with an EFI boot stub (PE header).
+
+    Args:
+        vmlinuz_path: Path to vmlinuz file (local filesystem)
+
+    Returns:
+        Architecture string from lief enum (e.g., "ARCH.x86_64", "MACHINE_TYPES.AMD64")
+
+    Raises:
+        ValueError: If file cannot be parsed or architecture cannot be determined
+    """
+    binary = lief.parse(vmlinuz_path)
+
+    if binary is None:
+        raise ValueError(f"Failed to parse {vmlinuz_path}")
+
+    # Raw ELF kernel (e.g., vmlinux)
+    if isinstance(binary, lief.ELF.Binary):
+        return str(binary.header.machine_type)
+
+    # Compressed vmlinuz with EFI boot stub (PE header)
+    if isinstance(binary, lief.PE.Binary):
+        return str(binary.header.machine)
+
+    raise ValueError(f"Unsupported binary format: {vmlinuz_path}")
+
+
+def get_boot_directory(root: Tree) -> Optional[Tree]:
+    """Navigate to /boot directory.
+
+    Args:
+        root: Root filesystem tree
+
+    Returns:
+        Tree node for /boot directory, or None if not found or not a directory
+    """
+    try:
+        boot = root.get_child_at_path(PurePath("/boot"))
+        if isinstance(boot, Tree):
+            return boot
+    except FileNotFoundError:
+        pass
+    return None
+
+
+def find_kernel_versions(boot: Tree, plugin: "AbstractPlugin") -> List[KernelInfo]:
+    """Find kernel versions from /boot directory contents.
+
+    Args:
+        boot: /boot directory tree
+        plugin: Plugin instance (for downloading blobs)
+
+    Returns:
+        Sorted list of unique kernel information (version, hash, filename, architecture)
+    """
+    from plugins.syscalls.kernel_parser import parse_kernel_version
+
+    kernel_infos = []
+    seen_versions = set()
+
+    for name, child in boot.iter_children():
+        if not isinstance(child, Blob) or not name.startswith("vmlinuz-"):
+            continue
+
+        try:
+            version = parse_kernel_version(name)
+
+            # Skip duplicate versions (multiple builds of same version)
+            if version in seen_versions:
+                continue
+
+            # Download blob and detect architecture
+            with plugin.downloaded_file(child.hash) as vmlinuz_path:
+                try:
+                    architecture = detect_kernel_arch(vmlinuz_path)
+                except ValueError as e:
+                    plugin.logger.warning(f"Failed to detect architecture for {name}: {e}")
+                    continue
+
+            kernel_info = KernelInfo(
+                version=version,
+                blob_hash=child.hash,
+                filename=name,
+                architecture=architecture,
+            )
+            kernel_infos.append(kernel_info)
+            seen_versions.add(version)
+
+        except ValueError:
+            # Skip files that don't parse as valid kernel versions
+            continue
+
+    # Sort by version for deterministic ordering
+    return sorted(kernel_infos, key=lambda k: k.version)

plugins/syscalls/kernel_parser.py

@@ -0,0 +1,36 @@
+"""Kernel version and syscall parsing from boot filenames and headers."""
+
+import re
+from dataclasses import dataclass
+
+# Compiled regex pattern for kernel filename parsing
+# Pattern: vmlinuz-{major}.{minor}.{patch}-{build}-{flavor}
+KERNEL_VERSION_PATTERN = re.compile(r"^vmlinuz-(\d+)\.(\d+)\..*")
+
+
+@dataclass(frozen=True)
+class SyscallIndex:
+    """Represents a syscall with its name and index."""
+
+    name: str
+    index: int
+
+
+def parse_kernel_version(filename: str) -> str:
+    """Parse kernel version from vmlinuz filename.
+
+    Args:
+        filename: Boot filename like 'vmlinuz-5.15.0-91-generic'
+
+    Returns:
+        Kernel version like 'v5.15'
+
+    Raises:
+        ValueError: If filename format is invalid
+    """
+    match = KERNEL_VERSION_PATTERN.match(filename)
+    if not match:
+        raise ValueError(f"Invalid kernel filename format: {filename}")
+
+    major, minor = match.groups()
+    return f"v{major}.{minor}"

plugins/syscalls/kernel_repo_manager.py

@@ -0,0 +1,106 @@
+"""Kernel repository management using git show for blob extraction."""
+
+from pathlib import Path
+
+import git
+from git.exc import GitCommandError
+
+from .exceptions import KernelVersionNotFoundError, PreKernel2011Error, SyscallFileNotFoundError
+
+
+def ensure_kernel_repo(cache_dir: str) -> git.Repo:
+    """Ensure Linux kernel repository exists in cache directory.
+
+    Args:
+        cache_dir: Cache directory path
+
+    Returns:
+        Git repository object
+    """
+    cache_path = Path(cache_dir)
+    linux_path = cache_path / "linux"
+
+    if linux_path.exists() and (linux_path / ".git").exists():
+        # Repository already exists, open it and fetch latest tags
+        repo = git.Repo(linux_path)
+        # Fetch latest tags
+        repo.remotes.origin.fetch(tags=True)
+        return repo
+    else:
+        # Clone the repository
+        repo = git.Repo.clone_from("https://github.com/torvalds/linux.git", linux_path)
+        # Fetch tags (clone doesn't fetch tags by default in some Git versions)
+        repo.remotes.origin.fetch(tags=True)
+        return repo
+
+
+def get_file_content(repo: git.Repo, version: str, file_path: str) -> str:
+    """Get file content at specific version using git show.
+
+    Args:
+        repo: Git repository object
+        version: Git tag/commit like 'v5.15'
+        file_path: File path relative to repo root
+
+    Returns:
+        File content as string
+
+    Raises:
+        KernelVersionNotFoundError: If kernel version doesn't exist
+        SyscallFileNotFoundError: If file doesn't exist at that version
+    """
+    try:
+        return repo.git.show(f"{version}:{file_path}")  # noqa: E231
+    except GitCommandError as e:
+        error_str = str(e)
+        # Check for invalid revision/version
+        if "bad revision" in error_str.lower() or "unknown revision" in error_str.lower():
+            raise KernelVersionNotFoundError(f"Kernel version {version} not found in repository")  # noqa: E713
+        # Check for path not found in tree
+        elif "path" in error_str and "not in" in error_str:
+            raise SyscallFileNotFoundError(f"File {file_path} not found in version {version}")  # noqa: E713
+        # Check for other "does not exist" errors
+        elif "does not exist" in error_str:
+            if version in error_str:
+                raise KernelVersionNotFoundError(f"Kernel version {version} not found in repository")  # noqa: E713
+            else:
+                raise SyscallFileNotFoundError(f"File {file_path} not found in version {version}")  # noqa: E713
+        # Re-raise original error if we can't classify it
+        raise
+
+
+def get_syscall_files(repo: git.Repo, version: str) -> tuple[str, str]:
+    """Get syscall table and header file contents for a kernel version.
+
+    Args:
+        repo: Git repository object
+        version: Kernel version like 'v5.15'
+
+    Returns:
+        Tuple of (table_content, header_content)
+
+    Raises:
+        PreKernel2011Error: If kernel predates 2011 syscall table format
+        KernelVersionNotFoundError: If kernel version doesn't exist
+        SyscallFileNotFoundError: If syscall files don't exist
+    """
+    # Try post-2011 location first
+    table_path = "arch/x86/entry/syscalls/syscall_64.tbl"
+    header_path = "include/linux/syscalls.h"
+
+    try:
+        table_content = get_file_content(repo, version, table_path)
+        header_content = get_file_content(repo, version, header_path)
+        return table_content, header_content
+    except SyscallFileNotFoundError as e:
+        # Check if it's a pre-2011 kernel (no .tbl files)
+        if table_path in str(e):
+            try:
+                # Try to get syscalls.h to see if the version exists
+                get_file_content(repo, version, header_path)
+                # If syscalls.h exists but .tbl doesn't, it's pre-2011
+                raise PreKernel2011Error(f"Kernel {version} predates 2011 syscall table format")
+            except (SyscallFileNotFoundError, KernelVersionNotFoundError):
+                # Neither file exists - version might be invalid
+                raise KernelVersionNotFoundError(f"Kernel version {version} not found")
+        raise