machine-dialect 0.1.0a1__py3-none-any.whl

machine_dialect/codegen/symtab.py

@@ -0,0 +1,208 @@
+"""Symbol table for variable management during code generation.
+
+This module provides scope management and variable resolution for
+local and global variables.
+"""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional
+
+
+class SymbolType(Enum):
+    """Type of symbol in the symbol table."""
+
+    LOCAL = "local"
+    GLOBAL = "global"
+    PARAMETER = "parameter"
+
+
+@dataclass
+class Symbol:
+    """Represents a symbol in the symbol table.
+
+    Attributes:
+        name: The symbol's identifier name.
+        symbol_type: The type of symbol (local, global, or parameter).
+        slot: Slot index for locals/parameters, -1 for globals.
+    """
+
+    name: str
+    symbol_type: SymbolType
+    slot: int  # Slot index for locals/parameters, -1 for globals
+
+
+class Scope:
+    """Represents a lexical scope in the program."""
+
+    def __init__(self, parent: Optional["Scope"] = None, name: str = "global") -> None:
+        """Initialize a new scope.
+
+        Args:
+            parent: Parent scope, None for global scope.
+            name: Name of the scope for debugging.
+        """
+        self.parent = parent
+        self.name = name
+        self.symbols: dict[str, Symbol] = {}
+        self.next_slot = 0  # Next available local slot
+        self.is_global = parent is None
+
+    def define_local(self, name: str) -> Symbol:
+        """Define a new local variable in this scope.
+
+        Args:
+            name: Variable name.
+
+        Returns:
+            The created symbol.
+        """
+        if name in self.symbols:
+            # Variable already exists in this scope
+            return self.symbols[name]
+
+        symbol = Symbol(name, SymbolType.LOCAL, self.next_slot)
+        self.symbols[name] = symbol
+        self.next_slot += 1
+        return symbol
+
+    def define_parameter(self, name: str) -> Symbol:
+        """Define a function parameter in this scope.
+
+        Args:
+            name: Parameter name.
+
+        Returns:
+            The created symbol.
+        """
+        symbol = Symbol(name, SymbolType.PARAMETER, self.next_slot)
+        self.symbols[name] = symbol
+        self.next_slot += 1
+        return symbol
+
+    def define_global(self, name: str) -> Symbol:
+        """Define a global variable.
+
+        Args:
+            name: Variable name.
+
+        Returns:
+            The created symbol.
+        """
+        symbol = Symbol(name, SymbolType.GLOBAL, -1)
+        self.symbols[name] = symbol
+        return symbol
+
+    def resolve(self, name: str) -> Symbol | None:
+        """Resolve a variable name in this scope or parent scopes.
+
+        Args:
+            name: Variable name to resolve.
+
+        Returns:
+            The symbol if found, None otherwise.
+        """
+        # Check current scope
+        if name in self.symbols:
+            return self.symbols[name]
+
+        # Check parent scopes
+        if self.parent:
+            return self.parent.resolve(name)
+
+        return None
+
+    def num_locals(self) -> int:
+        """Get the number of local variables in this scope.
+
+        Returns:
+            Number of local slots used.
+        """
+        return self.next_slot
+
+
+class SymbolTable:
+    """Manages nested scopes and symbol resolution."""
+
+    def __init__(self) -> None:
+        """Initialize with a global scope."""
+        self.global_scope = Scope(name="global")
+        self.current_scope = self.global_scope
+
+    def enter_scope(self, name: str = "block") -> None:
+        """Enter a new nested scope.
+
+        Args:
+            name: Name of the scope for debugging.
+        """
+        new_scope = Scope(parent=self.current_scope, name=name)
+        self.current_scope = new_scope
+
+    def exit_scope(self) -> None:
+        """Exit the current scope and return to parent scope.
+
+        Raises:
+            RuntimeError: If trying to exit global scope.
+        """
+        if self.current_scope.parent is None:
+            raise RuntimeError("Cannot exit global scope")
+        self.current_scope = self.current_scope.parent
+
+    def define(self, name: str, is_parameter: bool = False) -> Symbol:
+        """Define a new variable in the current scope.
+
+        Args:
+            name: Variable name.
+            is_parameter: Whether this is a function parameter.
+
+        Returns:
+            The created symbol.
+        """
+        if self.current_scope.is_global:
+            return self.current_scope.define_global(name)
+        elif is_parameter:
+            return self.current_scope.define_parameter(name)
+        else:
+            return self.current_scope.define_local(name)
+
+    def resolve(self, name: str) -> Symbol | None:
+        """Resolve a variable name starting from current scope.
+
+        Args:
+            name: Variable name to resolve.
+
+        Returns:
+            The symbol if found, None otherwise.
+        """
+        symbol = self.current_scope.resolve(name)
+
+        # If not found anywhere, treat as global
+        if symbol is None and self.current_scope != self.global_scope:
+            # Create implicit global reference
+            symbol = Symbol(name, SymbolType.GLOBAL, -1)
+
+        return symbol
+
+    def is_global_scope(self) -> bool:
+        """Check if currently in global scope.
+
+        Returns:
+            True if in global scope, False otherwise.
+        """
+        return self.current_scope.is_global
+
+    def num_locals(self) -> int:
+        """Get the number of locals in current scope.
+
+        Returns:
+            Number of local slots used in current scope.
+        """
+        return self.current_scope.num_locals()
+
+    def current_scope_name(self) -> str:
+        """Get the name of the current scope.
+
+        Returns:
+            Current scope name.
+        """
+        return self.current_scope.name

machine_dialect/codegen/tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for the codegen module."""

machine_dialect/codegen/tests/test_array_operations_codegen.py

@@ -0,0 +1,295 @@
+"""Unit tests for array operations bytecode generation."""
+
+from machine_dialect.codegen.opcodes import Opcode
+from machine_dialect.codegen.register_codegen import (
+    RegisterAllocation,
+    RegisterBytecodeGenerator,
+)
+from machine_dialect.mir.mir_instructions import (
+    ArrayFindIndex,
+    ArrayInsert,
+    ArrayRemove,
+)
+from machine_dialect.mir.mir_types import MIRType
+from machine_dialect.mir.mir_values import Temp
+
+
+def create_test_generator() -> RegisterBytecodeGenerator:
+    """Create a generator with proper allocation setup."""
+    generator = RegisterBytecodeGenerator(debug=False)
+    # Initialize allocation
+    generator.allocation = RegisterAllocation()
+    generator.allocation.max_registers = 256
+    generator.allocation.next_register = 0
+    generator.allocation.value_to_register = {}
+
+    # Map test temps to registers
+    for i in range(10):
+        temp = Temp(MIRType.INT, i)
+        generator.allocation.value_to_register[temp] = i
+
+    return generator
+
+
+class TestArrayFindIndexCodegen:
+    """Test ArrayFindIndex bytecode generation."""
+
+    def test_generates_loop_structure(self) -> None:
+        """Test that ArrayFindIndex generates a proper loop."""
+        generator = create_test_generator()
+
+        # Create test instruction
+        dest = Temp(MIRType.INT, 0)
+        array = Temp(MIRType.ARRAY, 1)
+        value = Temp(MIRType.INT, 2)
+        inst = ArrayFindIndex(dest, array, value, (1, 1))
+
+        # Generate bytecode
+        generator.generate_array_find_index(inst)
+
+        # Check that bytecode was generated
+        assert len(generator.bytecode) > 0
+
+        # Verify key opcodes are present
+        bytecode = generator.bytecode
+        opcodes_used = []
+        i = 0
+        while i < len(bytecode):
+            if i < len(bytecode):
+                opcodes_used.append(bytecode[i])
+                i += 1
+                # Skip operands (simplified - real parsing would be more complex)
+                if bytecode[i - 1] in [Opcode.LOAD_CONST_R]:
+                    i += 3  # dst + 16-bit const
+                elif bytecode[i - 1] in [Opcode.ARRAY_LEN_R, Opcode.MOVE_R]:
+                    i += 2  # two registers
+                elif bytecode[i - 1] in [Opcode.LT_R, Opcode.EQ_R, Opcode.ADD_R, Opcode.ARRAY_GET_R]:
+                    i += 3  # three registers
+                elif bytecode[i - 1] in [Opcode.JUMP_R]:
+                    i += 4  # 32-bit offset
+                elif bytecode[i - 1] in [Opcode.JUMP_IF_R, Opcode.JUMP_IF_NOT_R]:
+                    i += 5  # register + 32-bit offset
+
+        # Check for essential opcodes
+        assert Opcode.ARRAY_LEN_R in opcodes_used  # Get array length
+        assert Opcode.LT_R in opcodes_used  # Compare index < length
+        assert Opcode.ARRAY_GET_R in opcodes_used  # Get array element
+        assert Opcode.EQ_R in opcodes_used  # Compare element with value
+        assert Opcode.ADD_R in opcodes_used  # Increment index
+
+    def test_labels_are_unique(self) -> None:
+        """Test that multiple ArrayFindIndex operations generate unique labels."""
+        generator = create_test_generator()
+
+        # Generate first find operation
+        inst1 = ArrayFindIndex(Temp(MIRType.INT, 0), Temp(MIRType.ARRAY, 1), Temp(MIRType.INT, 2), (1, 1))
+        generator.generate_array_find_index(inst1)
+        labels1 = set(generator.block_offsets.keys())
+
+        # Generate second find operation
+        inst2 = ArrayFindIndex(Temp(MIRType.INT, 3), Temp(MIRType.ARRAY, 4), Temp(MIRType.INT, 5), (2, 1))
+        generator.generate_array_find_index(inst2)
+        labels2 = set(generator.block_offsets.keys())
+
+        # Labels should be different
+        new_labels = labels2 - labels1
+        assert len(new_labels) > 0  # New labels were added
+
+        # Check label patterns
+        for label in new_labels:
+            assert "find_" in label  # Labels follow naming convention
+
+
+class TestArrayInsertCodegen:
+    """Test ArrayInsert bytecode generation."""
+
+    def test_generates_copy_loops(self) -> None:
+        """Test that ArrayInsert generates copy loops."""
+        generator = create_test_generator()
+
+        # Create test instruction
+        array = Temp(MIRType.ARRAY, 0)
+        index = Temp(MIRType.INT, 1)
+        value = Temp(MIRType.INT, 2)
+        inst = ArrayInsert(array, index, value, (1, 1))
+
+        # Generate bytecode
+        generator.generate_array_insert(inst)
+
+        # Check that bytecode was generated
+        assert len(generator.bytecode) > 0
+
+        # Check for essential operations
+        bytecode = generator.bytecode
+        opcodes_used = []
+        i = 0
+        while i < len(bytecode) - 1:
+            opcodes_used.append(bytecode[i])
+            # Skip to next opcode (simplified)
+            if bytecode[i] in [Opcode.LOAD_CONST_R]:
+                i += 4
+            elif bytecode[i] in [Opcode.ARRAY_LEN_R, Opcode.MOVE_R]:
+                i += 3
+            elif bytecode[i] in [Opcode.NEW_ARRAY_R]:
+                i += 3
+            elif bytecode[i] in [Opcode.ADD_R, Opcode.SUB_R, Opcode.LT_R, Opcode.ARRAY_GET_R, Opcode.ARRAY_SET_R]:
+                i += 4
+            elif bytecode[i] in [Opcode.JUMP_R]:
+                i += 5
+            elif bytecode[i] in [Opcode.JUMP_IF_NOT_R]:
+                i += 6
+            else:
+                i += 1
+
+        # Verify key operations
+        assert Opcode.ARRAY_LEN_R in opcodes_used  # Get original length
+        assert Opcode.ADD_R in opcodes_used  # Calculate new length
+        assert Opcode.NEW_ARRAY_R in opcodes_used  # Create new array
+        assert Opcode.ARRAY_GET_R in opcodes_used  # Copy elements
+        assert Opcode.ARRAY_SET_R in opcodes_used  # Set elements in new array
+        assert Opcode.MOVE_R in opcodes_used  # Replace original array
+
+    def test_handles_position_correctly(self) -> None:
+        """Test that insert position is handled correctly."""
+        generator = create_test_generator()
+
+        # Test with constant position
+        array = Temp(MIRType.ARRAY, 0)
+        index = Temp(MIRType.INT, 1)
+        value = Temp(MIRType.INT, 2)
+        inst = ArrayInsert(array, index, value, (1, 1))
+
+        # Should generate without errors
+        generator.generate_array_insert(inst)
+
+        # Check that labels were created
+        assert any("insert_" in label for label in generator.block_offsets.keys())
+
+
+class TestArrayRemoveCodegen:
+    """Test ArrayRemove bytecode generation."""
+
+    def test_generates_copy_with_skip(self) -> None:
+        """Test that ArrayRemove generates copy loop that skips removed element."""
+        generator = create_test_generator()
+
+        # Create test instruction
+        array = Temp(MIRType.ARRAY, 0)
+        index = Temp(MIRType.INT, 1)
+        inst = ArrayRemove(array, index, (1, 1))
+
+        # Generate bytecode
+        generator.generate_array_remove(inst)
+
+        # Check that bytecode was generated
+        assert len(generator.bytecode) > 0
+
+        # Verify key operations
+        bytecode = generator.bytecode
+        opcodes_used = []
+        i = 0
+        while i < len(bytecode) - 1:
+            opcodes_used.append(bytecode[i])
+            # Skip to next opcode (simplified)
+            if bytecode[i] in [Opcode.LOAD_CONST_R]:
+                i += 4
+            elif bytecode[i] in [Opcode.ARRAY_LEN_R, Opcode.MOVE_R]:
+                i += 3
+            elif bytecode[i] in [Opcode.NEW_ARRAY_R]:
+                i += 3
+            elif bytecode[i] in [
+                Opcode.SUB_R,
+                Opcode.LT_R,
+                Opcode.EQ_R,
+                Opcode.ADD_R,
+                Opcode.ARRAY_GET_R,
+                Opcode.ARRAY_SET_R,
+            ]:
+                i += 4
+            elif bytecode[i] in [Opcode.JUMP_R]:
+                i += 5
+            elif bytecode[i] in [Opcode.JUMP_IF_R, Opcode.JUMP_IF_NOT_R]:
+                i += 6
+            else:
+                i += 1
+
+        # Check for essential operations
+        assert Opcode.ARRAY_LEN_R in opcodes_used  # Get original length
+        assert Opcode.SUB_R in opcodes_used  # Calculate new length (old - 1)
+        assert Opcode.NEW_ARRAY_R in opcodes_used  # Create new array
+        assert Opcode.EQ_R in opcodes_used  # Check if current index is removal point
+        assert Opcode.ARRAY_GET_R in opcodes_used  # Copy elements
+        assert Opcode.ARRAY_SET_R in opcodes_used  # Set elements in new array
+
+    def test_unique_labels(self) -> None:
+        """Test that multiple remove operations use unique labels."""
+        generator = create_test_generator()
+
+        # First remove
+        inst1 = ArrayRemove(Temp(MIRType.ARRAY, 0), Temp(MIRType.INT, 1), (1, 1))
+        generator.generate_array_remove(inst1)
+        labels1 = set(generator.block_offsets.keys())
+
+        # Second remove
+        inst2 = ArrayRemove(Temp(MIRType.ARRAY, 2), Temp(MIRType.INT, 3), (2, 1))
+        generator.generate_array_remove(inst2)
+        labels2 = set(generator.block_offsets.keys())
+
+        # Should have new labels
+        new_labels = labels2 - labels1
+        assert len(new_labels) > 0
+        assert any("remove_" in label for label in new_labels)
+
+
+class TestIntegration:
+    """Integration tests for array operations."""
+
+    def test_combined_operations(self) -> None:
+        """Test that multiple array operations can be generated together."""
+        generator = create_test_generator()
+
+        # Generate a sequence of operations
+        array = Temp(MIRType.ARRAY, 0)
+
+        # Create array
+        # ArrayCreate would normally be called here, but we're testing insert directly
+        # create_inst = ArrayCreate(array, Constant(3, MIRType.INT), (1, 1))
+
+        # Find index
+        find_inst = ArrayFindIndex(Temp(MIRType.INT, 1), array, Temp(MIRType.INT, 2), (2, 1))
+        generator.generate_array_find_index(find_inst)
+
+        # Insert element
+        insert_inst = ArrayInsert(array, Temp(MIRType.INT, 3), Temp(MIRType.INT, 4), (3, 1))
+        generator.generate_array_insert(insert_inst)
+
+        # Remove element
+        remove_inst = ArrayRemove(array, Temp(MIRType.INT, 5), (4, 1))
+        generator.generate_array_remove(remove_inst)
+
+        # Should generate substantial bytecode
+        assert len(generator.bytecode) > 100  # These operations generate lots of code
+
+        # All operations should have their labels
+        labels = generator.block_offsets.keys()
+        assert any("find_" in label for label in labels)
+        assert any("insert_" in label for label in labels)
+        assert any("remove_" in label for label in labels)
+
+    def test_register_allocation(self) -> None:
+        """Test that operations use appropriate temporary registers."""
+        generator = create_test_generator()
+
+        # ArrayFindIndex uses high registers for temps
+        inst = ArrayFindIndex(Temp(MIRType.INT, 0), Temp(MIRType.ARRAY, 1), Temp(MIRType.INT, 2), (1, 1))
+        generator.generate_array_find_index(inst)
+
+        # Check bytecode includes high register numbers (250-254)
+        bytecode = generator.bytecode
+        high_registers_used = False
+        for i in range(len(bytecode)):
+            if bytecode[i] >= 247:  # High register range
+                high_registers_used = True
+                break
+
+        assert high_registers_used, "Should use high temporary registers"

machine_dialect/codegen/tests/test_bytecode_serializer.py

@@ -0,0 +1,185 @@
+"""Tests for bytecode serialization."""
+
+from __future__ import annotations
+
+import struct
+import tempfile
+from pathlib import Path
+
+import pytest
+
+from machine_dialect.codegen.bytecode_serializer import BytecodeWriter, serialize_bytecode_module
+
+
+class TestBytecodeWriter:
+    """Test bytecode writer functionality."""
+
+    def test_create_writer(self) -> None:
+        """Test creating a bytecode writer."""
+        writer = BytecodeWriter()
+        assert writer.module_name == "__main__"
+        assert len(writer.constants) == 0
+        assert len(writer.instructions) == 0
+        assert len(writer.functions) == 0
+        assert len(writer.global_names) == 0
+
+    def test_add_constants(self) -> None:
+        """Test adding constants to the pool."""
+        writer = BytecodeWriter()
+
+        idx1 = writer.add_int_constant(42)
+        idx2 = writer.add_float_constant(3.14)
+        idx3 = writer.add_string_constant("hello")
+        idx4 = writer.add_bool_constant(True)
+        idx5 = writer.add_empty_constant()
+
+        assert idx1 == 0
+        assert idx2 == 1
+        assert idx3 == 2
+        assert idx4 == 3
+        assert idx5 == 4
+        assert len(writer.constants) == 5
+
+        assert writer.constants[0] == (0x01, 42)
+        assert writer.constants[1] == (0x02, 3.14)
+        assert writer.constants[2] == (0x03, "hello")
+        assert writer.constants[3] == (0x04, True)
+        assert writer.constants[4] == (0x05, None)
+
+    def test_emit_instructions(self) -> None:
+        """Test emitting various instructions."""
+        writer = BytecodeWriter()
+
+        # Add some constants
+        const_idx = writer.add_int_constant(100)
+
+        # Emit instructions
+        writer.emit_load_const(0, const_idx)
+        writer.emit_move(1, 0)
+        writer.emit_add(2, 0, 1)
+        writer.emit_return(2)
+
+        assert len(writer.instructions) == 4
+
+        # Check instruction encoding
+        assert writer.instructions[0] == struct.pack("<BBH", 0, 0, const_idx)
+        assert writer.instructions[1] == struct.pack("<BBB", 1, 1, 0)
+        assert writer.instructions[2] == struct.pack("<BBBB", 7, 2, 0, 1)
+        assert writer.instructions[3] == struct.pack("<BBB", 26, 1, 2)
+
+    def test_write_bytecode_file(self) -> None:
+        """Test writing bytecode to a file."""
+        writer = BytecodeWriter()
+        writer.set_module_name("test_module")
+
+        # Add some constants and instructions
+        idx = writer.add_int_constant(42)
+        writer.emit_load_const(0, idx)
+        writer.emit_return(0)
+
+        # Write to temporary file
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = Path(tmpdir) / "test"
+            writer.write_to_file(path)
+
+            # Check file was created
+            bytecode_path = path.with_suffix(".mdbc")
+            assert bytecode_path.exists()
+
+            # Read and verify header
+            with open(bytecode_path, "rb") as f:
+                magic = f.read(4)
+                assert magic == b"MDBC"
+
+                version = struct.unpack("<I", f.read(4))[0]
+                assert version == 1
+
+                flags = struct.unpack("<I", f.read(4))[0]
+                assert flags == 1  # Little-endian flag
+
+    def test_serialize_module(self) -> None:
+        """Test serialize_bytecode_module function."""
+        constants = [
+            (0x01, 100),
+            (0x03, "test"),
+            (0x04, False),
+        ]
+
+        instructions = [
+            struct.pack("<BBH", 0, 0, 0),  # LoadConstR r0, 0
+            struct.pack("<BBH", 0, 1, 1),  # LoadConstR r1, 1
+            struct.pack("<BB", 26, 0),  # ReturnR (no value)
+        ]
+
+        data = serialize_bytecode_module(
+            "my_module",
+            constants,
+            instructions,
+            functions={"main": 0},
+            global_names=["x", "y"],
+        )
+
+        # Verify magic number
+        assert data[:4] == b"MDBC"
+
+        # Verify version
+        version = struct.unpack("<I", data[4:8])[0]
+        assert version == 1
+
+    def test_bytecode_format(self) -> None:
+        """Test the complete bytecode format."""
+        writer = BytecodeWriter()
+        writer.set_module_name("format_test")
+
+        # Add various constants
+        int_idx = writer.add_int_constant(999)
+        float_idx = writer.add_float_constant(2.718)
+        writer.add_string_constant("bytecode")
+        writer.add_bool_constant(False)
+
+        # Add global names
+        x_idx = writer.add_global_name("x")
+        y_idx = writer.add_global_name("y")
+
+        # Add instructions
+        writer.emit_load_const(0, int_idx)
+        writer.emit_store_global(0, x_idx)
+        writer.emit_load_const(1, float_idx)
+        writer.emit_store_global(1, y_idx)
+        writer.emit_load_global(2, x_idx)
+        writer.emit_load_global(3, y_idx)
+        writer.emit_add(4, 2, 3)
+        writer.emit_return(4)
+
+        # Get serialized data
+        from io import BytesIO
+
+        stream = BytesIO()
+        writer.write_to_stream(stream)
+        data = stream.getvalue()
+
+        # Parse and verify structure
+        assert len(data) > 28  # At least header size
+
+        # Check header
+        assert data[0:4] == b"MDBC"
+        version = struct.unpack("<I", data[4:8])[0]
+        assert version == 1
+
+        flags = struct.unpack("<I", data[8:12])[0]
+        assert flags == 1
+
+        # Check offsets are present
+        name_offset = struct.unpack("<I", data[12:16])[0]
+        const_offset = struct.unpack("<I", data[16:20])[0]
+        func_offset = struct.unpack("<I", data[20:24])[0]
+        inst_offset = struct.unpack("<I", data[24:28])[0]
+
+        assert name_offset == 28  # Right after header
+        assert const_offset > name_offset
+        assert func_offset > const_offset
+        assert inst_offset > func_offset
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])