astreum 0.2.41__py3-none-any.whl → 0.3.1__py3-none-any.whl

astreum/{_storage/patricia.py → storage/models/trie.py}

@@ -1,14 +1,13 @@
-import blake3
-from typing import Dict, List, Optional, Tuple, TYPE_CHECKING
-
-from .atom import Atom, ZERO32
-
-if TYPE_CHECKING:
-    from .._node import Node
+from typing import Dict, List, Optional, Set, Tuple, TYPE_CHECKING
+
+from .atom import Atom, AtomKind, ZERO32
 
-class PatriciaNode:
+if TYPE_CHECKING:
+    from .._node import Node
+
+class TrieNode:
     """
-    A node in a compressed-key Patricia trie.
+    A node in a compressed-key Binary Radix Tree.
 
     Attributes:
         key_len (int): Number of bits in the `key` prefix that are meaningful.
@@ -31,100 +30,115 @@ class PatriciaNode:
         self.value = value
         self.child_0 = child_0
         self.child_1 = child_1
-        self._hash: Optional[bytes] = None
-
-    def hash(self) -> bytes:
-        """
-        Compute and cache the BLAKE3 hash of this node's serialized form.
-        """
-        if self._hash is None:
-            self._hash = blake3.blake3(self.to_bytes()).digest()
-        return self._hash
-    
-    def to_atoms(self) -> Tuple[bytes, List[Atom]]:
+        self._hash: Optional[bytes] = None
+
+    def hash(self) -> bytes:
         """
-        Materialise this node as a flat atom chain containing the fields in a
-        traversal-friendly order: key length prefix (single byte) + key bytes,
-        child_0 hash, child_1 hash, and value bytes. Returns the head atom hash
-        and the atom list.
+        Compute and cache the canonical hash for this node (its type-atom id).
         """
-        if self.key_len > 255:
-            raise ValueError("Patricia key length > 255 bits cannot be encoded in a single atom field")
+        if self._hash is None:
+            head_hash, _ = self._render_atoms()
+            self._hash = head_hash
+        return self._hash
 
+    def to_bytes(self) -> bytes:
+        """
+        Serialize for hashing: key_len (u16 big-endian) + key payload +
+        child_0 (or ZERO32) + child_1 (or ZERO32) + value.
+        """
+        key_len_bytes = self.key_len.to_bytes(2, "big", signed=False)
+        child0 = self.child_0 or ZERO32
+        child1 = self.child_1 or ZERO32
+        value = self.value or b""
+        return key_len_bytes + self.key + child0 + child1 + value
+    
+    def _render_atoms(self) -> Tuple[bytes, List[Atom]]:
+        """
+        Materialise this node with the canonical atom layout used by the
+        storage layer: a leading SYMBOL atom with payload ``b"trie"`` whose
+        ``next`` pointer links to four BYTES atoms containing, in order:
+        key (len byte + key payload), child_0 hash, child_1 hash, value bytes.
+        Returns the top atom hash and the emitted atoms.
+        """
         entries: List[bytes] = [
-            bytes([self.key_len]) + self.key,
+            self.key_len.to_bytes(2, "big", signed=False) + self.key,
             self.child_0 or ZERO32,
             self.child_1 or ZERO32,
             self.value or b"",
         ]
 
-        atoms: List[Atom] = []
+        data_atoms: List[Atom] = []
         next_hash = ZERO32
         for payload in reversed(entries):
-            atom = Atom.from_data(data=payload, next_hash=next_hash)
-            atoms.append(atom)
+            atom = Atom(data=payload, next_id=next_hash, kind=AtomKind.BYTES)
+            data_atoms.append(atom)
             next_hash = atom.object_id()
 
-        head = next_hash
-        atoms.reverse()
-        return head, atoms
-
-    @classmethod
-    def from_atoms(
-        cls,
-        node: "Node",
-        head_hash: bytes,
-    ) -> "PatriciaNode":
-        """
-        Reconstruct a node from the atom chain rooted at `head_hash`, using the
-        supplied `node` instance to resolve atom object ids.
-        """
-        if head_hash == ZERO32:
-            raise ValueError("empty atom chain for Patricia node")
-
-        entries: List[bytes] = []
-        current = head_hash
-        hops = 0
+        data_atoms.reverse()
 
-        while current != ZERO32 and hops < 4:
-            atom = node._local_get(current)
-            if atom is None:
-                raise ValueError("missing atom while decoding Patricia node")
-            entries.append(atom.data)
-            current = atom.next
-            hops += 1
+        type_atom = Atom(data=b"trie", next_id=next_hash, kind=AtomKind.SYMBOL)
 
-        if current != ZERO32:
-            raise ValueError("too many fields while decoding Patricia node")
+        atoms = data_atoms + [type_atom]
+        return type_atom.object_id(), atoms
 
-        if len(entries) != 4:
-            raise ValueError("incomplete atom sequence for Patricia node")
-
-        key_entry = entries[0]
-        if not key_entry:
+    def to_atoms(self) -> Tuple[bytes, List[Atom]]:
+        head_hash, atoms = self._render_atoms()
+        self._hash = head_hash
+        return head_hash, atoms
+
+    @classmethod
+    def from_atoms(
+        cls,
+        node: "Node",
+        head_hash: bytes,
+    ) -> "TrieNode":
+        """
+        Reconstruct a node from the atom chain rooted at `head_hash`, using the
+        supplied `node` instance to resolve atom object ids.
+        """
+        if head_hash == ZERO32:
+            raise ValueError("empty atom chain for Patricia node")
+
+        atom_chain = node.get_atom_list_from_storage(head_hash)
+        if atom_chain is None or len(atom_chain) != 5:
+            raise ValueError("malformed Patricia atom chain")
+
+        type_atom, key_atom, child0_atom, child1_atom, value_atom = atom_chain
+
+        if type_atom.kind is not AtomKind.SYMBOL:
+            raise ValueError("malformed Patricia node (type atom kind)")
+        if type_atom.data != b"trie":
+            raise ValueError("not a Patricia node (type mismatch)")
+
+        for detail in (key_atom, child0_atom, child1_atom, value_atom):
+            if detail.kind is not AtomKind.BYTES:
+                raise ValueError("Patricia node detail atoms must be bytes")
+
+        key_entry = key_atom.data
+        if len(key_entry) < 2:
             raise ValueError("missing key entry while decoding Patricia node")
-        key_len = key_entry[0]
-        key = key_entry[1:]
-        child_0 = entries[1] if entries[1] != ZERO32 else None
-        child_1 = entries[2] if entries[2] != ZERO32 else None
-        value = entries[3]
-
-        return cls(key_len=key_len, key=key, value=value, child_0=child_0, child_1=child_1)
+        key_len = int.from_bytes(key_entry[:2], "big", signed=False)
+        key = key_entry[2:]
+        child_0 = child0_atom.data if child0_atom.data != ZERO32 else None
+        child_1 = child1_atom.data if child1_atom.data != ZERO32 else None
+        value = value_atom.data
+
+        return cls(key_len=key_len, key=key, value=value, child_0=child_0, child_1=child_1)
 
-class PatriciaTrie:
+class Trie:
     """
-    A compressed-key Patricia trie supporting get and put.
+    A compressed-key Binary Radix Tree supporting get and put.
     """
 
-    def __init__(
-        self,
-        root_hash: Optional[bytes] = None,
-    ) -> None:
-        """
-        :param root_hash: optional hash of existing root node
-        """
-        self.nodes: Dict[bytes, PatriciaNode] = {}
-        self.root_hash = root_hash
+    def __init__(
+        self,
+        root_hash: Optional[bytes] = None,
+    ) -> None:
+        """
+        :param root_hash: optional hash of existing root node
+        """
+        self.nodes: Dict[bytes, TrieNode] = {}
+        self.root_hash = root_hash
 
     @staticmethod
     def _bit(buf: bytes, idx: int) -> bool:
@@ -154,67 +168,113 @@ class PatriciaTrie:
                 return False
         return True
 
-    def _fetch(self, storage_node: "Node", h: bytes) -> Optional[PatriciaNode]:
-        """
-        Fetch a node by hash, consulting the in-memory cache first and falling
-        back to the atom storage provided by `storage_node`.
-        """
-        cached = self.nodes.get(h)
-        if cached is not None:
-            return cached
-
-        if storage_node._local_get(h) is None:
-            return None
-
-        pat_node = PatriciaNode.from_atoms(storage_node, h)
-        self.nodes[h] = pat_node
-        return pat_node
+    def _fetch(self, storage_node: "Node", h: bytes) -> Optional[TrieNode]:
+        """
+        Fetch a node by hash, consulting the in-memory cache first and falling
+        back to the atom storage provided by `storage_node`.
+        """
+        cached = self.nodes.get(h)
+        if cached is not None:
+            return cached
+
+        if storage_node.storage_get(h) is None:
+            return None
+
+        pat_node = TrieNode.from_atoms(storage_node, h)
+        self.nodes[h] = pat_node
+        return pat_node
 
     def get(self, storage_node: "Node", key: bytes) -> Optional[bytes]:
         """
         Return the stored value for `key`, or None if absent.
         """
-        # Empty trie?
-        if self.root_hash is None:
-            return None
+        # Empty trie?
+        if self.root_hash is None:
+            return None
+
+        current = self._fetch(storage_node, self.root_hash)
+        if current is None:
+            return None
+
+        key_pos = 0  # bit offset into key
+
+        while current is not None:
+            # 1) Check that this node's prefix matches the key here
+            if not self._match_prefix(current.key, current.key_len, key, key_pos):
+                return None
+            key_pos += current.key_len
+
+            # 2) If we've consumed all bits of the search key:
+            if key_pos == len(key) * 8:
+                # Return value only if this node actually stores one
+                return current.value
+
+            # 3) Decide which branch to follow via next bit
+            try:
+                next_bit = self._bit(key, key_pos)
+            except IndexError:
+                return None
+
+            child_hash = current.child_1 if next_bit else current.child_0
+            if child_hash is None:
+                return None  # dead end
+
+            # 4) Fetch child and continue descent
+            current = self._fetch(storage_node, child_hash)
+            if current is None:
+                return None  # dangling pointer
+
+            key_pos += 1  # consumed routing bit
 
-        current = self._fetch(storage_node, self.root_hash)
-        if current is None:
-            return None
+        return None
 
-        key_pos = 0  # bit offset into key
+    def get_all(self, storage_node: "Node") -> Dict[bytes, bytes]:
+        """
+        Return a mapping of every key/value pair stored in the trie.
+        """
+        if self.root_hash is None or self.root_hash == ZERO32:
+            return {}
 
-        while current is not None:
-            # 1) Check that this node's prefix matches the key here
-            if not self._match_prefix(current.key, current.key_len, key, key_pos):
-                return None
-            key_pos += current.key_len
+        def _bits_from_payload(payload: bytes, bit_length: int) -> str:
+            if bit_length <= 0 or not payload:
+                return ""
+            bit_stream = "".join(f"{byte:08b}" for byte in payload)
+            return bit_stream[:bit_length]
 
-            # 2) If we've consumed all bits of the search key:
-            if key_pos == len(key) * 8:
-                # Return value only if this node actually stores one
-                return current.value
+        def _bits_to_bytes(bit_string: str) -> bytes:
+            if not bit_string:
+                return b""
+            pad = (8 - (len(bit_string) % 8)) % 8
+            bit_string = bit_string + ("0" * pad)
+            byte_len = len(bit_string) // 8
+            return int(bit_string, 2).to_bytes(byte_len, "big")
 
-            # 3) Decide which branch to follow via next bit
-            try:
-                next_bit = self._bit(key, key_pos)
-            except IndexError:
-                return None
+        results: Dict[bytes, bytes] = {}
+        stack: List[Tuple[bytes, str]] = [(self.root_hash, "")]
+        visited: Set[bytes] = set()
 
-            child_hash = current.child_1 if next_bit else current.child_0
-            if child_hash is None:
-                return None  # dead end
+        while stack:
+            node_hash, prefix_bits = stack.pop()
+            if not node_hash or node_hash == ZERO32 or node_hash in visited:
+                continue
+            visited.add(node_hash)
 
-            # 4) Fetch child and continue descent
-            current = self._fetch(storage_node, child_hash)
-            if current is None:
-                return None  # dangling pointer
+            pat_node = TrieNode.from_atoms(storage_node, node_hash)
+            node_bits = _bits_from_payload(pat_node.key, pat_node.key_len)
+            combined_bits = prefix_bits + node_bits
 
-            key_pos += 1  # consumed routing bit
+            if pat_node.value is not None:
+                key_bytes = _bits_to_bytes(combined_bits)
+                results[key_bytes] = pat_node.value
 
-        return None
+            if pat_node.child_0:
+                stack.append((pat_node.child_0, combined_bits + "0"))
+            if pat_node.child_1:
+                stack.append((pat_node.child_1, combined_bits + "1"))
+
+        return results
 
-    def put(self, storage_node: "Node", key: bytes, value: bytes) -> None:
+    def put(self, storage_node: "Node", key: bytes, value: bytes) -> None:
         """
         Insert or update `key` with `value` in-place.
         """
@@ -227,64 +287,64 @@ class PatriciaTrie:
             return
 
         # S2 – traversal bookkeeping
-        stack: List[Tuple[PatriciaNode, bytes, int]] = []  # (parent, parent_hash, dir_bit)
-        current = self._fetch(storage_node, self.root_hash)
-        assert current is not None
+        stack: List[Tuple[TrieNode, bytes, int]] = []  # (parent, parent_hash, dir_bit)
+        current = self._fetch(storage_node, self.root_hash)
+        assert current is not None
         key_pos = 0
 
         # S4 – main descent loop
         while True:
             # 4.1 – prefix mismatch? → split
-            if not self._match_prefix(current.key, current.key_len, key, key_pos):
-                self._split_and_insert(current, stack, key, key_pos, value)
+            if not self._match_prefix(current.key, current.key_len, key, key_pos):
+                self._split_and_insert(current, stack, key, key_pos, value)
                 return
 
             # 4.2 – consume this prefix
-            key_pos += current.key_len
+            key_pos += current.key_len
 
             # 4.3 – matched entire key → update value
             if key_pos == total_bits:
-                old_hash = current.hash()
-                current.value = value
-                self._invalidate_hash(current)
-                new_hash = current.hash()
-                if new_hash != old_hash:
-                    self.nodes.pop(old_hash, None)
-                self.nodes[new_hash] = current
+                old_hash = current.hash()
+                current.value = value
+                self._invalidate_hash(current)
+                new_hash = current.hash()
+                if new_hash != old_hash:
+                    self.nodes.pop(old_hash, None)
+                self.nodes[new_hash] = current
                 self._bubble(stack, new_hash)
                 return
 
             # 4.4 – routing bit
             next_bit = self._bit(key, key_pos)
-            child_hash = current.child_1 if next_bit else current.child_0
+            child_hash = current.child_1 if next_bit else current.child_0
 
             # 4.6 – no child → easy append leaf
             if child_hash is None:
-                self._append_leaf(current, next_bit, key, key_pos, value, stack)
+                self._append_leaf(current, next_bit, key, key_pos, value, stack)
                 return
 
             # 4.7 – push current node onto stack
-            stack.append((current, current.hash(), int(next_bit)))
+            stack.append((current, current.hash(), int(next_bit)))
 
             # 4.8 – fetch child and continue
-            child = self._fetch(storage_node, child_hash)
-            if child is None:
-                # Dangling pointer: treat as missing child
-                parent, _, _ = stack[-1]
-                self._append_leaf(parent, next_bit, key, key_pos, value, stack[:-1])
-                return
-
-            current = child
-            key_pos += 1  # consumed routing bit
+            child = self._fetch(storage_node, child_hash)
+            if child is None:
+                # Dangling pointer: treat as missing child
+                parent, _, _ = stack[-1]
+                self._append_leaf(parent, next_bit, key, key_pos, value, stack[:-1])
+                return
+
+            current = child
+            key_pos += 1  # consumed routing bit
 
     def _append_leaf(
         self,
-        parent: PatriciaNode,
+        parent: TrieNode,
         dir_bit: bool,
         key: bytes,
         key_pos: int,
         value: bytes,
-        stack: List[Tuple[PatriciaNode, bytes, int]],
+        stack: List[Tuple[TrieNode, bytes, int]],
     ) -> None:
         tail_len = len(key) * 8 - (key_pos + 1)
         tail_bits, tail_len = self._bit_slice(key, key_pos + 1, tail_len)
@@ -307,8 +367,8 @@ class PatriciaTrie:
 
     def _split_and_insert(
         self,
-        node: PatriciaNode,
-        stack: List[Tuple[PatriciaNode, bytes, int]],
+        node: TrieNode,
+        stack: List[Tuple[TrieNode, bytes, int]],
         key: bytes,
         key_pos: int,
         value: bytes,
@@ -381,46 +441,45 @@ class PatriciaTrie:
         value: Optional[bytes],
         child0: Optional[bytes],
         child1: Optional[bytes],
-    ) -> PatriciaNode:
-        node = PatriciaNode(prefix_len, prefix_bits, value, child0, child1)
+    ) -> TrieNode:
+        node = TrieNode(prefix_len, prefix_bits, value, child0, child1)
         self.nodes[node.hash()] = node
         return node
 
-    def _invalidate_hash(self, node: PatriciaNode) -> None:
+    def _invalidate_hash(self, node: TrieNode) -> None:
         """Clear cached hash so next .hash() recomputes."""
         node._hash = None  # type: ignore
 
     def _bubble(
         self,
-        stack: List[Tuple[PatriciaNode, bytes, int]],
+        stack: List[Tuple[TrieNode, bytes, int]],
         new_hash: bytes
     ) -> None:
         """
         Propagate updated child-hash `new_hash` up the ancestor stack,
         rebasing each parent's pointer, invalidating and re-hashing.
         """
-        while stack:
-            parent, old_hash, dir_bit = stack.pop()
-
-            if dir_bit == 0:
-                parent.child_0 = new_hash
+        while stack:
+            parent, old_hash, dir_bit = stack.pop()
+
+            if dir_bit == 0:
+                parent.child_0 = new_hash
             else:
                 parent.child_1 = new_hash
 
             self._invalidate_hash(parent)
             new_hash = parent.hash()
             if new_hash != old_hash:
-                self.nodes.pop(old_hash, None)
-            self.nodes[new_hash] = parent
-            
-        self.root_hash = new_hash
-
-
-    def _bit_slice(
-        self,
-        buf: bytes,
-        start_bit: int,
-        length: int
+                self.nodes.pop(old_hash, None)
+            self.nodes[new_hash] = parent
+            
+        self.root_hash = new_hash
+
+    def _bit_slice(
+        self,
+        buf: bytes,
+        start_bit: int,
+        length: int
     ) -> tuple[bytes, int]:
         """
         Extract `length` bits from `buf` starting at `start_bit` (MSB-first),

astreum/storage/setup.py

@@ -1,15 +1,44 @@
-from pathlib import Path
-from typing import Optional, Dict, Tuple, Any
-
-def storage_setup(config: dict) -> Tuple[Optional[Path], Dict[bytes, Any], int, Dict[bytes, bytes]]:
-    storage_path_str = config.get('storage_path')
-    if storage_path_str is None:
-        storage_path, memory_storage = None, {}
-    else:
-        storage_path = Path(storage_path_str)
-        storage_path.mkdir(parents=True, exist_ok=True)
-        memory_storage = None
-
-    timeout = config.get('storage_get_relay_timeout', 5)
-    storage_index: Dict[bytes, bytes] = {}
-    return storage_path, memory_storage, timeout, storage_index
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+
+def storage_setup(node: Any, config: dict) -> None:
+    """Initialize hot/cold storage helpers on the node."""
+
+    node.logger.info("Setting up node storage")
+
+    node.hot_storage = {}
+    node.hot_storage_hits = {}
+    node.storage_index = {}
+    node.hot_storage_size = 0
+    hot_storage_default_limit = 1 << 30  # 1 GiB
+    hot_storage_limit_value = config.get("hot_storage_limit", hot_storage_default_limit)
+    try:
+        node.hot_storage_limit = int(hot_storage_limit_value)
+    except (TypeError, ValueError):
+        node.hot_storage_limit = hot_storage_default_limit
+
+    node.cold_storage_size = 0
+    cold_storage_default_limit = 10 << 30  # 10 GiB
+    cold_storage_limit_value = config.get("cold_storage_limit", cold_storage_default_limit)
+    try:
+        node.cold_storage_limit = int(cold_storage_limit_value)
+    except (TypeError, ValueError):
+        node.cold_storage_limit = cold_storage_default_limit
+
+    cold_storage_path = config.get("cold_storage_path")
+    if cold_storage_path:
+        try:
+            Path(cold_storage_path).mkdir(parents=True, exist_ok=True)
+        except OSError:
+            cold_storage_path = None
+    node.cold_storage_path = cold_storage_path
+
+    node.logger.info(
+        "Storage ready (hot_limit=%s bytes, cold_limit=%s bytes, cold_path=%s)",
+        node.hot_storage_limit,
+        node.cold_storage_limit,
+        node.cold_storage_path or "disabled",
+    )

astreum/utils/bytes.py

@@ -0,0 +1,24 @@
+from typing import Optional
+
+
+def hex_to_bytes(value: str, *, expected_length: Optional[int] = None) -> bytes:
+    """Convert a 0x-prefixed hex string into raw bytes."""
+    if not isinstance(value, str):
+        raise TypeError("hex value must be provided as a string")
+
+    if not value.startswith(("0x", "0X")):
+        raise ValueError("hex value must start with '0x'")
+
+    hex_digits = value[2:]
+    if len(hex_digits) % 2:
+        raise ValueError("hex value must have an even number of digits")
+
+    try:
+        result = bytes.fromhex(hex_digits)
+    except ValueError as exc:
+        raise ValueError("hex value contains non-hexadecimal characters") from exc
+
+    if expected_length is not None and len(result) != expected_length:
+        raise ValueError(f"hex value must decode to exactly {expected_length} bytes")
+
+    return result

astreum/utils/integer.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from typing import Optional, Union
+
+ByteLike = Union[bytes, bytearray, memoryview]
+
+
+def int_to_bytes(value: Optional[int]) -> bytes:
+    """Convert an integer to a little-endian byte string with minimal length."""
+    if value is None:
+        return b""
+    value = int(value)
+    if value == 0:
+        return b"\x00"
+    length = (value.bit_length() + 7) // 8
+    return value.to_bytes(length, "little", signed=False)
+
+
+def bytes_to_int(data: Optional[ByteLike]) -> int:
+    """Convert a little-endian byte string to an integer."""
+    if not data:
+        return 0
+    if isinstance(data, memoryview):
+        data = data.tobytes()
+    return int.from_bytes(bytes(data), "little", signed=False)