astreum 0.1.14__py3-none-any.whl → 0.1.16__py3-none-any.whl

astreum/node/{models.py → storage/storage.py}

@@ -1,15 +1,16 @@
-import socket
-from pathlib import Path
-from typing import Optional, Tuple, Dict
-from astreum.machine import AstreumMachine
-from .relay import Relay
-from .relay.message import Topic
-from .relay.route import RouteTable
-from .relay.peer import Peer
-import os
-import struct
+"""
+Storage implementation for the Astreum node.
+
+This module provides a Storage class that manages persistent storage
+of data using either in-memory dictionaries or file system storage.
+It supports network-based retrieval for missing data.
+"""
+
 import threading
 import time
+from pathlib import Path
+from typing import Optional, Dict, Set, Tuple, List, Any
+
 
 class Storage:
     def __init__(self, config: dict):
@@ -212,98 +213,39 @@ class Storage:
             # Try to get the object (which may start a network request)
             obj_data = self.get(current_hash, timeout)
             if obj_data is None:
-                # Failed to get this object, but we continue with the rest
-                print(f"Warning: Failed to get object {current_hash.hex()}")
+                # Object not found, continue with other objects
                 continue
                 
-            # Store the object in our result
+            # Store the object
             objects[current_hash] = obj_data
             
-            # Only process non-leaf nodes for recursion
-            try:
-                # Extract leaf flag and type
-                is_leaf = struct.unpack("?", obj_data[0:1])[0]
-                if is_leaf:
-                    # Leaf node, no need to recurse
-                    continue
-                    
-                type_indicator = obj_data[1:2]
-                next_depth = current_depth + 1
-                
-                if type_indicator == b'L':  # List
-                    # Non-leaf list has child element hashes
-                    elements_bytes = obj_data[2:]
-                    element_hashes = [elements_bytes[i:i+32] for i in range(0, len(elements_bytes), 32)]
-                    
-                    # Add each element hash to the queue
-                    for elem_hash in element_hashes:
-                        pending_queue.append((elem_hash, next_depth))
-                        
-                elif type_indicator == b'F':  # Function
-                    # Non-leaf function has body hash
-                    remaining_bytes = obj_data[2:]
-                    
-                    # Find the separator between params and body hash
-                    params_end = remaining_bytes.find(b',', remaining_bytes.rfind(b','))
-                    if params_end == -1:
-                        params_end = 0  # No params
-                        
-                    body_hash = remaining_bytes[params_end+1:]
-                    
-                    # Add body hash to the queue
-                    pending_queue.append((body_hash, next_depth))
-                    
-            except Exception as e:
-                print(f"Error processing object {current_hash.hex()}: {e}")
-                continue
-                
+            # Queue child objects if not at max depth
+            if current_depth < max_depth:
+                # Try to detect child objects in the data
+                # This depends on the data format, so this is just a placeholder
+                # In a real implementation, you would parse the data based on its format
+                # and extract references to other objects
+                child_hashes = self._extract_child_hashes(obj_data)
+                for child_hash in child_hashes:
+                    pending_queue.append((child_hash, current_depth + 1))
+        
         return objects
-
-class Account:
-    def __init__(self, public_key: bytes, balance: int, counter: int):
-        self.public_key = public_key
-        self.balance = balance
-        self.counter = counter
-
-class Block:
-    def __init__(
-        self,
-        accounts: bytes,
-        chain: Chain,
-        difficulty: int,
-        delay: int,
-        number: int,
-        previous: Block,
-        receipts: bytes,
-        aster: int,
-        time: int,
-        transactions: bytes,
-        validator: Account,
-        signature: bytes
-    ):
-        self.accounts = accounts
-        self.chain = chain
-        self.difficulty = difficulty
-        self.delay = delay
-        self.number = number
-        self.previous = previous
-        self.receipts = receipts
-        self.aster = aster
-        self.time = time
-        self.transactions = transactions
-        self.validator = validator
-        self.signature = signature
-
-class Chain:
-    def __init__(self, latest_block: Block):
-        self.latest_block = latest_block
+    
+    def _extract_child_hashes(self, data: bytes) -> List[bytes]:
+        """
+        Extract child object hashes from object data.
+        This is a placeholder method that should be overridden or adapted based on the object format.
+        
+        Args:
+            data: The object data
+            
+        Returns:
+            List of child object hashes
+        """
+        # In a real implementation, this would parse the data based on its format
+        # and extract references to other objects
+        # For example, if the data is a serialized Merkle node, you might extract
+        # left and right child hashes
         
-class Transaction:
-    def __init__(self, chain: Chain, receipient: Account, sender: Account, counter: int, amount: int, signature: bytes, data: bytes):
-        self.chain = chain
-        self.receipient = receipient
-        self.sender = sender
-        self.counter = counter
-        self.amount = amount
-        self.signature = signature
-        self.data = data
+        # For now, return an empty list
+        return []

astreum/node/storage/trie.py

@@ -0,0 +1,146 @@
+from astreum.utils.bytes_format import encode, decode
+from astreum.utils.hash import hash_data
+from typing import Optional, List
+from ..storage import Storage
+
+
+class Trie:
+    def __init__(self, root: TrieNode, storage: Storage):
+        self.root = root
+        self.storage = storage
+
+    def insert(self, key: bytes, data: bytes):
+        self.root = self._insert(self.root, key, data)
+
+    def _insert(self, node: TrieNode, key: bytes, data: bytes) -> TrieNode:
+        if node is None:
+            return TrieNode(key, data)
+        if key < node.key:
+            node.children = self._insert(node.children, key, data)
+        elif key > node.key:
+            node.children = self._insert(node.children, key, data)
+        else:
+            node.data = data
+        return node
+
+    def lookup(self, key: bytes) -> bytes:
+        """
+        Look up a key in the trie.
+        
+        Args:
+            key: The key to look up
+            
+        Returns:
+            The data associated with the key, or None if not found
+        """
+        return self._lookup(self.root, key)
+
+    def _lookup(self, node: Optional[TrieNode], key: bytes) -> bytes:
+        """
+        Recursive helper for looking up a key in the trie.
+        
+        Args:
+            node: The current node being examined
+            key: The key to look up
+            
+        Returns:
+            The data associated with the key, or None if not found
+        """
+        if node is None:
+            return None
+            
+        # If we found an exact match, return the data
+        if node.key == key:
+            return node.data
+            
+        # Make sure node has a storage reference
+        if node.storage is None:
+            node.storage = self.storage
+            
+        # Use child_lookup to find the most promising child
+        child_node = node.child_lookup(key)
+        if child_node is None:
+            return None
+            
+        # Create and traverse to the child node
+        child_node = TrieNode.from_bytes(child_data, self.storage)
+        return self._lookup(child_node, key)
+
+class TrieNode:
+    """
+    A node in a trie.
+
+    Attributes:
+        key: The key of the node
+        data: The data stored in the node
+        children: The children of the node
+        storage: Reference to storage service (not serialized)
+    """
+    def __init__(self, key: bytes, data: bytes = None, children: bytes = None, storage = None):
+        """
+        Initialize a new TrieNode.
+        
+        Args:
+            key: The key of the node
+            data: The data stored in the node
+            children: a byte string of children hashes each are 32 bytes long
+            storage: Storage instance for retrieving child nodes (not serialized)
+        """
+        self.key = key
+        self.data = data
+        self.children = children
+        self.storage = storage
+
+    def to_bytes(self) -> bytes:
+        """Serialize the node data (excluding storage reference)"""
+        return encode([self.key, self.data, self.children])
+
+    @classmethod
+    def from_bytes(cls, data: bytes, storage = None) -> 'TrieNode':
+        """
+        Deserialize node data and optionally attach a storage reference.
+        
+        Args:
+            data: The serialized node data
+            storage: Optional storage instance to attach to the node
+        
+        Returns:
+            A new TrieNode instance
+        """
+        key, data, children = decode(data)
+        return TrieNode(key, data, children, storage)
+
+    def hash(self) -> bytes:
+        return hash_data(self.to_bytes())
+
+    def child_lookup(self, key: bytes) -> Optional[TrieNode]:
+        """
+        Does a binary lookup of the keys in the children of this node.
+        Uses storage to look up children and finds a starting match for the key.
+        """
+        if self.children is None or self.storage is None:
+            return None
+        
+        # Parse children bytes into a list of 32-byte hashes
+        children_hashes = []
+        for i in range(0, len(self.children), 32):
+            if i + 32 <= len(self.children):
+                children_hashes.append(self.children[i:i+32])
+        
+        # Look up each child in storage and compare keys
+        for child_hash in children_hashes:
+            # Get child node data from storage
+            child_data = self.storage.get(child_hash)
+            if child_data is None:
+                continue  # Skip if not found in storage
+            
+            # Deserialize the child node
+            child_node = TrieNode.from_bytes(child_data, self.storage)
+            
+            # Check if this child's key is a prefix of the lookup key
+            # or if the lookup key is a prefix of this child's key
+            min_len = min(len(child_node.key), len(key))
+            if child_node.key[:min_len] == key[:min_len]:
+                return child_node
+        
+        return None

astreum/node/storage/utils.py

@@ -0,0 +1,137 @@
+"""
+Utility functions for working with the storage module and Merkle trees,
+with special focus on validator stake operations and binary searches.
+"""
+
+from typing import List, Dict, Optional, Tuple, Any, Callable, TypeVar
+from .merkle import MerkleTree, MerkleProof, MerkleNode, MerkleNodeType
+from .merkle import find_first, find_all, map, binary_search
+
+T = TypeVar('T')
+
+
+def create_ordered_merkle_tree(items: List[bytes], storage=None) -> Tuple[bytes, MerkleTree]:
+    """
+    Create a new ordered Merkle tree from items.
+    
+    Args:
+        items: List of items to include in the tree, will be sorted
+        storage: Optional storage backend to persist the tree
+        
+    Returns:
+        Tuple of (root_hash, merkle_tree)
+    """
+    tree = MerkleTree(storage)
+    root_hash = tree.add_sorted(items)
+    return root_hash, tree
+
+
+def query_validator_stake(storage, stake_root_hash: bytes, validator_address: bytes) -> Optional[int]:
+    """
+    Query a validator's stake by their address from a stake Merkle tree.
+    
+    Args:
+        storage: Storage instance used by the Merkle tree
+        stake_root_hash: Root hash of the stake Merkle tree
+        validator_address: Address of the validator to look up
+        
+    Returns:
+        The validator's stake amount as an integer, or None if not found
+    """
+    # Define a comparison function for binary search (assuming address is first part of data)
+    def compare_address(data: bytes) -> int:
+        # Extract address from data (format depends on how stakes are stored)
+        # Assuming format: [address][stake]
+        data_address = data[:len(validator_address)]
+        
+        if data_address < validator_address:
+            return 1  # Data is less than target
+        elif data_address > validator_address:
+            return -1  # Data is greater than target
+        else:
+            return 0  # Match found
+    
+    # Binary search for the validator's stake
+    stake_data = binary_search(storage, stake_root_hash, compare_address)
+    
+    if stake_data:
+        # Extract stake amount from data
+        # Assuming format: [address][stake_amount as 8-byte integer]
+        address_len = len(validator_address)
+        stake_amount = int.from_bytes(stake_data[address_len:address_len+8], byteorder='big')
+        return stake_amount
+    
+    return None
+
+
+def find_validator_stakes(storage, stake_root_hash: bytes, min_stake: int = 0) -> List[Tuple[bytes, int]]:
+    """
+    Find all validators with stakes above a minimum threshold.
+    
+    Args:
+        storage: Storage instance used by the Merkle tree
+        stake_root_hash: Root hash of the stake Merkle tree
+        min_stake: Minimum stake threshold (default: 0)
+        
+    Returns:
+        List of (validator_address, stake_amount) tuples
+    """
+    # Define predicate to filter validators by minimum stake
+    def has_min_stake(data: bytes) -> bool:
+        # Assuming format: [address][stake_amount as 8-byte integer]
+        address_len = len(data) - 8  # Adjust based on your actual format
+        stake_amount = int.from_bytes(data[address_len:], byteorder='big')
+        return stake_amount >= min_stake
+    
+    # Define transform to extract address and stake
+    def extract_address_and_stake(data: bytes) -> Tuple[bytes, int]:
+        # Assuming format: [address][stake_amount as 8-byte integer]
+        address_len = len(data) - 8  # Adjust based on your actual format
+        address = data[:address_len]
+        stake = int.from_bytes(data[address_len:], byteorder='big')
+        return (address, stake)
+    
+    # Find all validators meeting criteria and transform results
+    matching_validators = find_all(storage, stake_root_hash, has_min_stake)
+    return [extract_address_and_stake(data) for data in matching_validators]
+
+
+def get_total_stake(storage, stake_root_hash: bytes) -> int:
+    """
+    Calculate the total stake across all validators.
+    
+    Args:
+        storage: Storage instance used by the Merkle tree
+        stake_root_hash: Root hash of the stake Merkle tree
+        
+    Returns:
+        Total stake amount
+    """
+    # Define transform to extract stake amount
+    def extract_stake(data: bytes) -> int:
+        # Assuming format: [address][stake_amount as 8-byte integer]
+        address_len = len(data) - 8  # Adjust based on your actual format
+        return int.from_bytes(data[address_len:], byteorder='big')
+    
+    # Map all leaves to their stake values and sum
+    stakes = map(storage, stake_root_hash, extract_stake)
+    return sum(stakes)
+
+
+def query_with_custom_resolver(storage, root_hash: bytes, 
+                              resolver_fn: Callable[[bytes], T]) -> T:
+    """
+    Query a Merkle tree using a custom resolver function.
+    
+    This is a general-purpose function that allows custom logic to be applied
+    to tree traversal and data extraction.
+    
+    Args:
+        storage: Storage instance used by the Merkle tree
+        root_hash: Root hash of the Merkle tree
+        resolver_fn: Function that takes a root hash and returns a result
+        
+    Returns:
+        Whatever the resolver function returns
+    """
+    return resolver_fn(root_hash)

astreum/node/utils.py

@@ -0,0 +1,34 @@
+"""
+Utility functions for the Astreum blockchain.
+"""
+
+import blake3
+
+def hash_data(data: bytes) -> bytes:
+    """
+    Hash data using BLAKE3.
+    
+    Args:
+        data: Data to hash
+        
+    Returns:
+        32-byte BLAKE3 hash
+    """
+    return blake3.blake3(data).digest()
+
+def hash_object(obj) -> bytes:
+    """
+    Hash a Python object by converting it to a string and then hashing.
+    
+    Args:
+        obj: Python object to hash
+        
+    Returns:
+        32-byte BLAKE3 hash
+    """
+    if isinstance(obj, bytes):
+        return hash_data(obj)
+    elif isinstance(obj, str):
+        return hash_data(obj.encode('utf-8'))
+    else:
+        return hash_data(str(obj).encode('utf-8'))

astreum/node/validation/__init__.py

@@ -0,0 +1,84 @@
+"""
+Validation module for the Astreum blockchain.
+
+This module provides functions for validating blocks and transactions,
+computing and verifying VDFs, and selecting validators.
+"""
+
+# Export validation functions
+from .block import (
+    validate_block,
+    create_block,
+    create_genesis_block,
+    select_validator,
+    select_validator_for_slot
+)
+
+# Export VDF functions
+from .vdf import (
+    compute_vdf,
+    verify_vdf,
+    validate_block_vdf
+)
+
+# Export account functions
+from .account import (
+    Account,
+    get_validator_stake,
+    is_validator
+)
+
+# Export constants
+from .constants import (
+    VALIDATION_ADDRESS,
+    BURN_ADDRESS,
+    MIN_STAKE_AMOUNT,
+    SLOT_DURATION,
+    VDF_DIFFICULTY
+)
+
+# Export blockchain state functions
+from .state import (
+    add_block_to_state,
+    validate_and_apply_block,
+    create_account_state,
+    get_validator_for_slot,
+    select_best_chain,
+    compare_chains,
+    get_validator_set
+)
+
+__all__ = [
+    # Block validation
+    'validate_block',
+    'create_block',
+    'create_genesis_block',
+    'select_validator',
+    'select_validator_for_slot',
+    
+    # VDF functions
+    'compute_vdf',
+    'verify_vdf',
+    'validate_block_vdf',
+    
+    # Account functions
+    'Account',
+    'get_validator_stake',
+    'is_validator',
+    
+    # Constants
+    'VALIDATION_ADDRESS',
+    'BURN_ADDRESS',
+    'MIN_STAKE_AMOUNT',
+    'SLOT_DURATION',
+    'VDF_DIFFICULTY',
+    
+    # Blockchain state
+    'add_block_to_state',
+    'validate_and_apply_block',
+    'create_account_state',
+    'get_validator_for_slot',
+    'select_best_chain',
+    'compare_chains',
+    'get_validator_set'
+]