astreum 0.1.8__py3-none-any.whl → 0.1.10__py3-none-any.whl

astreum/node/__init__.py

@@ -1,55 +1,49 @@
 import os
 import hashlib
 import time
-from typing import Tuple
+from typing import Tuple, Optional
+import json
 
 from .relay import Relay, Topic
 from .relay.peer import Peer
-from .storage import Storage
-from .route_table import RouteTable
+from .models import Storage, Block, Transaction
 from .machine import AstreumMachine
 from .utils import encode, decode
-from .models import Block, Transaction
 from astreum.lispeum.storage import store_expr, get_expr_from_storage
 
 class Node:
     def __init__(self, config: dict):
         self.config = config
-        self.node_id = config.get('node_id', os.urandom(32))  # Default to random ID if not provided
         self.relay = Relay(config)
+        # Get the node_id from relay instead of generating our own
+        self.node_id = self.relay.node_id
         self.storage = Storage(config)
+        self.storage.node = self  # Set the storage node reference to self
         
         # Latest block of the chain this node is following
         self.latest_block = None
         self.followed_chain_id = config.get('followed_chain_id', None)
         
-        # Candidate chains that might be adopted
-        self.candidate_chains = {}  # chain_id -> {'latest_block': block, 'timestamp': time.time()}
-        
-        # Initialize route table with our node ID
-        self.route_table = RouteTable(config, self.node_id)
-        
-        # Initialize machine after storage so it can use it
-        # Pass self to machine so it can access the storage
+        # Initialize machine
         self.machine = AstreumMachine(node=self)
         
         # Register message handlers
-        self._register_message_handlers()
+        self.relay.message_handlers[Topic.PEER_ROUTE] = self._handle_peer_route
+        self.relay.message_handlers[Topic.PING] = self._handle_ping
+        self.relay.message_handlers[Topic.PONG] = self._handle_pong
+        self.relay.message_handlers[Topic.OBJECT_REQUEST] = self._handle_object_request
+        self.relay.message_handlers[Topic.OBJECT_RESPONSE] = self._handle_object_response
+        self.relay.message_handlers[Topic.ROUTE_REQUEST] = self._handle_route_request
+        self.relay.message_handlers[Topic.ROUTE] = self._handle_route
+        self.relay.message_handlers[Topic.LATEST_BLOCK_REQUEST] = self._handle_latest_block_request
+        self.relay.message_handlers[Topic.LATEST_BLOCK] = self._handle_latest_block
+        self.relay.message_handlers[Topic.TRANSACTION] = self._handle_transaction
         
         # Initialize latest block from storage if available
         self._initialize_latest_block()
         
-    def _register_message_handlers(self):
-        """Register handlers for different message topics."""
-        self.relay.register_message_handler(Topic.PING, self._handle_ping)
-        self.relay.register_message_handler(Topic.PONG, self._handle_pong)
-        self.relay.register_message_handler(Topic.OBJECT_REQUEST, self._handle_object_request)
-        self.relay.register_message_handler(Topic.OBJECT, self._handle_object)
-        self.relay.register_message_handler(Topic.ROUTE_REQUEST, self._handle_route_request)
-        self.relay.register_message_handler(Topic.ROUTE, self._handle_route)
-        self.relay.register_message_handler(Topic.LATEST_BLOCK_REQUEST, self._handle_latest_block_request)
-        self.relay.register_message_handler(Topic.LATEST_BLOCK, self._handle_latest_block)
-        self.relay.register_message_handler(Topic.TRANSACTION, self._handle_transaction)
+        # Candidate chains that might be adopted
+        self.candidate_chains = {}  # chain_id -> {'latest_block': block, 'timestamp': time.time()}
         
     def _handle_ping(self, body: bytes, addr: Tuple[str, int], envelope):
         """
@@ -70,7 +64,7 @@ class Node:
             difficulty = int.from_bytes(difficulty_bytes, byteorder='big')
             
             # Store peer information in routing table
-            peer = self.route_table.update_peer(addr, public_key, difficulty)
+            peer = self.relay.add_peer(addr, public_key, difficulty)
             
             # Process the routes the sender is participating in
             if routes_data:
@@ -104,7 +98,7 @@ class Node:
             difficulty = int.from_bytes(difficulty_bytes, byteorder='big')
             
             # Update peer information in routing table
-            peer = self.route_table.update_peer(addr, public_key, difficulty)
+            peer = self.relay.add_peer(addr, public_key, difficulty)
             
             # Process the routes the sender is participating in
             if routes_data:
@@ -116,21 +110,65 @@ class Node:
     
     def _handle_object_request(self, body: bytes, addr: Tuple[str, int], envelope):
         """
-        Handle request for an object by its hash.
-        Check storage and return if available, otherwise ignore.
+        Handle an object request from a peer.
+        
+        Args:
+            body: Message body containing the object hash
+            addr: Address of the requesting peer
+            envelope: Full message envelope
         """
         try:
-            # The body is the hash of the requested object
-            object_hash = body
-            object_data = self.storage.get(object_hash)
+            # Decode the request
+            request = json.loads(body.decode('utf-8'))
+            object_hash = bytes.fromhex(request.get('hash'))
+            
+            # Check if we have the requested object
+            if not self.storage.contains(object_hash):
+                # We don't have the object, ignore the request
+                return
+                
+            # Get the object data
+            object_data = self.storage._local_get(object_hash)
+            if not object_data:
+                return
+                
+            # Create a response message
+            response = {
+                'hash': object_hash.hex(),
+                'data': object_data.hex()
+            }
+            
+            # Send the response
+            self.relay.send_message_to_addr(
+                addr, 
+                Topic.OBJECT_RESPONSE, 
+                json.dumps(response).encode('utf-8')
+            )
             
-            if object_data:
-                # Object found, send it back
-                self.relay.send_message(object_data, Topic.OBJECT, addr)
-            # If object not found, simply ignore the request
         except Exception as e:
             print(f"Error handling object request: {e}")
     
+    def _handle_object_response(self, body: bytes, addr: Tuple[str, int], envelope):
+        """
+        Handle an object response from a peer.
+        
+        Args:
+            body: Message body containing the object hash and data
+            addr: Address of the responding peer
+            envelope: Full message envelope
+        """
+        try:
+            # Decode the response
+            response = json.loads(body.decode('utf-8'))
+            object_hash = bytes.fromhex(response.get('hash'))
+            object_data = bytes.fromhex(response.get('data'))
+            
+            # Store the object
+            self.storage.put(object_hash, object_data)
+            
+        except Exception as e:
+            print(f"Error handling object response: {e}")
+    
     def _handle_object(self, body: bytes, addr: Tuple[str, int], envelope):
         """
         Handle receipt of an object.
@@ -147,6 +185,68 @@ class Node:
         except Exception as e:
             print(f"Error handling object: {e}")
     
+    def request_object(self, object_hash: bytes, max_attempts: int = 3) -> Optional[bytes]:
+        """
+        Request an object from the network by its hash.
+        
+        This method sends an object request to peers closest to the object hash
+        and waits for a response until timeout.
+        
+        Args:
+            object_hash: The hash of the object to request
+            max_attempts: Maximum number of request attempts
+            
+        Returns:
+            The object data if found, None otherwise
+        """
+        # First check if we already have the object
+        if self.storage.contains(object_hash):
+            return self.storage._local_get(object_hash)
+            
+        # Find the bucket containing the peers closest to the object's hash
+        closest_peers = self.relay.get_closest_peers(object_hash, count=3)
+        if not closest_peers:
+            return None
+            
+        # Create a message to request the object
+        topic = Topic.OBJECT_REQUEST
+        object_request_msg = {
+            'hash': object_hash.hex()
+        }
+        
+        # Track which peers we've already tried
+        attempted_peers = set()
+        
+        # We'll try up to max_attempts times
+        for _ in range(max_attempts):
+            # Find peers we haven't tried yet
+            untried_peers = [p for p in closest_peers if p.id not in attempted_peers]
+            if not untried_peers:
+                break
+                
+            # Send the request to all untried peers
+            request_sent = False
+            for peer in untried_peers:
+                try:
+                    self.relay.send_message_to_peer(peer, topic, object_request_msg)
+                    attempted_peers.add(peer.id)
+                    request_sent = True
+                except Exception as e:
+                    print(f"Failed to send object request to peer {peer.id.hex()}: {e}")
+            
+            if not request_sent:
+                break
+                
+            # Short wait to allow for response
+            time.sleep(0.5)
+            
+            # Check if any of the requests succeeded
+            if self.storage.contains(object_hash):
+                return self.storage._local_get(object_hash)
+                
+        # If we get here, we couldn't get the object
+        return None
+    
     def _handle_route_request(self, body: bytes, addr: Tuple[str, int], envelope):
         """
         Handle request for routing information.
@@ -157,8 +257,8 @@ class Node:
             route_peers = []
             
             # Get one peer from each bucket
-            for bucket_index in range(self.route_table.num_buckets):
-                peers = self.route_table.get_bucket_peers(bucket_index)
+            for bucket_index in range(self.relay.num_buckets):
+                peers = self.relay.get_bucket_peers(bucket_index)
                 if peers and len(peers) > 0:
                     # Add one peer from this bucket
                     route_peers.append(peers[0])
@@ -206,7 +306,7 @@ class Node:
                     
                     # Ping this peer if it's not already in our routing table
                     # and it's not our own address
-                    if (not self.route_table.has_peer(peer_address) and 
+                    if (not self.relay.has_peer(peer_address) and 
                             peer_address != self.relay.get_address()):
                         # Create ping message with our info and routes
                         # Encode our peer and validation routes
@@ -339,21 +439,10 @@ class Node:
             print(f"Error handling transaction: {e}")
             
     def _initialize_latest_block(self):
-        """Initialize the latest block from storage if available."""
-        try:
-            if self.followed_chain_id:
-                # Get the latest block for the chain we're following
-                self.latest_block = self.machine.get_latest_block(self.followed_chain_id)
-            else:
-                # If no specific chain is set to follow, get the latest block from the default chain
-                self.latest_block = self.machine.get_latest_block()
-                
-                # If we have a latest block, set the followed chain ID
-                if self.latest_block:
-                    self.followed_chain_id = self.latest_block.chain_id
-        except Exception as e:
-            print(f"Error initializing latest block: {e}")
-            
+        """Initialize latest block from storage if available."""
+        # Implementation would load the latest block from storage
+        pass
+    
     def set_followed_chain(self, chain_id):
         """
         Set the chain that this node follows.

astreum/node/models.py

@@ -1,12 +1,15 @@
 import socket
 from pathlib import Path
-from typing import Optional, Tuple
+from typing import Optional, Tuple, Dict
 from astreum.machine import AstreumMachine
 from .relay import Relay
 from .relay.message import Topic
 from .relay.route import RouteTable
 from .relay.peer import Peer
 import os
+import struct
+import threading
+import time
 
 class Storage:
     def __init__(self, config: dict):
@@ -14,6 +17,13 @@ class Storage:
         self.current_space = 0
         self.storage_path = Path(config.get('storage_path', 'storage'))
         self.storage_path.mkdir(parents=True, exist_ok=True)
+        self.max_object_recursion = config.get('max_object_recursion', 50)
+        self.network_request_timeout = config.get('network_request_timeout', 5.0)  # Default 5 second timeout
+        self.node = None  # Will be set by the Node after initialization
+        
+        # In-progress requests tracking
+        self.pending_requests = {}  # hash -> (start_time, event)
+        self.request_lock = threading.Lock()
         
         # Calculate current space usage
         self.current_space = sum(f.stat().st_size for f in self.storage_path.glob('*') if f.is_file())
@@ -33,18 +43,196 @@ class Storage:
         # Store the data
         file_path.write_bytes(data)
         self.current_space += data_size
+        
+        # If this was a pending request, mark it as complete
+        with self.request_lock:
+            if data_hash in self.pending_requests:
+                _, event = self.pending_requests[data_hash]
+                event.set()  # Signal that the data is now available
+        
         return True
 
-    def get(self, data_hash: bytes) -> Optional[bytes]:
-        """Retrieve data by its hash. Returns None if not found."""
+    def _local_get(self, data_hash: bytes) -> Optional[bytes]:
+        """Get data from local storage only, no network requests."""
         file_path = self.storage_path / data_hash.hex()
-        if not file_path.exists():
+        if file_path.exists():
+            return file_path.read_bytes()
+        return None
+
+    def get(self, data_hash: bytes, timeout: Optional[float] = None) -> Optional[bytes]:
+        """
+        Retrieve data by its hash, with network fallback.
+        
+        This function will first check local storage. If not found and a node is attached,
+        it will initiate a network request asynchronously.
+        
+        Args:
+            data_hash: The hash of the data to retrieve
+            timeout: Timeout in seconds to wait for network request, None for default
+            
+        Returns:
+            The data bytes if found, None otherwise
+        """
+        if timeout is None:
+            timeout = self.network_request_timeout
+            
+        # First check local storage
+        local_data = self._local_get(data_hash)
+        if local_data:
+            return local_data
+            
+        # If no node is attached, we can't make network requests
+        if self.node is None:
             return None
-        return file_path.read_bytes()
+            
+        # Check if there's already a pending request for this hash
+        with self.request_lock:
+            if data_hash in self.pending_requests:
+                start_time, event = self.pending_requests[data_hash]
+                # If this request has been going on too long, cancel it and start a new one
+                elapsed = time.time() - start_time
+                if elapsed > timeout:
+                    # Cancel the old request
+                    self.pending_requests.pop(data_hash)
+                else:
+                    # Wait for the existing request to complete
+                    wait_time = timeout - elapsed
+            else:
+                # No existing request, create a new one
+                event = threading.Event()
+                self.pending_requests[data_hash] = (time.time(), event)
+                # Start the actual network request in a separate thread
+                threading.Thread(
+                    target=self._request_from_network,
+                    args=(data_hash,),
+                    daemon=True
+                ).start()
+                wait_time = timeout
+                
+        # Wait for the request to complete or timeout
+        if event.wait(wait_time):
+            # Event was set, data should be available now
+            with self.request_lock:
+                if data_hash in self.pending_requests:
+                    self.pending_requests.pop(data_hash)
+            
+            # Check if data is now in local storage
+            return self._local_get(data_hash)
+        else:
+            # Timed out waiting for data
+            with self.request_lock:
+                if data_hash in self.pending_requests:
+                    self.pending_requests.pop(data_hash)
+            return None
+    
+    def _request_from_network(self, data_hash: bytes):
+        """
+        Request object from the network.
+        This is meant to be run in a separate thread.
+        
+        Args:
+            data_hash: The hash of the object to request
+        """
+        try:
+            if hasattr(self.node, 'request_object'):
+                # Use the node's request_object method
+                self.node.request_object(data_hash)
+            # Note: We don't need to return anything or signal completion here
+            # The put() method will signal completion when the object is received
+        except Exception as e:
+            print(f"Error requesting object {data_hash.hex()} from network: {e}")
 
     def contains(self, data_hash: bytes) -> bool:
         """Check if data exists in storage."""
         return (self.storage_path / data_hash.hex()).exists()
+        
+    def get_recursive(self, root_hash: bytes, max_depth: Optional[int] = None, 
+                     timeout: Optional[float] = None) -> Dict[bytes, bytes]:
+        """
+        Recursively retrieve all objects starting from a root hash.
+        
+        Objects not found locally will be requested from the network.
+        This method will continue processing objects that are available
+        while waiting for network responses.
+        
+        Args:
+            root_hash: The hash of the root object
+            max_depth: Maximum recursion depth, defaults to self.max_object_recursion
+            timeout: Time to wait for each object request, None for default
+            
+        Returns:
+            Dictionary mapping object hashes to their data
+        """
+        if max_depth is None:
+            max_depth = self.max_object_recursion
+            
+        if timeout is None:
+            timeout = self.network_request_timeout
+            
+        # Start with the root object
+        objects = {}
+        pending_queue = [(root_hash, 0)]  # (hash, depth)
+        processed = set()
+        
+        # Process objects in the queue
+        while pending_queue:
+            current_hash, current_depth = pending_queue.pop(0)
+            
+            # Skip if already processed or too deep
+            if current_hash in processed or current_depth > max_depth:
+                continue
+                
+            processed.add(current_hash)
+            
+            # Try to get the object (which may start a network request)
+            obj_data = self.get(current_hash, timeout)
+            if obj_data is None:
+                # Failed to get this object, but we continue with the rest
+                print(f"Warning: Failed to get object {current_hash.hex()}")
+                continue
+                
+            # Store the object in our result
+            objects[current_hash] = obj_data
+            
+            # Only process non-leaf nodes for recursion
+            try:
+                # Extract leaf flag and type
+                is_leaf = struct.unpack("?", obj_data[0:1])[0]
+                if is_leaf:
+                    # Leaf node, no need to recurse
+                    continue
+                    
+                type_indicator = obj_data[1:2]
+                next_depth = current_depth + 1
+                
+                if type_indicator == b'L':  # List
+                    # Non-leaf list has child element hashes
+                    elements_bytes = obj_data[2:]
+                    element_hashes = [elements_bytes[i:i+32] for i in range(0, len(elements_bytes), 32)]
+                    
+                    # Add each element hash to the queue
+                    for elem_hash in element_hashes:
+                        pending_queue.append((elem_hash, next_depth))
+                        
+                elif type_indicator == b'F':  # Function
+                    # Non-leaf function has body hash
+                    remaining_bytes = obj_data[2:]
+                    
+                    # Find the separator between params and body hash
+                    params_end = remaining_bytes.find(b',', remaining_bytes.rfind(b','))
+                    if params_end == -1:
+                        params_end = 0  # No params
+                        
+                    body_hash = remaining_bytes[params_end+1:]
+                    
+                    # Add body hash to the queue
+                    pending_queue.append((body_hash, next_depth))
+                    
+            except Exception as e:
+                print(f"Error processing object {current_hash.hex()}: {e}")
+                continue
+                
+        return objects
 
 class Account:
     def __init__(self, public_key: bytes, balance: int, counter: int):

astreum/node/relay/__init__.py

@@ -11,20 +11,43 @@ from .envelope import Envelope
 from .bucket import KBucket
 from .peer import Peer, PeerManager
 from .route import RouteTable
+import json
+from cryptography.hazmat.primitives.asymmetric import ed25519
 
 class Relay:
     def __init__(self, config: dict):
+        """Initialize relay with configuration."""
+        self.config = config
         self.use_ipv6 = config.get('use_ipv6', False)
         incoming_port = config.get('incoming_port', 7373)
         self.max_message_size = config.get('max_message_size', 65536)  # Max UDP datagram size
         self.num_workers = config.get('num_workers', 4)
 
+        # Generate Ed25519 keypair for this node
+        if 'private_key' in config:
+            # Load existing private key if provided
+            try:
+                private_key_bytes = bytes.fromhex(config['private_key'])
+                self.private_key = ed25519.Ed25519PrivateKey.from_private_bytes(private_key_bytes)
+            except Exception as e:
+                print(f"Error loading private key: {e}, generating new one")
+                self.private_key = ed25519.Ed25519PrivateKey.generate()
+        else:
+            # Generate new keypair
+            self.private_key = ed25519.Ed25519PrivateKey.generate()
+            
+        # Use public key as node ID
+        self.public_key = self.private_key.public_key()
+        self.node_id = self.public_key.public_bytes_raw()
+        
+        # Save private key bytes for config persistence
+        self.private_key_bytes = self.private_key.private_bytes_raw()
+
         # Routes that this node participates in (0 = peer route, 1 = validation route)
         self.routes: List[int] = []
         
-        # Initialize routes from config if provided
-        if config.get('peer_route', False):
-            self.routes.append(0)  # Peer route
+        # Peer route is always enabled
+        self.routes.append(0)  # Peer route
             
         if config.get('validation_route', False):
             self.routes.append(1)  # Validation route
@@ -54,11 +77,18 @@ class Relay:
         self.outgoing_queue = Queue()
         
         # Message handling
-        self.message_handlers: Dict[Topic, Callable] = {}
+        self.message_handlers: Dict[Topic, Callable] = {
+            Topic.PEER_ROUTE: None,  # set by Node later
+            Topic.OBJECT_REQUEST: None,  # set by Node later
+            Topic.OBJECT_RESPONSE: None,  # set by Node later
+        }
 
         # Route buckets (peers for each route)
         self.peer_route_bucket = KBucket(k=20)  # Bucket for peer route
         self.validation_route_bucket = KBucket(k=20)  # Bucket for validation route
+        
+        # Initialize route table with our node ID
+        self.route_table = RouteTable(self)
 
         # Start worker threads
         self._start_workers()
@@ -197,6 +227,12 @@ class Relay:
             elif envelope.message.topic in (Topic.LATEST_BLOCK_REQUEST, Topic.GET_BLOCKS):
                 # Allow all nodes to request blocks for syncing
                 self.message_handlers[envelope.message.topic](envelope.message.body, addr, envelope)
+            elif envelope.message.topic == Topic.OBJECT_REQUEST:
+                # Handle object request
+                self.message_handlers[envelope.message.topic](envelope.message.body, addr, envelope)
+            elif envelope.message.topic == Topic.OBJECT_RESPONSE:
+                # Handle object response
+                self.message_handlers[envelope.message.topic](envelope.message.body, addr, envelope)
             else:
                 # For other message types, always process
                 self.message_handlers[envelope.message.topic](envelope.message.body, addr, envelope)
@@ -240,9 +276,69 @@ class Relay:
         encoded_data = envelope.to_bytes()
         self.send(encoded_data, addr)
 
+    def send_message_to_addr(self, addr: tuple, topic: Topic, body: bytes):
+        """
+        Send a message to a specific address.
+        
+        Args:
+            addr: Tuple of (ip, port) to send to
+            topic: Message topic
+            body: Message body
+        """
+        try:
+            # Create an envelope with our node id and the message
+            message = Message(self.node_id, topic, body)
+            envelope = Envelope(message)
+            
+            # Serialize and send
+            self.outgoing_socket.sendto(envelope.to_bytes(), addr)
+        except Exception as e:
+            print(f"Error sending message to {addr}: {e}")
+            
+    def send_message_to_peer(self, peer: Peer, topic: Topic, body):
+        """
+        Send a message to a specific peer.
+        
+        Args:
+            peer: Peer to send to
+            topic: Message topic
+            body: Message body (bytes or JSON serializable)
+        """
+        # Convert body to bytes if it's not already
+        if not isinstance(body, bytes):
+            if isinstance(body, dict) or isinstance(body, list):
+                body = json.dumps(body).encode('utf-8')
+            else:
+                body = str(body).encode('utf-8')
+                
+        # Send to the peer's address
+        self.send_message_to_addr(peer.address, topic, body)
+
     def stop(self):
         """Stop all worker threads."""
         self.running = False
         # Wait for queues to be processed
         self.incoming_queue.join()
         self.outgoing_queue.join()
+
+    # RouteTable wrapper methods
+    def add_peer(self, addr, public_key, difficulty):
+        """Add a peer to the routing table."""
+        return self.route_table.update_peer(addr, public_key, difficulty)
+        
+    def get_closest_peers(self, target_id, count=3):
+        """Get the closest peers to the target ID."""
+        return self.route_table.get_closest_peers(target_id, count=count)
+        
+    @property
+    def num_buckets(self):
+        """Get the number of buckets in the routing table."""
+        return self.route_table.num_buckets
+        
+    def get_bucket_peers(self, bucket_index):
+        """Get peers from a specific bucket."""
+        return self.route_table.get_bucket_peers(bucket_index)
+        
+    def has_peer(self, addr):
+        """Check if a peer with the given address exists in the routing table."""
+        return self.route_table.has_peer(addr)