astreum 0.2.29__py3-none-any.whl → 0.2.61__py3-none-any.whl

astreum/__init__.py

@@ -1 +1,9 @@
-from .node import Node, Expr
+"""Lightweight package initializer to avoid circular imports during tests.
+
+Exports are intentionally minimal; import submodules directly as needed:
+ - Node, Expr, Env, tokenize, parse -> from astreum._node or astreum.lispeum
+ - Validation types -> from astreum._validation
+ - Storage types -> from astreum._storage
+"""
+
+__all__: list[str] = []

astreum/_communication/__init__.py

@@ -0,0 +1,11 @@
+from .message import Message
+from .peer import Peer
+from .route import Route
+from .setup import communication_setup
+
+__all__ = [
+    "Message",
+    "Peer",
+    "Route",
+    "communication_setup",
+]

astreum/{models → _communication}/message.py

@@ -1,64 +1,101 @@
-from enum import IntEnum
-from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PublicKey
-
-class MessageTopic(IntEnum):
-    PING = 0
-    OBJECT_REQUEST = 1
-    OBJECT_RESPONSE = 2
-    ROUTE_REQUEST = 3
-    ROUTE_RESPONSE = 4
-
-class Message:
-    handshake: bool
-    sender: X25519PublicKey
-
-    topic: MessageTopic
-    content: bytes
-
-    def to_bytes(self):
-        if self.handshake:
-            # handshake byte (1) + raw public key bytes
-            return bytes([1]) + self.sender.public_bytes(
-                encoding=serialization.Encoding.Raw,
-                format=serialization.PublicFormat.Raw
-            )
-        else:
-            # normal message: 0 + topic + content
-            return bytes([0, self.topic.value]) + self.content
-
-    @classmethod
-    def from_bytes(cls, data: bytes) -> "Message":
-        if len(data) < 1:
-            raise ValueError("Cannot parse Message: no data")
-        flag = data[0]
-        # create empty instance
-        msg = cls.__new__(cls)
-
-        if flag == 1:
-            # handshake message: the rest is the peer’s public key
-            key_bytes = data[1:]
-            try:
-                sender = X25519PublicKey.from_public_bytes(key_bytes)
-            except ValueError:
-                raise ValueError("Invalid public key bytes")
-            msg.handshake = True
-            msg.sender = sender
-            msg.topic = None
-            msg.content = b''
-        elif flag == 0:
-            # normal message: next byte is topic, rest is content
-            if len(data) < 2:
-                raise ValueError("Cannot parse Message: missing topic byte")
-            topic_val = data[1]
-            try:
-                topic = MessageTopic(topic_val)
-            except ValueError:
-                raise ValueError(f"Unknown MessageTopic: {topic_val}")
-            msg.handshake = False
-            msg.sender = None
-            msg.topic = topic
-            msg.content = data[2:]
-        else:
-            raise ValueError(f"Invalid handshake flag: {flag}")
-
-        return msg
+from enum import IntEnum
+from typing import Optional
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PublicKey
+
+class MessageTopic(IntEnum):
+    PING = 0
+    OBJECT_REQUEST = 1
+    OBJECT_RESPONSE = 2
+    ROUTE_REQUEST = 3
+    ROUTE_RESPONSE = 4
+    TRANSACTION = 5
+    STORAGE_REQUEST = 6
+    
+
+class Message:
+    handshake: bool
+    sender: Optional[X25519PublicKey]
+
+    topic: Optional[MessageTopic]
+    content: bytes
+
+    def __init__(
+        self,
+        *,
+        handshake: bool = False,
+        sender: Optional[X25519PublicKey] = None,
+        topic: Optional[MessageTopic] = None,
+        content: bytes = b"",
+        body: Optional[bytes] = None,
+    ) -> None:
+        if body is not None:
+            if content and content != b"":
+                raise ValueError("specify only one of 'content' or 'body'")
+            content = body
+
+        self.handshake = handshake
+        self.sender = sender
+        self.topic = topic
+        self.content = content or b""
+
+        if self.handshake:
+            if self.sender is None:
+                raise ValueError("handshake Message requires a sender public key")
+            self.topic = None
+            self.content = b""
+        else:
+            if self.topic is None:
+                raise ValueError("non-handshake Message requires a topic")
+
+    def to_bytes(self):
+        if self.handshake:
+            # handshake byte (1) + raw public key bytes
+            return bytes([1]) + self.sender.public_bytes(
+                encoding=serialization.Encoding.Raw,
+                format=serialization.PublicFormat.Raw
+            )
+        else:
+            # normal message: 0 + topic + content
+            return bytes([0, self.topic.value]) + self.content
+
+    @classmethod
+    def from_bytes(cls, data: bytes) -> "Message":
+        if len(data) < 1:
+            raise ValueError("Cannot parse Message: no data")
+        flag = data[0]
+        # create empty instance
+        msg = cls.__new__(cls)
+
+        if flag == 1:
+            # handshake message: the rest is the peer’s public key
+            key_bytes = data[1:]
+            if not key_bytes:
+                raise ValueError("Handshake message missing sender public key bytes")
+            try:
+                sender = X25519PublicKey.from_public_bytes(key_bytes)
+            except ValueError:
+                raise ValueError("Invalid public key bytes")
+            if sender is None:
+                raise ValueError("Handshake message missing sender public key")
+            msg.handshake = True
+            msg.sender = sender
+            msg.topic = None
+            msg.content = b''
+        elif flag == 0:
+            # normal message: next byte is topic, rest is content
+            if len(data) < 2:
+                raise ValueError("Cannot parse Message: missing topic byte")
+            topic_val = data[1]
+            try:
+                topic = MessageTopic(topic_val)
+            except ValueError:
+                raise ValueError(f"Unknown MessageTopic: {topic_val}")
+            msg.handshake = False
+            msg.sender = None
+            msg.topic = topic
+            msg.content = data[2:]
+        else:
+            raise ValueError(f"Invalid handshake flag: {flag}")
+
+        return msg

astreum/_communication/peer.py

@@ -0,0 +1,23 @@
+from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey, X25519PublicKey
+from cryptography.hazmat.primitives import serialization
+from datetime import datetime, timezone
+from typing import Optional, Tuple
+
+class Peer:
+    shared_key: bytes
+    timestamp: datetime
+    latest_block: bytes
+    address: Optional[Tuple[str, int]]
+    public_key: X25519PublicKey
+    public_key_bytes: bytes
+
+    def __init__(self, my_sec_key: X25519PrivateKey, peer_pub_key: X25519PublicKey):
+        self.shared_key = my_sec_key.exchange(peer_pub_key)
+        self.timestamp = datetime.now(timezone.utc)
+        self.latest_block = b""
+        self.address = None
+        self.public_key = peer_pub_key
+        self.public_key_bytes = peer_pub_key.public_bytes(
+            encoding=serialization.Encoding.Raw,
+            format=serialization.PublicFormat.Raw,
+        )

astreum/_communication/ping.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+class PingFormatError(ValueError):
+    """Raised when ping payload bytes are invalid."""
+
+
+@dataclass
+class Ping:
+    is_validator: bool
+    latest_block: bytes
+
+    PAYLOAD_SIZE = 33
+
+    def __post_init__(self) -> None:
+        lb = bytes(self.latest_block or b"")
+        if len(lb) != 32:
+            raise ValueError("latest_block must be exactly 32 bytes")
+        self.latest_block = lb
+
+    def to_bytes(self) -> bytes:
+        return (b"\x01" if self.is_validator else b"\x00") + self.latest_block
+
+    @classmethod
+    def from_bytes(cls, data: bytes) -> "Ping":
+        if len(data) != cls.PAYLOAD_SIZE:
+            raise PingFormatError("ping payload must be exactly 33 bytes")
+        flag = data[0]
+        if flag not in (0, 1):
+            raise PingFormatError("ping validator flag must be 0 or 1")
+        return cls(is_validator=bool(flag), latest_block=data[1:])

astreum/_communication/route.py

@@ -0,0 +1,95 @@
+from typing import Dict, List, Optional, Union
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PublicKey
+from .peer import Peer
+
+PeerKey = Union[X25519PublicKey, bytes, bytearray]
+
+
+class Route:
+    def __init__(self, relay_public_key: X25519PublicKey, bucket_size: int = 16):
+        self.relay_public_key_bytes = relay_public_key.public_bytes(
+            encoding=serialization.Encoding.Raw,
+            format=serialization.PublicFormat.Raw,
+        )
+        self.bucket_size = bucket_size
+        self.buckets: Dict[int, List[bytes]] = {
+            i: [] for i in range(len(self.relay_public_key_bytes) * 8)
+        }
+        self.peers: Dict[bytes, Peer] = {}
+
+    @staticmethod
+    def _matching_leading_bits(a: bytes, b: bytes) -> int:
+        for byte_index, (ba, bb) in enumerate(zip(a, b)):
+            diff = ba ^ bb
+            if diff:
+                return byte_index * 8 + (8 - diff.bit_length())
+        return len(a) * 8
+
+    def _normalize_peer_key(self, peer_public_key: PeerKey) -> bytes:
+        if isinstance(peer_public_key, X25519PublicKey):
+            return peer_public_key.public_bytes(
+                encoding=serialization.Encoding.Raw,
+                format=serialization.PublicFormat.Raw,
+            )
+        if isinstance(peer_public_key, (bytes, bytearray)):
+            key_bytes = bytes(peer_public_key)
+            if len(key_bytes) != len(self.relay_public_key_bytes):
+                raise ValueError("peer key must be raw 32-byte public key")
+            return key_bytes
+        raise TypeError("peer_public_key must be raw bytes or X25519PublicKey")
+
+    @staticmethod
+    def _xor_distance(a: bytes, b: bytes) -> int:
+        if len(a) != len(b):
+            raise ValueError("xor distance requires equal-length operands")
+        return int.from_bytes(bytes(x ^ y for x, y in zip(a, b)), "big", signed=False)
+
+    def add_peer(self, peer_public_key: PeerKey, peer: Optional[Peer] = None):
+        peer_public_key_bytes = self._normalize_peer_key(peer_public_key)
+        bucket_idx = self._matching_leading_bits(self.relay_public_key_bytes, peer_public_key_bytes)
+        if len(self.buckets[bucket_idx]) < self.bucket_size:
+            bucket = self.buckets[bucket_idx]
+            if peer_public_key_bytes not in bucket:
+                bucket.append(peer_public_key_bytes)
+        if peer is not None:
+            self.peers[peer_public_key_bytes] = peer
+
+    def remove_peer(self, peer_public_key: PeerKey):
+        peer_public_key_bytes = self._normalize_peer_key(peer_public_key)
+        bucket_idx = self._matching_leading_bits(self.relay_public_key_bytes, peer_public_key_bytes)
+        bucket = self.buckets.get(bucket_idx)
+        if not bucket:
+            return
+        try:
+            bucket.remove(peer_public_key_bytes)
+        except ValueError:
+            pass
+        self.peers.pop(peer_public_key_bytes, None)
+
+    def closest_peer_for_hash(self, target_hash: bytes) -> Optional[Peer]:
+        """Return the peer with the minimal XOR distance to ``target_hash``."""
+        if not isinstance(target_hash, (bytes, bytearray)):
+            raise TypeError("target_hash must be bytes-like")
+
+        target = bytes(target_hash)
+        if len(target) != len(self.relay_public_key_bytes):
+            raise ValueError("target_hash must match peer key length (32 bytes)")
+
+        closest_key: Optional[bytes] = None
+        closest_distance: Optional[int] = None
+
+        for bucket in self.buckets.values():
+            for peer_key in bucket:
+                try:
+                    distance = self._xor_distance(target, peer_key)
+                except ValueError:
+                    continue
+                if closest_distance is None or distance < closest_distance:
+                    closest_distance = distance
+                    closest_key = peer_key
+
+        if closest_key is None:
+            return None
+        peer = self.peers.get(closest_key)
+        return peer

astreum/_communication/setup.py

@@ -0,0 +1,322 @@
+import socket, threading
+from datetime import datetime, timezone
+from queue import Queue
+from typing import Tuple, Optional
+from cryptography.hazmat.primitives.asymmetric import ed25519
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.x25519 import (
+    X25519PrivateKey,
+    X25519PublicKey,
+)
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from .. import Node
+
+from . import Route, Message
+from .message import MessageTopic
+from .peer import Peer
+from .ping import Ping
+from .util import address_str_to_host_and_port
+
+def load_x25519(hex_key: Optional[str]) -> X25519PrivateKey:
+    """DH key for relaying (always X25519)."""
+    return 
+
+def load_ed25519(hex_key: Optional[str]) -> Optional[ed25519.Ed25519PrivateKey]:
+    """Signing key for validation (Ed25519), or None if absent."""
+    return ed25519.Ed25519PrivateKey.from_private_bytes(bytes.fromhex(hex_key)) \
+           if hex_key else None
+
+def make_routes(
+    relay_pk: X25519PublicKey,
+    val_sk: Optional[ed25519.Ed25519PrivateKey]
+) -> Tuple[Route, Optional[Route]]:
+    """Peer route (DH pubkey) + optional validation route (ed pubkey)."""
+    peer_rt = Route(relay_pk)
+    val_rt  = Route(val_sk.public_key()) if val_sk else None
+    return peer_rt, val_rt
+
+def setup_outgoing(
+    use_ipv6: bool
+) -> Tuple[socket.socket, Queue, threading.Thread]:
+    fam  = socket.AF_INET6 if use_ipv6 else socket.AF_INET
+    sock = socket.socket(fam, socket.SOCK_DGRAM)
+    q    = Queue()
+    thr  = threading.Thread(target=lambda: None, daemon=True)
+    thr.start()
+    return sock, q, thr
+
+def make_maps():
+    """Empty lookup maps: peers and addresses."""
+    return
+
+
+def process_incoming_messages(node: "Node") -> None:
+    """Process incoming messages (placeholder)."""
+    while True:
+        try:
+            data, addr = node.incoming_queue.get()
+        except Exception as exc:
+            print(f"Error taking from incoming queue: {exc}")
+            continue
+
+        try:
+            message = Message.from_bytes(data)
+        except Exception as exc:
+            print(f"Error decoding message: {exc}")
+            continue
+
+        if message.handshake:
+            sender_key = message.sender
+
+            try:
+                sender_public_key_bytes = sender_key.public_bytes(
+                    encoding=serialization.Encoding.Raw,
+                    format=serialization.PublicFormat.Raw,
+                )
+            except Exception as exc:
+                print(f"Error extracting sender key bytes: {exc}")
+                continue
+
+            # Normalize remote address (IPv6 tuples may be 4 elements)
+            try:
+                host, port = addr[0], int(addr[1])
+            except Exception:
+                continue
+            address_key = (host, port)
+
+            old_key_bytes = node.addresses.get(address_key)
+            node.addresses[address_key] = sender_public_key_bytes
+
+            if old_key_bytes is None:
+                # brand-new address -> brand-new peer
+                try:
+                    peer = Peer(node.relay_secret_key, sender_key)
+                except Exception:
+                    continue
+                peer.address = address_key
+
+                node.peers[sender_public_key_bytes] = peer
+                node.peer_route.add_peer(sender_public_key_bytes, peer)
+
+                response = Message(handshake=True, sender=node.relay_public_key)
+                node.outgoing_queue.put((response.to_bytes(), address_key))
+                continue
+            
+            elif old_key_bytes == sender_public_key_bytes:
+                # existing mapping with same key -> nothing to change
+                peer = node.peers.get(sender_public_key_bytes)
+                if peer is not None:
+                    peer.address = address_key
+            
+            else:
+                # address reused with a different key -> replace peer
+                node.peers.pop(old_key_bytes, None)
+                try:
+                    node.peer_route.remove_peer(old_key_bytes)
+                except Exception:
+                    pass
+                try:
+                    peer = Peer(node.relay_secret_key, sender_key)
+                except Exception:
+                    continue
+                peer.address = address_key
+
+                node.peers[sender_public_key_bytes] = peer
+                node.peer_route.add_peer(sender_public_key_bytes, peer)
+
+        match message.topic:
+            case MessageTopic.PING:
+                try:
+                    host, port = addr[0], int(addr[1])
+                except Exception:
+                    continue
+                address_key = (host, port)
+                sender_public_key_bytes = node.addresses.get(address_key)
+                if sender_public_key_bytes is None:
+                    continue
+                peer = node.peers.get(sender_public_key_bytes)
+                if peer is None:
+                    continue
+                try:
+                    ping = Ping.from_bytes(message.content)
+                except Exception as exc:
+                    print(f"Error decoding ping: {exc}")
+                    continue
+
+                peer.timestamp = datetime.now(timezone.utc)
+                peer.latest_block = ping.latest_block
+
+                validation_route = node.validation_route
+                if validation_route is None:
+                    continue
+                if ping.is_validator:
+                    try:
+                        validation_route.add_peer(sender_public_key_bytes)
+                    except Exception:
+                        pass
+                else:
+                    try:
+                        validation_route.remove_peer(sender_public_key_bytes)
+                    except Exception:
+                        pass
+            case MessageTopic.OBJECT_REQUEST:
+                pass
+            case MessageTopic.OBJECT_RESPONSE:
+                pass
+            case MessageTopic.ROUTE_REQUEST:
+                pass
+            case MessageTopic.ROUTE_RESPONSE:
+                pass
+            case MessageTopic.TRANSACTION:
+                if node.validation_secret_key is None:
+                    continue
+                node._validation_transaction_queue.put(message.content)
+
+            case MessageTopic.STORAGE_REQUEST:
+                payload = message.content
+                if len(payload) < 32:
+                    continue
+
+                atom_id = payload[:32]
+                provider_bytes = payload[32:]
+                if not provider_bytes:
+                    continue
+
+                try:
+                    provider_str = provider_bytes.decode("utf-8")
+                except UnicodeDecodeError:
+                    continue
+
+                try:
+                    host, port = addr[0], int(addr[1])
+                except Exception:
+                    continue
+                address_key = (host, port)
+                sender_key_bytes = node.addresses.get(address_key)
+                if sender_key_bytes is None:
+                    continue
+
+                try:
+                    local_key_bytes = node.relay_public_key.public_bytes(
+                        encoding=serialization.Encoding.Raw,
+                        format=serialization.PublicFormat.Raw,
+                    )
+                except Exception:
+                    continue
+
+                def xor_distance(target: bytes, key: bytes) -> int:
+                    return int.from_bytes(
+                        bytes(a ^ b for a, b in zip(target, key)),
+                        byteorder="big",
+                        signed=False,
+                    )
+
+                self_distance = xor_distance(atom_id, local_key_bytes)
+
+                try:
+                    closest_peer = node.peer_route.closest_peer_for_hash(atom_id)
+                except Exception:
+                    closest_peer = None
+
+                if (
+                    closest_peer is not None
+                    and closest_peer.public_key_bytes != sender_key_bytes
+                ):
+                    closest_distance = xor_distance(atom_id, closest_peer.public_key_bytes)
+                    if closest_distance < self_distance:
+                        target_addr = closest_peer.address
+                        if target_addr is not None and target_addr != addr:
+                            node.outgoing_queue.put((message.to_bytes(), target_addr))
+                            continue
+
+                node.storage_index[atom_id] = provider_str.strip()
+            
+            case _:
+                continue
+
+
+def populate_incoming_messages(node: "Node") -> None:
+    """Receive UDP packets and feed the incoming queue (placeholder)."""
+    while True:
+        try:
+            data, addr = node.incoming_socket.recvfrom(4096)
+            node.incoming_queue.put((data, addr))
+        except Exception as exc:
+            print(f"Error populating incoming queue: {exc}")
+
+def communication_setup(node: "Node", config: dict):
+    node.use_ipv6              = config.get('use_ipv6', False)
+
+    # key loading
+    node.relay_secret_key      = load_x25519(config.get('relay_secret_key'))
+    node.validation_secret_key = load_ed25519(config.get('validation_secret_key'))
+
+    # derive pubs + routes
+    node.relay_public_key      = node.relay_secret_key.public_key()
+    node.validation_public_key = (
+        node.validation_secret_key.public_key().public_bytes(
+            encoding=serialization.Encoding.Raw,
+            format=serialization.PublicFormat.Raw,
+        )
+        if node.validation_secret_key
+        else None
+    )
+    node.peer_route, node.validation_route = make_routes(
+        node.relay_public_key,
+        node.validation_secret_key
+    )
+
+    # sockets + queues + threads
+    incoming_port = config.get('incoming_port', 7373)
+    fam = socket.AF_INET6 if node.use_ipv6 else socket.AF_INET
+    node.incoming_socket = socket.socket(fam, socket.SOCK_DGRAM)
+    if node.use_ipv6:
+        node.incoming_socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
+    node.incoming_socket.bind(("::" if node.use_ipv6 else "0.0.0.0", incoming_port or 0))
+    node.incoming_port = node.incoming_socket.getsockname()[1]
+    node.incoming_queue = Queue()
+    node.incoming_populate_thread = threading.Thread(
+        target=populate_incoming_messages,
+        args=(node,),
+        daemon=True,
+    )
+    node.incoming_process_thread = threading.Thread(
+        target=process_incoming_messages,
+        args=(node,),
+        daemon=True,
+    )
+    node.incoming_populate_thread.start()
+    node.incoming_process_thread.start()
+
+    (node.outgoing_socket,
+        node.outgoing_queue,
+        node.outgoing_thread
+    ) = setup_outgoing(node.use_ipv6)
+
+    # other workers & maps
+    node.object_request_queue = Queue()
+    node.peer_manager_thread  = threading.Thread(
+        target=node._relay_peer_manager,
+        daemon=True
+    )
+    node.peer_manager_thread.start()
+
+    node.peers, node.addresses = {}, {} # peers: Dict[bytes,Peer], addresses: Dict[(str,int),bytes]
+    latest_hash = getattr(node, "latest_block_hash", None)
+    if not isinstance(latest_hash, (bytes, bytearray)) or len(latest_hash) != 32:
+        node.latest_block_hash = bytes(32)
+    else:
+        node.latest_block_hash = bytes(latest_hash)
+
+    # bootstrap pings
+    for addr in config.get('bootstrap', []):
+        try:
+            host, port = address_str_to_host_and_port(addr)  # type: ignore[arg-type]
+        except Exception:
+            continue
+
+        handshake_message = Message(handshake=True, sender=node.relay_public_key)
+        
+        node.outgoing_queue.put((handshake_message.to_bytes(), (host, port)))