astreum 0.2.39__py3-none-any.whl → 0.2.41__py3-none-any.whl

astreum/_consensus/setup.py

@@ -1,15 +1,14 @@
 from __future__ import annotations
 
 import threading
-import time
-from queue import Empty, Queue
-from typing import Any, Dict, Optional, Tuple
+from queue import Queue
+from typing import Any
 
-from .block import Block
-from .chain import Chain
-from .fork import Fork
-from .transaction import Transaction
-from .._storage.atom import ZERO32, Atom
+from .workers import (
+    make_discovery_worker,
+    make_validation_worker,
+    make_verify_worker,
+)
 
 
 def current_validator(node: Any) -> bytes:
@@ -17,11 +16,6 @@ def current_validator(node: Any) -> bytes:
     raise NotImplementedError("current_validator must be implemented by the host node")
 
 
-def apply_transaction(node: Any, block: object, transaction_hash: bytes) -> None:
-    """Apply transaction to the candidate block. Override downstream."""
-    pass
-
-
 def consensus_setup(node: Any) -> None:
     # Shared state
     node.validation_lock = getattr(node, "validation_lock", threading.RLock())
@@ -52,158 +46,23 @@ def consensus_setup(node: Any) -> None:
 
     node.enqueue_transaction_hash = enqueue_transaction_hash
 
-    def _process_peers_latest_block(latest_block_hash: bytes, peer_ids: set[Any]) -> None:
-        """Assign a peer to a fork for its latest block without merging forks.
-
-        Flow:
-        - Create a new Fork for `latest_block_hash` and validate it, using
-          stop_heads composed of current fork heads to short-circuit when
-          ancestry meets an existing fork head.
-        - If a matching fork head is found and is not malicious, copy its
-          structural fields (root, validated_upto, chain_fork_position) onto
-          the new fork.
-        - Add all peers in `peer_ids` to the new fork and remove each from any
-          previous fork they followed.
-        - Persist the new fork under `node.forks[latest_block_hash]`.
-        """
-        new_fork = Fork(head=latest_block_hash)
-
-        current_fork_heads = {fk.head for fk in node.forks.values() if fk.head != latest_block_hash}
-
-        new_fork.validate(storage_get=node._local_get, stop_heads=current_fork_heads)
-
-        # update new_fork with details of the fork with head of validated_upto
-        if new_fork.validated_upto and new_fork.validated_upto in node.forks:
-            ref = node.forks[new_fork.validated_upto]
-            # if the matched fork is malicious, disregard this new fork entirely
-            if getattr(ref, "malicious_block_hash", None):
-                return
-            # copy structural fields exactly
-            new_fork.root = ref.root
-            new_fork.validated_upto = ref.validated_upto
-            new_fork.chain_fork_position = ref.chain_fork_position
-
-        # add peers to new fork and remove them from any old forks
-        for peer_id in peer_ids:
-            new_fork.add_peer(peer_id)
-            # Remove this peer from all other forks
-            for h, fk in list(node.forks.items()):
-                if h != latest_block_hash:
-                    fk.remove_peer(peer_id)
-
-        # persist the fork
-        node.forks[latest_block_hash] = new_fork
-
-
-    # Discovery worker: watches peers and enqueues head changes
-    def _discovery_worker():
-        stop = node._validation_stop_event
-        while not stop.is_set():
-            try:
-                peers = getattr(node, "peers", None)
-                if isinstance(peers, dict):
-                    # Snapshot as (peer_id, latest_block_hash) pairs
-                    pairs = [
-                        (peer_id, bytes(latest))
-                        for peer_id, peer in list(peers.items())
-                        if isinstance((latest := getattr(peer, "latest_block", None)), (bytes, bytearray)) and latest
-                    ]
-                    # Group peers by latest block hash
-                    latest_keys = {hb for _, hb in pairs}
-                    grouped: Dict[bytes, set[Any]] = {
-                        hb: {pid for pid, phb in pairs if phb == hb}
-                        for hb in latest_keys
-                    }
-
-                    # Replace queue contents with current groups
-                    try:
-                        while True:
-                            node._validation_verify_queue.get_nowait()
-                    except Empty:
-                        pass
-                    for latest_b, peer_set in grouped.items():
-                        node._validation_verify_queue.put((latest_b, peer_set))
-            except Exception:
-                pass
-            finally:
-                time.sleep(0.5)
-
-    # Verification worker: computes root/height and applies peer→fork assignment
-    def _verify_worker():
-        stop = node._validation_stop_event
-        while not stop.is_set():
-            # Take a snapshot of all currently queued groups
-            batch: list[tuple[bytes, set[Any]]] = []
-            try:
-                while True:
-                    item = node._validation_verify_queue.get_nowait()
-                    batch.append(item)
-            except Empty:
-                pass
-
-            if not batch:
-                time.sleep(0.1)
-                continue
-
-            # Process the snapshot; new items enqueued during processing
-            # will be handled in the next iteration
-            for latest_b, peers in batch:
-                try:
-                    _process_peers_latest_block(latest_b, peers)
-                except Exception:
-                    pass
-
-    def _validation_worker() -> None:
-        """Consume pending transactions when scheduled to validate."""
-        stop = node._validation_stop_event
-        while not stop.is_set():
-            validation_public_key = getattr(node, "validation_public_key", None)
-            if not validation_public_key:
-                time.sleep(0.5)
-                continue
-
-            scheduled_validator = current_validator(node)
-
-            if scheduled_validator != validation_public_key:
-                time.sleep(0.5)
-                continue
-
-            try:
-                current_hash = node._validation_transaction_queue.get_nowait()
-            except Empty:
-                time.sleep(0.1)
-                continue
-
-            new_block = Block()
-            new_block.validator_public_key = getattr(node, "validation_public_key", None)
-            
-            while True:
-                try:
-                    apply_transaction(node, new_block, current_hash)
-                except NotImplementedError:
-                    node._validation_transaction_queue.put(current_hash)
-                    time.sleep(0.5)
-                    break
-                except Exception:
-                    # Skip problematic transaction; leave block as-is.
-                    pass
-
-                try:
-                    current_hash = node._validation_transaction_queue.get_nowait()
-                except Empty:
-                    break
+    verify_worker = make_verify_worker(node)
+    validation_worker = make_validation_worker(
+        node, current_validator=current_validator
+    )
 
     # Start workers as daemons
-    node.validation_discovery_thread = threading.Thread(
-        target=_discovery_worker, daemon=True, name="validation-discovery"
+    discovery_worker = make_discovery_worker(node)
+    node.consensus_discovery_thread = threading.Thread(
+        target=discovery_worker, daemon=True, name="consensus-discovery"
     )
-    node.validation_verify_thread = threading.Thread(
-        target=_verify_worker, daemon=True, name="validation-verify"
+    node.consensus_verify_thread = threading.Thread(
+        target=verify_worker, daemon=True, name="consensus-verify"
     )
-    node.validation_worker_thread = threading.Thread(
-        target=_validation_worker, daemon=True, name="validation-worker"
+    node.consensus_validation_thread = threading.Thread(
+        target=validation_worker, daemon=True, name="consensus-validation"
     )
-    node.validation_discovery_thread.start()
-    node.validation_verify_thread.start()
+    node.consensus_discovery_thread.start()
+    node.consensus_verify_thread.start()
     if getattr(node, "validation_secret_key", None):
-        node.validation_worker_thread.start()
+        node.consensus_validation_thread.start()

astreum/_consensus/transaction.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Callable, List, Optional, Tuple
+from typing import Any, List, Optional, Tuple
 
 from .._storage.atom import Atom, ZERO32
+from .receipt import Receipt, STATUS_SUCCESS
 
 
 def _int_to_be_bytes(value: Optional[int]) -> bytes:
@@ -22,12 +23,6 @@ def _be_bytes_to_int(data: Optional[bytes]) -> int:
     return int.from_bytes(data, "big")
 
 
-def _make_typed_bytes(payload: bytes) -> Tuple[bytes, List[Atom]]:
-    value_atom = Atom.from_data(data=payload)
-    type_atom = Atom.from_data(data=b"byte", next_hash=value_atom.object_id())
-    return type_atom.object_id(), [value_atom, type_atom]
-
-
 def _make_list(child_ids: List[bytes]) -> Tuple[bytes, List[Atom]]:
     atoms: List[Atom] = []
     next_hash = ZERO32
@@ -54,6 +49,7 @@ class Transaction:
     recipient: bytes = b""
     sender: bytes = b""
     signature: bytes = b""
+    hash: bytes = ZERO32
 
     def to_atom(self) -> Tuple[bytes, List[Atom]]:
         """Serialise the transaction, returning (object_id, atoms)."""
@@ -61,9 +57,9 @@ class Transaction:
         acc: List[Atom] = []
 
         def emit(payload: bytes) -> None:
-            oid, atoms = _make_typed_bytes(payload)
-            body_child_ids.append(oid)
-            acc.extend(atoms)
+            atom = Atom.from_data(data=payload)
+            body_child_ids.append(atom.object_id())
+            acc.append(atom)
 
         emit(_int_to_be_bytes(self.amount))
         emit(_int_to_be_bytes(self.counter))
@@ -90,9 +86,13 @@ class Transaction:
     @classmethod
     def from_atom(
         cls,
-        storage_get: Callable[[bytes], Optional[Atom]],
+        node: Any,
         transaction_id: bytes,
     ) -> Transaction:
+        storage_get = node._local_get
+        if not callable(storage_get):
+            raise NotImplementedError("node does not expose a storage getter")
+
         top_type_atom = storage_get(transaction_id)
         if top_type_atom is None or top_type_atom.data != b"list":
             raise ValueError("not a transaction (outer list missing)")
@@ -141,23 +141,20 @@ class Transaction:
         if len(body_entries) < 5:
             body_entries.extend([ZERO32] * (5 - len(body_entries)))
 
-        def read_typed_bytes(entry_id: bytes) -> bytes:
-            if not entry_id or entry_id == ZERO32:
+        def read_detail_bytes(entry_id: bytes) -> bytes:
+            if entry_id == ZERO32:
                 return b""
             elem = storage_get(entry_id)
             if elem is None:
                 return b""
-            type_atom = storage_get(elem.data)
-            if type_atom is None or type_atom.data != b"byte":
-                return b""
-            value_atom = storage_get(type_atom.next)
-            return value_atom.data if value_atom is not None else b""
+            detail_atom = storage_get(elem.data)
+            return detail_atom.data if detail_atom is not None else b""
 
-        amount_bytes = read_typed_bytes(body_entries[0])
-        counter_bytes = read_typed_bytes(body_entries[1])
-        data_bytes = read_typed_bytes(body_entries[2])
-        recipient_bytes = read_typed_bytes(body_entries[3])
-        sender_bytes = read_typed_bytes(body_entries[4])
+        amount_bytes = read_detail_bytes(body_entries[0])
+        counter_bytes = read_detail_bytes(body_entries[1])
+        data_bytes = read_detail_bytes(body_entries[2])
+        recipient_bytes = read_detail_bytes(body_entries[3])
+        sender_bytes = read_detail_bytes(body_entries[4])
 
         signature_atom = storage_get(signature_atom_id)
         signature_bytes = signature_atom.data if signature_atom is not None else b""
@@ -169,4 +166,27 @@ class Transaction:
             recipient=recipient_bytes,
             sender=sender_bytes,
             signature=signature_bytes,
+            hash=bytes(transaction_id),
         )
+
+
+def apply_transaction(node: Any, block: object, transaction_hash: bytes) -> None:
+    """Apply transaction to the candidate block. Override downstream."""
+    transaction = Transaction.from_atom(node, transaction_hash)
+
+    if block.transactions is None:
+        block.transactions = []
+    block.transactions.append(transaction)
+
+    receipt = Receipt(
+        transaction_hash=bytes(transaction_hash),
+        cost=0,
+        logs=b"",
+        status=STATUS_SUCCESS,
+    )
+    receipt.atomize()
+    if block.receipts is None:
+        block.receipts = []
+    block.receipts.append(receipt)
+
+    # Downstream implementations can extend this to apply state changes.

astreum/_consensus/workers/__init__.py

@@ -0,0 +1,9 @@
+"""
+Worker thread factories for the consensus subsystem.
+"""
+
+from .discovery import make_discovery_worker
+from .validation import make_validation_worker
+from .verify import make_verify_worker
+
+__all__ = ["make_discovery_worker", "make_verify_worker", "make_validation_worker"]

astreum/_consensus/workers/discovery.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import time
+from queue import Empty
+from typing import Any, Dict, Set, Tuple
+
+
+def make_discovery_worker(node: Any):
+    """
+    Build the discovery worker bound to the given node.
+
+    The returned callable mirrors the previous inline worker in ``setup.py``.
+    """
+
+    def _discovery_worker() -> None:
+        stop = node._validation_stop_event
+        while not stop.is_set():
+            try:
+                peers = getattr(node, "peers", None)
+                if isinstance(peers, dict):
+                    pairs: list[Tuple[Any, bytes]] = [
+                        (peer_id, bytes(latest))
+                        for peer_id, peer in list(peers.items())
+                        if isinstance(
+                            (latest := getattr(peer, "latest_block", None)),
+                            (bytes, bytearray),
+                        )
+                        and latest
+                    ]
+                    latest_keys: Set[bytes] = {hb for _, hb in pairs}
+                    grouped: Dict[bytes, set[Any]] = {
+                        hb: {pid for pid, phb in pairs if phb == hb}
+                        for hb in latest_keys
+                    }
+
+                    try:
+                        while True:
+                            node._validation_verify_queue.get_nowait()
+                    except Empty:
+                        pass
+                    for latest_b, peer_set in grouped.items():
+                        node._validation_verify_queue.put((latest_b, peer_set))
+            except Exception:
+                pass
+            finally:
+                time.sleep(0.5)
+
+    return _discovery_worker

astreum/_consensus/workers/validation.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import time
+from queue import Empty
+from typing import Any, Callable
+
+from ..block import Block
+from ..transaction import apply_transaction
+from ..._storage.atom import bytes_list_to_atoms
+from ..._storage.patricia import PatriciaTrie
+from ..._communication.message import Message, MessageTopic
+from ..._communication.ping import Ping
+
+
+def make_validation_worker(
+    node: Any,
+    *,
+    current_validator: Callable[[Any], bytes],
+) -> Callable[[], None]:
+    """Build the validation worker bound to the given node."""
+
+    def _validation_worker() -> None:
+        stop = node._validation_stop_event
+        while not stop.is_set():
+            validation_public_key = getattr(node, "validation_public_key", None)
+            if not validation_public_key:
+                time.sleep(0.5)
+                continue
+
+            scheduled_validator = current_validator(node)
+
+            if scheduled_validator != validation_public_key:
+                time.sleep(0.5)
+                continue
+
+            try:
+                current_hash = node._validation_transaction_queue.get_nowait()
+            except Empty:
+                time.sleep(0.1)
+                continue
+
+            # create thread to perform vdf
+
+            new_block = Block()
+            new_block.validator_public_key = validation_public_key
+            new_block.previous_block_hash = node.latest_block_hash
+            try:
+                new_block.previous_block = Block.from_atom(node, new_block.previous_block_hash)
+            except Exception:
+                continue
+            new_block.accounts = PatriciaTrie(root_hash=new_block.previous_block.accounts_hash)
+
+            # we may want to add a timer to process part of the txs only on a slow computer
+            while True:
+                try:
+                    apply_transaction(node, new_block, current_hash)
+                except NotImplementedError:
+                    node._validation_transaction_queue.put(current_hash)
+                    time.sleep(0.5)
+                    break
+                except Exception:
+                    pass
+
+                try:
+                    current_hash = node._validation_transaction_queue.get_nowait()
+                except Empty:
+                    break
+
+            # create an atom list of transactions, save the list head hash as the block's transactions_hash
+            transactions = new_block.transactions or []
+            tx_hashes = [bytes(tx.hash) for tx in transactions if tx.hash]
+            head_hash, _ = bytes_list_to_atoms(tx_hashes)
+            new_block.transactions_hash = head_hash
+
+            receipts = new_block.receipts or []
+            receipt_hashes = [bytes(rcpt.hash) for rcpt in receipts if rcpt.hash]
+            receipts_head, _ = bytes_list_to_atoms(receipt_hashes)
+            new_block.receipts_hash = receipts_head
+
+            # get vdf result, default to 0 for now
+
+            # get timestamp or wait for a the next second from the previous block, rule is the next block must be atleast 1 second after the previous
+            now = time.time()
+            min_allowed = new_block.previous_block.timestamp + 1
+            if now < min_allowed:
+                time.sleep(max(0.0, min_allowed - now))
+                now = time.time()
+            new_block.timestamp = max(int(now), min_allowed)
+
+            # atomize block
+            new_block_hash, _ = new_block.to_atom()
+            # put as own latest block hash
+            node.latest_block_hash = new_block_hash
+
+            # ping peers in the validation route to update there records
+            if node.validation_route and node.outgoing_queue and node.addresses:
+                route_peers = {
+                    peer_key
+                    for bucket in getattr(node.validation_route, "buckets", {}).values()
+                    for peer_key in bucket
+                }
+                if route_peers:
+                    ping_payload = Ping(
+                        is_validator=True,
+                        latest_block=new_block_hash,
+                    ).to_bytes()
+
+                    message_bytes = Message(
+                        topic=MessageTopic.PING,
+                        content=ping_payload,
+                    ).to_bytes()
+
+                    for address, peer_key in node.addresses.items():
+                        if peer_key in route_peers:
+                            try:
+                                node.outgoing_queue.put((message_bytes, address))
+                            except Exception:
+                                pass
+
+            # store the new block and receipts
+
+    return _validation_worker

astreum/_consensus/workers/verify.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import time
+from queue import Empty
+from typing import Any, Set
+
+from ..fork import Fork
+
+
+def _process_peers_latest_block(
+    node: Any, latest_block_hash: bytes, peer_ids: Set[Any]
+) -> None:
+    """Assign peers to the fork that matches their reported head."""
+    new_fork = Fork(head=latest_block_hash)
+
+    current_fork_heads = {
+        fk.head for fk in node.forks.values() if fk.head != latest_block_hash
+    }
+
+    new_fork.validate(storage_get=node._local_get, stop_heads=current_fork_heads)
+
+    if new_fork.validated_upto and new_fork.validated_upto in node.forks:
+        ref = node.forks[new_fork.validated_upto]
+        if getattr(ref, "malicious_block_hash", None):
+            return
+        new_fork.root = ref.root
+        new_fork.validated_upto = ref.validated_upto
+        new_fork.chain_fork_position = ref.chain_fork_position
+
+    for peer_id in peer_ids:
+        new_fork.add_peer(peer_id)
+        for head, fork in list(node.forks.items()):
+            if head != latest_block_hash:
+                fork.remove_peer(peer_id)
+
+    node.forks[latest_block_hash] = new_fork
+
+
+def make_verify_worker(node: Any):
+    """Build the verify worker bound to the given node."""
+
+    def _verify_worker() -> None:
+        stop = node._validation_stop_event
+        while not stop.is_set():
+            batch: list[tuple[bytes, Set[Any]]] = []
+            try:
+                while True:
+                    latest_b, peers = node._validation_verify_queue.get_nowait()
+                    batch.append((latest_b, peers))
+            except Empty:
+                pass
+
+            if not batch:
+                time.sleep(0.1)
+                continue
+
+            for latest_b, peers in batch:
+                try:
+                    _process_peers_latest_block(node, latest_b, peers)
+                except Exception:
+                    pass
+
+    return _verify_worker

astreum/_storage/atom.py

@@ -52,11 +52,11 @@ class Atom:
         data = buf[len(ZERO32):]
         return Atom(data=data, next=next_hash, size=len(data))
 
-def expr_to_atoms(e: Expr) -> Tuple[bytes, List[Atom]]:
-    def symbol(value: str) -> Tuple[bytes, List[Atom]]:
-        val = value.encode("utf-8")
-        val_atom = Atom.from_data(data=val)
-        typ_atom = Atom.from_data(b"symbol", val_atom.object_id())
+def expr_to_atoms(e: Expr) -> Tuple[bytes, List[Atom]]:
+    def symbol(value: str) -> Tuple[bytes, List[Atom]]:
+        val = value.encode("utf-8")
+        val_atom = Atom.from_data(data=val)
+        typ_atom = Atom.from_data(b"symbol", val_atom.object_id())
         return typ_atom.object_id(), [val_atom, typ_atom]
 
     def bytes(data: bytes) -> Tuple[bytes, List[Atom]]:
@@ -96,5 +96,22 @@ def expr_to_atoms(e: Expr) -> Tuple[bytes, List[Atom]]:
     if isinstance(e, Expr.Error):
         return err(e.topic, e.origin)
     if isinstance(e, Expr.ListExpr):
-        return lst(e.elements)
-    raise TypeError("unknown Expr variant")
+        return lst(e.elements)
+    raise TypeError("unknown Expr variant")
+
+
+def bytes_list_to_atoms(values: List[bytes]) -> Tuple[bytes, List[Atom]]:
+    """Build a forward-ordered linked list of atoms from byte payloads.
+
+    Returns the head object's hash (ZERO32 if no values) and the atoms created.
+    """
+    next_hash = ZERO32
+    atoms: List[Atom] = []
+
+    for value in reversed(values):
+        atom = Atom.from_data(data=bytes(value), next_hash=next_hash)
+        atoms.append(atom)
+        next_hash = atom.object_id()
+
+    atoms.reverse()
+    return (next_hash if values else ZERO32), atoms