astreum 0.2.29__py3-none-any.whl → 0.2.61__py3-none-any.whl

astreum/_consensus/workers/validation.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import time
+from queue import Empty
+from typing import Any, Callable
+
+from ..block import Block
+from ..transaction import apply_transaction
+from ..._storage.atom import bytes_list_to_atoms
+from ..._storage.patricia import PatriciaTrie
+from ..._communication.message import Message, MessageTopic
+from ..._communication.ping import Ping
+
+
+def make_validation_worker(
+    node: Any,
+    *,
+    current_validator: Callable[[Any], bytes],
+) -> Callable[[], None]:
+    """Build the validation worker bound to the given node."""
+
+    def _validation_worker() -> None:
+        stop = node._validation_stop_event
+        while not stop.is_set():
+            validation_public_key = getattr(node, "validation_public_key", None)
+            if not validation_public_key:
+                time.sleep(0.5)
+                continue
+
+            scheduled_validator = current_validator(node)
+
+            if scheduled_validator != validation_public_key:
+                time.sleep(0.5)
+                continue
+
+            try:
+                current_hash = node._validation_transaction_queue.get_nowait()
+            except Empty:
+                time.sleep(0.1)
+                continue
+
+            # create thread to perform vdf
+
+            new_block = Block()
+            new_block.validator_public_key = validation_public_key
+            new_block.previous_block_hash = node.latest_block_hash
+            try:
+                new_block.previous_block = Block.from_atom(node, new_block.previous_block_hash)
+            except Exception:
+                continue
+            new_block.accounts = PatriciaTrie(root_hash=new_block.previous_block.accounts_hash)
+
+            # we may want to add a timer to process part of the txs only on a slow computer
+            while True:
+                try:
+                    apply_transaction(node, new_block, current_hash)
+                except NotImplementedError:
+                    node._validation_transaction_queue.put(current_hash)
+                    time.sleep(0.5)
+                    break
+                except Exception:
+                    pass
+
+                try:
+                    current_hash = node._validation_transaction_queue.get_nowait()
+                except Empty:
+                    break
+
+            # create an atom list of transactions, save the list head hash as the block's transactions_hash
+            transactions = new_block.transactions or []
+            tx_hashes = [bytes(tx.hash) for tx in transactions if tx.hash]
+            head_hash, _ = bytes_list_to_atoms(tx_hashes)
+            new_block.transactions_hash = head_hash
+
+            receipts = new_block.receipts or []
+            receipt_hashes = [bytes(rcpt.hash) for rcpt in receipts if rcpt.hash]
+            receipts_head, _ = bytes_list_to_atoms(receipt_hashes)
+            new_block.receipts_hash = receipts_head
+
+            # get vdf result, default to 0 for now
+
+            # get timestamp or wait for a the next second from the previous block, rule is the next block must be atleast 1 second after the previous
+            now = time.time()
+            min_allowed = new_block.previous_block.timestamp + 1
+            if now < min_allowed:
+                time.sleep(max(0.0, min_allowed - now))
+                now = time.time()
+            new_block.timestamp = max(int(now), min_allowed)
+
+            # atomize block
+            new_block_hash, new_block_atoms = new_block.to_atom()
+            # put as own latest block hash
+            node.latest_block_hash = new_block_hash
+
+            # ping peers in the validation route to update there records
+            if node.validation_route and node.outgoing_queue and node.addresses:
+                route_peers = {
+                    peer_key
+                    for bucket in getattr(node.validation_route, "buckets", {}).values()
+                    for peer_key in bucket
+                }
+                if route_peers:
+                    ping_payload = Ping(
+                        is_validator=True,
+                        latest_block=new_block_hash,
+                    ).to_bytes()
+
+                    message_bytes = Message(
+                        topic=MessageTopic.PING,
+                        content=ping_payload,
+                    ).to_bytes()
+
+                    for address, peer_key in node.addresses.items():
+                        if peer_key in route_peers:
+                            try:
+                                node.outgoing_queue.put((message_bytes, address))
+                            except Exception:
+                                pass
+
+            # upload block atoms
+            
+            # upload receipt atoms
+            # upload account atoms
+
+    return _validation_worker

astreum/_consensus/workers/verify.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import time
+from queue import Empty
+from typing import Any, Set
+
+from ..fork import Fork
+
+
+def _process_peers_latest_block(
+    node: Any, latest_block_hash: bytes, peer_ids: Set[Any]
+) -> None:
+    """Assign peers to the fork that matches their reported head."""
+    new_fork = Fork(head=latest_block_hash)
+
+    current_fork_heads = {
+        fk.head for fk in node.forks.values() if fk.head != latest_block_hash
+    }
+
+    new_fork.validate(storage_get=node.storage_get, stop_heads=current_fork_heads)
+
+    if new_fork.validated_upto and new_fork.validated_upto in node.forks:
+        ref = node.forks[new_fork.validated_upto]
+        if getattr(ref, "malicious_block_hash", None):
+            return
+        new_fork.root = ref.root
+        new_fork.validated_upto = ref.validated_upto
+        new_fork.chain_fork_position = ref.chain_fork_position
+
+    for peer_id in peer_ids:
+        new_fork.add_peer(peer_id)
+        for head, fork in list(node.forks.items()):
+            if head != latest_block_hash:
+                fork.remove_peer(peer_id)
+
+    node.forks[latest_block_hash] = new_fork
+
+
+def make_verify_worker(node: Any):
+    """Build the verify worker bound to the given node."""
+
+    def _verify_worker() -> None:
+        stop = node._validation_stop_event
+        while not stop.is_set():
+            batch: list[tuple[bytes, Set[Any]]] = []
+            try:
+                while True:
+                    latest_b, peers = node._validation_verify_queue.get_nowait()
+                    batch.append((latest_b, peers))
+            except Empty:
+                pass
+
+            if not batch:
+                time.sleep(0.1)
+                continue
+
+            for latest_b, peers in batch:
+                try:
+                    _process_peers_latest_block(node, latest_b, peers)
+                except Exception:
+                    pass
+
+    return _verify_worker

astreum/_lispeum/__init__.py

@@ -0,0 +1,16 @@
+from .expression import Expr
+from .environment import Env
+from .low_evaluation import low_eval
+from .meter import Meter
+from .parser import parse, ParseError
+from .tokenizer import tokenize
+
+__all__ = [
+    "Env",
+    "Expr",
+    "low_eval",
+    "Meter",
+    "parse",
+    "tokenize",
+    "ParseError",
+]

astreum/_lispeum/environment.py

@@ -0,0 +1,13 @@
+from ast import Expr
+from typing import Dict, Optional
+import uuid
+
+
+class Env:
+    def __init__(
+        self,
+        data: Optional[Dict[str, Expr]] = None,
+        parent_id: Optional[uuid.UUID] = None,
+    ):
+        self.data: Dict[bytes, Expr] = {} if data is None else data
+        self.parent_id = parent_id

astreum/_lispeum/expression.py

@@ -0,0 +1,190 @@
+from typing import Any, List, Optional, Tuple
+
+from .._storage.atom import Atom, AtomKind
+
+ZERO32 = b"\x00" * 32
+ERROR_SYMBOL = "error"
+
+
+class Expr:
+    class ListExpr:
+        def __init__(self, elements: List['Expr']):
+            self.elements = elements
+        
+        def __repr__(self):
+            if not self.elements:
+                return "()"
+            inner = " ".join(str(e) for e in self.elements)
+            return f"({inner})"
+
+        def to_atoms(self):
+            return Expr.to_atoms(self)
+        
+    class Symbol:
+        def __init__(self, value: str):
+            self.value = value
+
+        def __repr__(self):
+            return f"{self.value}"
+
+        def to_atoms(self):
+            return Expr.to_atoms(self)
+        
+    class Bytes:
+        def __init__(self, value: bytes):
+            self.value = value
+
+        def __repr__(self):
+            int_value = int.from_bytes(self.value, "big") if self.value else 0
+            return f"{int_value}"
+
+        def to_atoms(self):
+            return Expr.to_atoms(self)
+    @classmethod
+    def from_atoms(cls, node: Any, root_hash: bytes) -> "Expr":
+        """Rebuild an expression tree from stored atoms."""
+        if not isinstance(root_hash, (bytes, bytearray)):
+            raise TypeError("root hash must be bytes-like")
+
+        storage_get = getattr(node, "storage_get", None)
+        if not callable(storage_get):
+            raise TypeError("node must provide a callable 'storage_get'")
+
+        expr_id = bytes(root_hash)
+
+        def _require(atom_id: Optional[bytes], context: str):
+            if not atom_id:
+                raise ValueError(f"missing atom id while decoding {context}")
+            atom = storage_get(atom_id)
+            if atom is None:
+                raise ValueError(f"missing atom data while decoding {context}")
+            return atom
+
+        def _atom_kind(atom: Any) -> Optional[AtomKind]:
+            kind_value = getattr(atom, "kind", None)
+            if isinstance(kind_value, AtomKind):
+                return kind_value
+            if isinstance(kind_value, int):
+                try:
+                    return AtomKind(kind_value)
+                except ValueError:
+                    return None
+            return None
+
+        type_atom = _require(expr_id, "expression atom")
+
+        kind_enum = _atom_kind(type_atom)
+        if kind_enum is None:
+            raise ValueError("expression atom missing kind")
+
+        if kind_enum is AtomKind.SYMBOL:
+            try:
+                return cls.Symbol(type_atom.data.decode("utf-8"))
+            except UnicodeDecodeError as exc:
+                raise ValueError("symbol atom is not valid utf-8") from exc
+
+        if kind_enum is AtomKind.BYTES:
+            return cls.Bytes(type_atom.data)
+
+        if kind_enum is AtomKind.LIST:
+            # Empty list sentinel: zero-length payload and no next pointer.
+            if len(type_atom.data) == 0 and type_atom.next == ZERO32:
+                return cls.ListExpr([])
+
+            elements: List[Expr] = []
+            current_atom = type_atom
+            idx = 0
+            while True:
+                child_hash = current_atom.data
+                if not child_hash:
+                    raise ValueError("list element missing child hash")
+                if len(child_hash) != len(ZERO32):
+                    raise ValueError("list element hash has unexpected length")
+                child_expr = cls.from_atoms(node, child_hash)
+                elements.append(child_expr)
+                next_id = current_atom.next
+                if not next_id or next_id == ZERO32:
+                    break
+                next_atom = _require(next_id, f"list element {idx}")
+                next_kind = _atom_kind(next_atom)
+                if next_kind is not AtomKind.LIST:
+                    raise ValueError("list chain contains non-list atom")
+                current_atom = next_atom
+                idx += 1
+            return cls.ListExpr(elements)
+
+        raise ValueError(f"unknown expression kind: {kind_enum}")
+
+    @staticmethod
+    def to_atoms(e: "Expr") -> Tuple[bytes, List[Atom]]:
+        def symbol(value: str) -> Tuple[bytes, List[Atom]]:
+            atom = Atom.from_data(data=value.encode("utf-8"), kind=AtomKind.SYMBOL)
+            return atom.object_id(), [atom]
+
+        def bytes_value(data: bytes) -> Tuple[bytes, List[Atom]]:
+            atom = Atom.from_data(data=data, kind=AtomKind.BYTES)
+            return atom.object_id(), [atom]
+
+        def lst(items: List["Expr"]) -> Tuple[bytes, List[Atom]]:
+            acc: List[Atom] = []
+            child_hashes: List[bytes] = []
+            for it in items:
+                h, atoms = Expr.to_atoms(it)
+                acc.extend(atoms)
+                child_hashes.append(h)
+            next_hash = ZERO32
+            elem_atoms: List[Atom] = []
+            for h in reversed(child_hashes):
+                a = Atom.from_data(h, next_hash, kind=AtomKind.LIST)
+                next_hash = a.object_id()
+                elem_atoms.append(a)
+            elem_atoms.reverse()
+            if elem_atoms:
+                head = elem_atoms[0].object_id()
+            else:
+                empty_atom = Atom.from_data(data=b"", next_hash=ZERO32, kind=AtomKind.LIST)
+                elem_atoms = [empty_atom]
+                head = empty_atom.object_id()
+            return head, acc + elem_atoms
+
+        if isinstance(e, Expr.Symbol):
+            return symbol(e.value)
+        if isinstance(e, Expr.Bytes):
+            return bytes_value(e.value)
+        if isinstance(e, Expr.ListExpr):
+            return lst(e.elements)
+        raise TypeError("unknown Expr variant")
+
+def _expr_generate_id(expr) -> bytes:
+    expr_id, _ = Expr.to_atoms(expr)
+    return expr_id
+
+
+def _expr_cached_id(expr) -> bytes:
+    cached = getattr(expr, "_cached_id", None)
+    if cached is None:
+        cached = _expr_generate_id(expr)
+        setattr(expr, "_cached_id", cached)
+    return cached
+
+
+for _expr_cls in (Expr.ListExpr, Expr.Symbol, Expr.Bytes):
+    _expr_cls.generate_id = _expr_generate_id  # type: ignore[attr-defined]
+    _expr_cls.id = property(_expr_cached_id)  # type: ignore[attr-defined]
+
+
+def error_expr(topic: str, message: str) -> Expr.ListExpr:
+    """Encode an error as (error <topic-bytes> <message-bytes>)."""
+    try:
+        topic_bytes = topic.encode("utf-8")
+    except UnicodeEncodeError as exc:
+        raise ValueError("error topic must be valid utf-8") from exc
+    try:
+        message_bytes = message.encode("utf-8")
+    except UnicodeEncodeError as exc:
+        raise ValueError("error message must be valid utf-8") from exc
+    return Expr.ListExpr([
+        Expr.Symbol(ERROR_SYMBOL),
+        Expr.Bytes(topic_bytes),
+        Expr.Bytes(message_bytes),
+    ])

astreum/_lispeum/high_evaluation.py

@@ -0,0 +1,236 @@
+from typing import List, Optional, Union
+import uuid
+
+from .environment import Env
+from .expression import Expr, error_expr, ERROR_SYMBOL
+from .meter import Meter
+
+
+def _is_error(expr: Expr) -> bool:
+    return (
+        isinstance(expr, Expr.ListExpr)
+        and bool(expr.elements)
+        and isinstance(expr.elements[0], Expr.Symbol)
+        and expr.elements[0].value == ERROR_SYMBOL
+    )
+
+
+def _hex_symbol_to_bytes(value: Optional[str]) -> Optional[bytes]:
+    if not value:
+        return None
+    data = value.strip()
+    if data.startswith(("0x", "0X")):
+        data = data[2:]
+    if len(data) % 2:
+        data = "0" + data
+    try:
+        return bytes.fromhex(data)
+    except ValueError:
+        return None
+
+
+def _expr_to_bytes(expr: Expr) -> Optional[bytes]:
+    if isinstance(expr, Expr.Bytes):
+        return expr.value
+    if isinstance(expr, Expr.Symbol):
+        return _hex_symbol_to_bytes(expr.value)
+    return None
+
+
+def high_eval(self, env_id: uuid.UUID, expr: Expr, meter = None) -> Expr:
+    if meter is None:
+        meter = Meter()
+
+    call_env_id = uuid.uuid4()
+    self.environments[call_env_id] = Env(parent_id=env_id)
+    env_id = call_env_id
+
+    try:
+        # ---------- atoms ----------
+        if _is_error(expr):
+            return expr
+
+        if isinstance(expr, Expr.Symbol):
+            bound = self.env_get(env_id, expr.value.encode())
+            if bound is None:
+                return error_expr("eval", f"unbound symbol '{expr.value}'")
+            return bound
+
+        if not isinstance(expr, Expr.ListExpr):
+            return expr  # Expr.Byte or other literals passthrough
+
+        # ---------- empty / single ----------
+        if len(expr.elements) == 0:
+            return expr
+        if len(expr.elements) == 1:
+            return self.high_eval(env_id, expr.elements[0], meter)
+
+        tail = expr.elements[-1]
+
+        # ---------- (value name def) ----------
+        if isinstance(tail, Expr.Symbol) and tail.value == "def":
+            if len(expr.elements) < 3:
+                return error_expr("eval", "def expects (value name def)")
+            name_e = expr.elements[-2]
+            if not isinstance(name_e, Expr.Symbol):
+                return error_expr("eval", "def name must be symbol")
+            value_e = expr.elements[-3]
+            value_res = self.high_eval(env_id, value_e, meter)
+            if _is_error(value_res):
+                return value_res
+            self.env_set(env_id, name_e.value.encode(), value_res)
+            return value_res
+        
+        # Reference Call
+        # (atom_id ref)
+        if isinstance(tail, Expr.Symbol) and tail.value == "ref":
+            if len(expr.elements) != 2:
+                return error_expr("eval", "ref expects (atom_id ref)")
+            key_bytes = _expr_to_bytes(expr.elements[0])
+            if not key_bytes:
+                return error_expr("eval", "ref expects (atom_id ref)")
+            stored_list = self.get_expr_list_from_storage(key_bytes)
+            if stored_list is None:
+                return error_expr("eval", "ref target not found")
+            return stored_list
+
+        # Low Level Call
+        # (arg1 arg2 ... ((body) sk))
+        if isinstance(tail, Expr.ListExpr):
+            inner = tail.elements
+            if len(inner) >= 2 and isinstance(inner[-1], Expr.Symbol) and inner[-1].value == "sk":
+                body_expr = inner[-2]
+                if not isinstance(body_expr, Expr.ListExpr):
+                    return error_expr("eval", "sk body must be list")
+
+                # helper: turn an Expr into a contiguous bytes buffer
+                def to_bytes(v: Expr) -> Union[bytes, Expr]:
+                    if isinstance(v, Expr.Byte):
+                        return bytes([v.value & 0xFF])
+                    if isinstance(v, Expr.ListExpr):
+                        # expect a list of Expr.Byte
+                        out: bytearray = bytearray()
+                        for el in v.elements:
+                            if isinstance(el, Expr.Byte):
+                                out.append(el.value & 0xFF)
+                            else:
+                                return error_expr("eval", "byte list must contain only Byte elements")
+                        return bytes(out)
+                    if _is_error(v):
+                        return v
+                    return error_expr("eval", "argument must resolve to Byte or (Byte ...)")
+
+                # resolve ALL preceding args into bytes (can be Byte or List[Byte])
+                args_exprs = expr.elements[:-1]
+                arg_bytes: List[bytes] = []
+                for a in args_exprs:
+                    v = self.high_eval(env_id, a, meter)
+                    if _is_error(v):
+                        return v
+                    vb = to_bytes(v)
+                    if not isinstance(vb, bytes):
+                        if _is_error(vb):
+                            return vb
+                        return error_expr("eval", "unexpected expression while coercing to bytes")
+                    arg_bytes.append(vb)
+
+                # build low-level code with $0-based placeholders ($0 = first arg)
+                code: List[bytes] = []
+
+                def emit(tok: Expr) -> Union[None, Expr]:
+                    if isinstance(tok, Expr.Symbol):
+                        name = tok.value
+                        if name.startswith("$"):
+                            idx_s = name[1:]
+                            if not idx_s.isdigit():
+                                return error_expr("eval", "invalid sk placeholder")
+                            idx = int(idx_s)  # $0 is first
+                            if idx < 0 or idx >= len(arg_bytes):
+                                return error_expr("eval", "arity mismatch in sk placeholder")
+                            code.append(arg_bytes[idx])
+                            return None
+                        code.append(name.encode())
+                        return None
+
+                    if isinstance(tok, Expr.Byte):
+                        code.append(bytes([tok.value & 0xFF]))
+                        return None
+
+                    if isinstance(tok, Expr.ListExpr):
+                        rv = self.high_eval(env_id, tok, meter)
+                        if _is_error(rv):
+                            return rv
+                        rb = to_bytes(rv)
+                        if not isinstance(rb, bytes):
+                            if _is_error(rb):
+                                return rb
+                            return error_expr("eval", "unexpected expression while coercing list token to bytes")
+                        code.append(rb)
+                        return None
+
+                    if _is_error(tok):
+                        return tok
+
+                    return error_expr("eval", "invalid token in sk body")
+
+                for t in body_expr.elements:
+                    err = emit(t)
+                    if err is not None and _is_error(err):
+                        return err
+
+                # Execute low-level code built from sk-body using the caller's meter
+                res = self.low_eval(code, meter=meter)
+                return res
+
+        # High Level Call
+        # (arg1 arg2 ... ((body) (params) fn))
+        if isinstance(tail, Expr.ListExpr):
+            fn_form = tail
+            if (len(fn_form.elements) >= 3
+                and isinstance(fn_form.elements[-1], Expr.Symbol)
+                and fn_form.elements[-1].value == "fn"):
+
+                body_expr   = fn_form.elements[-3]
+                params_expr = fn_form.elements[-2]
+
+                if not isinstance(body_expr, Expr.ListExpr):
+                    return error_expr("eval", "fn body must be list")
+                if not isinstance(params_expr, Expr.ListExpr):
+                    return error_expr("eval", "fn params must be list")
+
+                params: List[bytes] = []
+                for p in params_expr.elements:
+                    if not isinstance(p, Expr.Symbol):
+                        return error_expr("eval", "fn param must be symbol")
+                    params.append(p.value.encode())
+
+                args_exprs = expr.elements[:-1]
+                if len(args_exprs) != len(params):
+                    return error_expr("eval", "arity mismatch")
+
+                arg_bytes: List[bytes] = []
+                for a in args_exprs:
+                    v = self.high_eval(env_id, a, meter)
+                    if _is_error(v):
+                        return v
+                    if not isinstance(v, Expr.Byte):
+                        return error_expr("eval", "argument must resolve to Byte")
+                    arg_bytes.append(bytes([v.value & 0xFF]))
+
+                # child env, bind params -> Expr.Byte
+                child_env = uuid.uuid4()
+                self.environments[child_env] = Env(parent_id=env_id)
+                try:
+                    for name_b, val_b in zip(params, arg_bytes):
+                        self.env_set(child_env, name_b, Expr.Byte(val_b[0]))
+
+                    # evaluate HL body, metered from the top
+                    return self.high_eval(child_env, body_expr, meter)
+                finally:
+                    self.environments.pop(child_env, None)
+
+        # ---------- default: resolve each element and return list ----------
+        resolved: List[Expr] = [self.high_eval(env_id, e, meter) for e in expr.elements]
+        return Expr.ListExpr(resolved)
+    finally:
+        self.environments.pop(call_env_id, None)