astreum 0.2.41__py3-none-any.whl → 0.2.61__py3-none-any.whl

astreum/_node.py

@@ -1,10 +1,23 @@
 from __future__ import annotations
-from typing import Dict, Optional
+from pathlib import Path
+from typing import Dict, List, Optional
 import uuid
 import threading
 
-from src.astreum._storage.atom import Atom
-from src.astreum._lispeum import Env, Expr, low_eval, parse, tokenize, ParseError
+from astreum._storage.atom import AtomKind
+
+from ._storage import Atom, storage_setup
+from ._lispeum import Env, Expr, Meter, low_eval, parse, tokenize, ParseError
+from .utils.logging import logging_setup
+
+__all__ = [
+    "Node",
+    "Env",
+    "Expr",
+    "Meter",
+    "parse",
+    "tokenize",
+]
 
 def bytes_touched(*vals: bytes) -> int:
     """For metering: how many bytes were manipulated (max of operands)."""
@@ -12,9 +25,10 @@ def bytes_touched(*vals: bytes) -> int:
 
 class Node:
     def __init__(self, config: dict):
+        self.logger = logging_setup(config)
+        self.logger.info("Starting Astreum Node")
         # Storage Setup
-        self.in_memory_storage: Dict[bytes, Atom] = {}
-        self.in_memory_storage_lock = threading.RLock()
+        storage_setup(self, config=config)
         # Lispeum Setup
         self.environments: Dict[uuid.UUID, Env] = {}
         self.machine_environments_lock = threading.RLock()
@@ -25,11 +39,13 @@ class Node:
             communication_setup(node=self, config=config)
         except Exception:
             pass
-        try:
-            from astreum._consensus import consensus_setup  # type: ignore
-            consensus_setup(node=self)
+        try:
+            from astreum._consensus import consensus_setup  # type: ignore
+            consensus_setup(node=self, config=config)
         except Exception:
             pass
+        
+
 
     # ---- Env helpers ----
     def env_get(self, env_id: uuid.UUID, key: bytes) -> Optional[Expr]:
@@ -49,10 +65,134 @@ class Node:
             return True
 
     # Storage
-    def _local_get(self, key: bytes) -> Optional[Atom]:
-        with self.in_memory_storage_lock:
-            return self.in_memory_storage.get(key)
+    def _hot_storage_get(self, key: bytes) -> Optional[Atom]:
+        atom = self.hot_storage.get(key)
+        if atom is not None:
+            self.hot_storage_hits[key] = self.hot_storage_hits.get(key, 0) + 1
+        return atom
+
+    def _hot_storage_set(self, key: bytes, value: Atom) -> bool:
+        """Store atom in hot storage without exceeding the configured limit."""
+        projected = self.hot_storage_size + value.size
+        if projected > self.hot_storage_limit:
+            return False
+
+        self.hot_storage[key] = value
+        self.hot_storage_size = projected
+        return True
+
+    def _network_get(self, key: bytes) -> Optional[Atom]:
+        # locate storage provider
+        # query storage provider
+        return None
+
+    def storage_get(self, key: bytes) -> Optional[Atom]:
+        """Retrieve an Atom by checking local storage first, then the network."""
+        atom = self._hot_storage_get(key)
+        if atom is not None:
+            return atom
+        atom = self._cold_storage_get(key)
+        if atom is not None:
+            return atom
+        return self._network_get(key)
+
+    def _cold_storage_get(self, key: bytes) -> Optional[Atom]:
+        """Read an atom from the cold storage directory if configured."""
+        if not self.cold_storage_path:
+            return None
+        filename = f"{key.hex().upper()}.bin"
+        file_path = Path(self.cold_storage_path) / filename
+        try:
+            data = file_path.read_bytes()
+        except FileNotFoundError:
+            return None
+        except OSError:
+            return None
+        try:
+            return Atom.from_bytes(data)
+        except ValueError:
+            return None
+
+    def _cold_storage_set(self, atom: Atom) -> None:
+        """Persist an atom into the cold storage directory if it already exists."""
+        if not self.cold_storage_path:
+            return
+        atom_bytes = atom.to_bytes()
+        projected = self.cold_storage_size + len(atom_bytes)
+        if self.cold_storage_limit and projected > self.cold_storage_limit:
+            return
+        directory = Path(self.cold_storage_path)
+        if not directory.exists():
+            return
+        atom_id = atom.object_id()
+        filename = f"{atom_id.hex().upper()}.bin"
+        file_path = directory / filename
+        try:
+            file_path.write_bytes(atom_bytes)
+            self.cold_storage_size = projected
+        except OSError:
+            return
+
+    def _network_set(self, atom: Atom) -> None:
+        """Advertise an atom to the closest known peer so they can fetch it from us."""
+        try:
+            from ._communication.message import Message, MessageTopic
+        except Exception:
+            return
+
+        atom_id = atom.object_id()
+        try:
+            closest_peer = self.peer_route.closest_peer_for_hash(atom_id)
+        except Exception:
+            return
+        if closest_peer is None or closest_peer.address is None:
+            return
+        target_addr = closest_peer.address
+
+        try:
+            provider_ip, provider_port = self.incoming_socket.getsockname()[:2]
+        except Exception:
+            return
+
+        provider_str = f"{provider_ip}:{int(provider_port)}"
+        try:
+            provider_bytes = provider_str.encode("utf-8")
+        except Exception:
+            return
+
+        payload = atom_id + provider_bytes
+        message = Message(topic=MessageTopic.STORAGE_REQUEST, content=payload)
+        self.outgoing_queue.put((message.to_bytes(), target_addr))
+
+    def get_expr_list_from_storage(self, key: bytes) -> Optional["ListExpr"]:
+        atoms = self.get_atom_list_from_storage(root_hash=key)
+        if atoms is None:
+            return None
+        
+        expr_list = []
+        for atom in atoms:
+            match atom.kind:
+                case AtomKind.SYMBOL:
+                    expr_list.append(Expr.Symbol(atom.data))
+                case AtomKind.BYTES:
+                    expr_list.append(Expr.Bytes(atom.data))
+                case AtomKind.LIST:
+                    expr_list.append(Expr.ListExpr([
+                        Expr.Bytes(atom.data),
+                        Expr.Symbol("ref")
+                    ]))
 
-    def _local_set(self, key: bytes, value: Atom) -> None:
-        with self.in_memory_storage_lock:
-            self.in_memory_storage[key] = value
+        expr_list.reverse()
+        return Expr.ListExpr(expr_list)
+    
+    def get_atom_list_from_storage(self, root_hash: bytes) -> Optional[List["Atom"]]:
+        next_id: Optional[bytes] = root_hash
+        atom_list: List["Atom"] = []
+        while next_id:
+            elem = self.storage_get(key=next_id)
+            if elem:
+                atom_list.append(elem)
+                next_id = elem.next
+            else:
+                return None
+        return atom_list

astreum/_storage/__init__.py

@@ -1,5 +1,7 @@
-from .atom import Atom
-
-__all__ = [
-    "Atom",
-]
+from .atom import Atom
+from .setup import storage_setup
+
+__all__ = [
+    "Atom",
+    "storage_setup",
+]

astreum/_storage/atom.py

@@ -1,9 +1,9 @@
 
 
-from typing import List, Optional, Tuple
+from enum import IntEnum
+from typing import List, Optional, Tuple
 
-from .._lispeum.expression import Expr
-from blake3 import blake3
+from blake3 import blake3
 
 ZERO32 = b"\x00"*32
 
@@ -13,105 +13,97 @@ def u64_le(n: int) -> bytes:
 def hash_bytes(b: bytes) -> bytes:
     return blake3(b).digest()
 
-class Atom:
+class AtomKind(IntEnum):
+    SYMBOL = 0
+    BYTES = 1
+    LIST = 2
+
+
+class Atom:
     data: bytes
     next: bytes
     size: int
 
-    def __init__(self, data: bytes, next: bytes = ZERO32, size: Optional[int] = None):
-        self.data = data
-        self.next = next
-        self.size = len(data) if size is None else size
-
-    @staticmethod
-    def from_data(data: bytes, next_hash: bytes = ZERO32) -> "Atom":
-        return Atom(data=data, next=next_hash, size=len(data))
-
-    @staticmethod
-    def object_id_from_parts(data_hash: bytes, next_hash: bytes, size: int) -> bytes:
-        return blake3(data_hash + next_hash + u64_le(size)).digest()
-
-    def data_hash(self) -> bytes:
-        return hash_bytes(self.data)
-
-    def object_id(self) -> bytes:
-        return self.object_id_from_parts(self.data_hash(), self.next, self.size)
-
-    @staticmethod
-    def verify_metadata(object_id: bytes, size: int, next_hash: bytes, data_hash: bytes) -> bool:
-        return object_id == blake3(data_hash + next_hash + u64_le(size)).digest()
-
-    def to_bytes(self) -> bytes:
-        return self.next + self.data
-
-    @staticmethod
-    def from_bytes(buf: bytes) -> "Atom":
-        if len(buf) < len(ZERO32):
-            raise ValueError("buffer too short for Atom header")
-        next_hash = buf[:len(ZERO32)]
-        data = buf[len(ZERO32):]
-        return Atom(data=data, next=next_hash, size=len(data))
-
-def expr_to_atoms(e: Expr) -> Tuple[bytes, List[Atom]]:
-    def symbol(value: str) -> Tuple[bytes, List[Atom]]:
-        val = value.encode("utf-8")
-        val_atom = Atom.from_data(data=val)
-        typ_atom = Atom.from_data(b"symbol", val_atom.object_id())
-        return typ_atom.object_id(), [val_atom, typ_atom]
-
-    def bytes(data: bytes) -> Tuple[bytes, List[Atom]]:
-        val_atom = Atom.from_data(data=data)
-        typ_atom = Atom.from_data(b"byte", val_atom.object_id())
-        return typ_atom.object_id(), [val_atom, typ_atom]
-
-    def err(topic: str, origin: Optional[Expr]) -> Tuple[bytes, List[Atom]]:
-        topic_bytes = topic.encode("utf-8")
-        topic_atom = Atom.from_data(data=topic_bytes)
-        typ_atom = Atom.from_data(data=b"error", next_hash=topic_atom.object_id())
-        return typ_atom.object_id(), [topic_atom, typ_atom]
-
-    def lst(items: List[Expr]) -> Tuple[bytes, List[Atom]]:
-        acc: List[Atom] = []
-        child_hashes: List[bytes] = []
-        for it in items:
-            h, atoms = expr_to_atoms(it)
-            acc.extend(atoms)
-            child_hashes.append(h)
-        next_hash = ZERO32
-        elem_atoms: List[Atom] = []
-        for h in reversed(child_hashes):
-            a = Atom.from_data(h, next_hash)
-            next_hash = a.object_id()
-            elem_atoms.append(a)
-        elem_atoms.reverse()
-        head = next_hash
-        val_atom = Atom.from_data(data=u64_le(len(items)), next_hash=head)
-        typ_atom = Atom.from_data(data=b"list", next_hash=val_atom.object_id())
-        return typ_atom.object_id(), acc + elem_atoms + [val_atom, typ_atom]
-
-    if isinstance(e, Expr.Symbol):
-        return symbol(e.value)
-    if isinstance(e, Expr.Bytes):
-        return bytes(e.value)
-    if isinstance(e, Expr.Error):
-        return err(e.topic, e.origin)
-    if isinstance(e, Expr.ListExpr):
-        return lst(e.elements)
-    raise TypeError("unknown Expr variant")
+    def __init__(
+        self,
+        data: bytes,
+        next: bytes = ZERO32,
+        size: Optional[int] = None,
+        kind: AtomKind = AtomKind.BYTES,
+    ):
+        self.data = data
+        self.next = next
+        self.size = len(data) if size is None else size
+        self.kind = kind
 
+    @staticmethod
+    def from_data(
+        data: bytes,
+        next_hash: bytes = ZERO32,
+        kind: AtomKind = AtomKind.BYTES,
+    ) -> "Atom":
+        return Atom(data=data, next=next_hash, size=len(data), kind=kind)
+
+    def generate_id(self) -> bytes:
+        """Compute the object id using this atom's metadata."""
+        kind_bytes = int(self.kind).to_bytes(1, "little", signed=False)
+        return blake3(
+            kind_bytes + self.data_hash() + self.next + u64_le(self.size)
+        ).digest()
 
-def bytes_list_to_atoms(values: List[bytes]) -> Tuple[bytes, List[Atom]]:
-    """Build a forward-ordered linked list of atoms from byte payloads.
+    def data_hash(self) -> bytes:
+        return hash_bytes(self.data)
 
-    Returns the head object's hash (ZERO32 if no values) and the atoms created.
-    """
-    next_hash = ZERO32
-    atoms: List[Atom] = []
+    def object_id(self) -> bytes:
+        return self.generate_id()
 
-    for value in reversed(values):
-        atom = Atom.from_data(data=bytes(value), next_hash=next_hash)
-        atoms.append(atom)
-        next_hash = atom.object_id()
+    @staticmethod
+    def verify_metadata(
+        object_id: bytes,
+        size: int,
+        next_hash: bytes,
+        data_hash: bytes,
+        kind: AtomKind,
+    ) -> bool:
+        kind_bytes = int(kind).to_bytes(1, "little", signed=False)
+        expected = blake3(kind_bytes + data_hash + next_hash + u64_le(size)).digest()
+        return object_id == expected
+
+    def to_bytes(self) -> bytes:
+        """Serialize as next-hash + kind byte + payload."""
+        kind_byte = int(self.kind).to_bytes(1, "little", signed=False)
+        return self.next + kind_byte + self.data
 
-    atoms.reverse()
-    return (next_hash if values else ZERO32), atoms
+    @staticmethod
+    def from_bytes(buf: bytes) -> "Atom":
+        header_len = len(ZERO32)
+        if len(buf) < header_len + 1:
+            raise ValueError("buffer too short for Atom header")
+        next_hash = buf[:header_len]
+        kind_value = buf[header_len]
+        data = buf[header_len + 1 :]
+        try:
+            kind = AtomKind(kind_value)
+        except ValueError as exc:
+            raise ValueError(f"unknown atom kind: {kind_value}") from exc
+        return Atom(data=data, next=next_hash, size=len(data), kind=kind)
+
+def bytes_list_to_atoms(values: List[bytes]) -> Tuple[bytes, List[Atom]]:
+    """Build a forward-ordered linked list of atoms from byte payloads.
+
+    Returns the head object's hash (ZERO32 if no values) and the atoms created.
+    """
+    next_hash = ZERO32
+    atoms: List[Atom] = []
+
+    for value in reversed(values):
+        atom = Atom.from_data(
+            data=bytes(value),
+            next_hash=next_hash,
+            kind=AtomKind.BYTES,
+        )
+        atoms.append(atom)
+        next_hash = atom.object_id()
+
+    atoms.reverse()
+    return (next_hash if values else ZERO32), atoms

astreum/_storage/patricia.py

@@ -1,7 +1,7 @@
 import blake3
 from typing import Dict, List, Optional, Tuple, TYPE_CHECKING
 
-from .atom import Atom, ZERO32
+from .atom import Atom, AtomKind, ZERO32
 
 if TYPE_CHECKING:
     from .._node import Node
@@ -43,10 +43,11 @@ class PatriciaNode:
     
     def to_atoms(self) -> Tuple[bytes, List[Atom]]:
         """
-        Materialise this node as a flat atom chain containing the fields in a
-        traversal-friendly order: key length prefix (single byte) + key bytes,
-        child_0 hash, child_1 hash, and value bytes. Returns the head atom hash
-        and the atom list.
+        Materialise this node with the canonical atom layout used by the
+        storage layer: a leading SYMBOL atom with payload ``b"radix"`` whose
+        ``next`` pointer links to four BYTES atoms containing, in order:
+        key (len byte + key payload), child_0 hash, child_1 hash, value bytes.
+        Returns the top atom hash and the emitted atoms.
         """
         if self.key_len > 255:
             raise ValueError("Patricia key length > 255 bits cannot be encoded in a single atom field")
@@ -58,16 +59,23 @@ class PatriciaNode:
             self.value or b"",
         ]
 
-        atoms: List[Atom] = []
+        data_atoms: List[Atom] = []
         next_hash = ZERO32
         for payload in reversed(entries):
-            atom = Atom.from_data(data=payload, next_hash=next_hash)
-            atoms.append(atom)
+            atom = Atom.from_data(data=payload, next_hash=next_hash, kind=AtomKind.BYTES)
+            data_atoms.append(atom)
             next_hash = atom.object_id()
 
-        head = next_hash
-        atoms.reverse()
-        return head, atoms
+        data_atoms.reverse()
+
+        type_atom = Atom.from_data(
+            data=b"radix",
+            next_hash=next_hash,
+            kind=AtomKind.SYMBOL,
+        )
+
+        atoms = data_atoms + [type_atom]
+        return type_atom.object_id(), atoms
 
     @classmethod
     def from_atoms(
@@ -82,19 +90,46 @@ class PatriciaNode:
         if head_hash == ZERO32:
             raise ValueError("empty atom chain for Patricia node")
 
+        def _atom_kind(atom: Optional[Atom]) -> Optional[AtomKind]:
+            kind_value = getattr(atom, "kind", None)
+            if isinstance(kind_value, AtomKind):
+                return kind_value
+            if isinstance(kind_value, int):
+                try:
+                    return AtomKind(kind_value)
+                except ValueError:
+                    return None
+            return None
+
+        def _require_atom(atom_hash: Optional[bytes], context: str) -> Atom:
+            if not atom_hash or atom_hash == ZERO32:
+                raise ValueError(f"missing {context}")
+            atom = node.storage_get(atom_hash)
+            if atom is None:
+                raise ValueError(f"missing {context}")
+            return atom
+
+        type_atom = _require_atom(head_hash, "Patricia type atom")
+        if _atom_kind(type_atom) is not AtomKind.SYMBOL:
+            raise ValueError("malformed Patricia node (type atom kind)")
+        if type_atom.data != b"radix":
+            raise ValueError("not a Patricia node (type mismatch)")
+
         entries: List[bytes] = []
-        current = head_hash
+        current = type_atom.next
         hops = 0
 
-        while current != ZERO32 and hops < 4:
-            atom = node._local_get(current)
+        while current and current != ZERO32 and hops < 4:
+            atom = node.storage_get(current)
             if atom is None:
                 raise ValueError("missing atom while decoding Patricia node")
+            if _atom_kind(atom) is not AtomKind.BYTES:
+                raise ValueError("Patricia node detail atoms must be bytes")
             entries.append(atom.data)
             current = atom.next
             hops += 1
 
-        if current != ZERO32:
+        if current and current != ZERO32:
             raise ValueError("too many fields while decoding Patricia node")
 
         if len(entries) != 4:
@@ -163,7 +198,7 @@ class PatriciaTrie:
         if cached is not None:
             return cached
 
-        if storage_node._local_get(h) is None:
+        if storage_node.storage_get(h) is None:
             return None
 
         pat_node = PatriciaNode.from_atoms(storage_node, h)

astreum/_storage/setup.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+
+def storage_setup(node: Any, config: dict) -> None:
+    """Initialize hot/cold storage helpers on the node."""
+
+    node.hot_storage = {}
+    node.hot_storage_hits = {}
+    node.storage_index = {}
+    node.hot_storage_size = 0
+    hot_storage_default_limit = 1 << 30  # 1 GiB
+    hot_storage_limit_value = config.get("hot_storage_limit", hot_storage_default_limit)
+    try:
+        node.hot_storage_limit = int(hot_storage_limit_value)
+    except (TypeError, ValueError):
+        node.hot_storage_limit = hot_storage_default_limit
+
+    node.cold_storage_size = 0
+    cold_storage_default_limit = 10 << 30  # 10 GiB
+    cold_storage_limit_value = config.get("cold_storage_limit", cold_storage_default_limit)
+    try:
+        node.cold_storage_limit = int(cold_storage_limit_value)
+    except (TypeError, ValueError):
+        node.cold_storage_limit = cold_storage_default_limit
+
+    cold_storage_path = config.get("cold_storage_path")
+    if cold_storage_path:
+        try:
+            Path(cold_storage_path).mkdir(parents=True, exist_ok=True)
+        except OSError:
+            cold_storage_path = None
+    node.cold_storage_path = cold_storage_path

astreum/models/block.py

@@ -103,9 +103,9 @@ class Block:
         stake_root = stake_trie.root_hash
 
         # 2. three Account bodies
-        validator_acct = Account.create(balance=0, data=b"", nonce=0)
-        treasury_acct  = Account.create(balance=1, data=stake_root, nonce=0)
-        burn_acct = Account.create(balance=0, data=b"", nonce=0)
+        validator_acct = Account.create(balance=0, data=b"", counter=0)
+        treasury_acct  = Account.create(balance=1, data=stake_root, counter=0)
+        burn_acct = Account.create(balance=0, data=b"", counter=0)
 
         # 3. global Accounts structure
         accts = Accounts()
@@ -190,7 +190,7 @@ class Block:
 
         def _credit(addr: bytes, amt: int):
             acc = blk.accounts.get_account(addr) or Account.create(0, b"", 0)
-            blk.accounts.set_account(addr, Account.create(acc.balance() + amt, acc.data(), acc.nonce()))
+            blk.accounts.set_account(addr, Account.create(acc.balance + amt, acc.data, acc.counter))
 
         if burn_amt:
             _credit(BURN, burn_amt)
@@ -280,17 +280,17 @@ class Block:
 
         sender_acct = self.accounts.get_account(sender_pk)
         if (sender_acct is None
-            or sender_acct.nonce() != nonce
-            or sender_acct.balance() < amount + fee):
+            or sender_acct.counter != nonce
+            or sender_acct.balance < amount + fee):
             raise ValueError("invalid or unaffordable transaction")
 
         # --- debit sender --------------------------------------------------
         self.accounts.set_account(
             sender_pk,
             Account.create(
-                balance=sender_acct.balance() - amount - fee,
-                data=sender_acct.data(),
-                nonce=sender_acct.nonce() + 1,
+                balance=sender_acct.balance - amount - fee,
+                data=sender_acct.data,
+                counter=sender_acct.counter + 1,
             )
         )
 
@@ -298,14 +298,14 @@ class Block:
         if recip_pk == TREASURY:
             treasury = self.accounts.get_account(TREASURY)
 
-            trie = PatriciaTrie(node_get=None, root_hash=treasury.data())
+            trie = PatriciaTrie(node_get=None, root_hash=treasury.data)
             stake_bytes = trie.get(sender_pk) or b""
             current_stake = int.from_bytes(stake_bytes, "big") if stake_bytes else 0
 
             if amount > 0:
                 # stake **deposit**
                 trie.put(sender_pk, (current_stake + amount).to_bytes(32, "big"))
-                new_treas_bal = treasury.balance() + amount
+                new_treas_bal = treasury.balance + amount
             else:
                 # stake **withdrawal**
                 if current_stake == 0:
@@ -315,13 +315,13 @@ class Block:
                 self.accounts.set_account(
                     sender_pk,
                     Account.create(
-                        balance=sender_after.balance() + current_stake,
-                        data=sender_after.data(),
-                        nonce=sender_after.nonce(),
+                        balance=sender_after.balance + current_stake,
+                        data=sender_after.data,
+                        counter=sender_after.counter,
                     )
                 )
                 trie.delete(sender_pk)
-                new_treas_bal = treasury.balance()  # treasury balance unchanged
+                new_treas_bal = treasury.balance  # treasury balance unchanged
 
             # write back treasury with new trie root
             self.accounts.set_account(
@@ -329,7 +329,7 @@ class Block:
                 Account.create(
                     balance=new_treas_bal,
                     data=trie.root_hash,
-                    nonce=treasury.nonce(),
+                    counter=treasury.counter,
                 )
             )
 
@@ -338,9 +338,9 @@ class Block:
             self.accounts.set_account(
                 recip_pk,
                 Account.create(
-                    balance=recip_acct.balance() + amount,
-                    data=recip_acct.data(),
-                    nonce=recip_acct.nonce(),
+                    balance=recip_acct.balance + amount,
+                    data=recip_acct.data,
+                    counter=recip_acct.counter,
                 )
             )
 
@@ -417,7 +417,7 @@ class Block:
             v_acct = dummy.accounts.get_account(self.validator_pk) or Account.create(0,b"",0)
             dummy.accounts.set_account(
                 self.validator_pk,
-                Account.create(v_acct.balance()+rew, v_acct.data(), v_acct.nonce())
+                Account.create(v_acct.balance+rew, v_acct.data, v_acct.counter)
             )
 
         if dummy.accounts.root_hash != self.accounts_hash: