astreum 0.2.41__py3-none-any.whl → 0.3.1__py3-none-any.whl

astreum/storage/__init__.py

@@ -0,0 +1,7 @@
+from .models.atom import Atom
+from .setup import storage_setup
+
+__all__ = [
+    "Atom",
+    "storage_setup",
+]

astreum/storage/actions/get.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+from ..models.atom import Atom
+
+
+def _hot_storage_get(self, key: bytes) -> Optional[Atom]:
+    """Retrieve an atom from in-memory cache while tracking hit statistics."""
+    node_logger = self.logger
+    atom = self.hot_storage.get(key)
+    if atom is not None:
+        self.hot_storage_hits[key] = self.hot_storage_hits.get(key, 0) + 1
+        node_logger.debug("Hot storage hit for %s", key.hex())
+    else:
+        node_logger.debug("Hot storage miss for %s", key.hex())
+    return atom
+
+
+def _network_get(self, key: bytes) -> Optional[Atom]:
+    """Attempt to fetch an atom from network peers when local storage misses."""
+    node_logger = self.logger
+    node_logger.debug("Attempting network fetch for %s", key.hex())
+    # locate storage provider
+    # query storage provider
+    node_logger.warning("Network fetch for %s is not implemented", key.hex())
+    return None
+
+
+def storage_get(self, key: bytes) -> Optional[Atom]:
+    """Retrieve an Atom by checking local storage first, then the network."""
+    node_logger = self.logger
+    node_logger.debug("Fetching atom %s", key.hex())
+    atom = self._hot_storage_get(key)
+    if atom is not None:
+        node_logger.debug("Returning atom %s from hot storage", key.hex())
+        return atom
+    atom = self._cold_storage_get(key)
+    if atom is not None:
+        node_logger.debug("Returning atom %s from cold storage", key.hex())
+        return atom
+    node_logger.debug("Falling back to network fetch for %s", key.hex())
+    return self._network_get(key)
+
+
+def _cold_storage_get(self, key: bytes) -> Optional[Atom]:
+    """Read an atom from the cold storage directory if configured."""
+    node_logger = self.logger
+    if not self.cold_storage_path:
+        node_logger.debug("Cold storage disabled; cannot fetch %s", key.hex())
+        return None
+    filename = f"{key.hex().upper()}.bin"
+    file_path = Path(self.cold_storage_path) / filename
+    try:
+        data = file_path.read_bytes()
+    except FileNotFoundError:
+        node_logger.debug("Cold storage miss for %s", key.hex())
+        return None
+    except OSError as exc:
+        node_logger.warning("Error reading cold storage file %s: %s", file_path, exc)
+        return None
+    try:
+        atom = Atom.from_bytes(data)
+        node_logger.debug("Loaded atom %s from cold storage", key.hex())
+        return atom
+    except ValueError as exc:
+        node_logger.warning("Cold storage data corrupted for %s: %s", file_path, exc)
+        return None

astreum/storage/actions/set.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from ..models.atom import Atom
+
+
+def _hot_storage_set(self, key: bytes, value: Atom) -> bool:
+    """Store atom in hot storage without exceeding the configured limit."""
+    node_logger = self.logger
+    projected = self.hot_storage_size + value.size
+    if projected > self.hot_storage_limit:
+        node_logger.warning(
+            "Hot storage limit reached (%s > %s); skipping atom %s",
+            projected,
+            self.hot_storage_limit,
+            key.hex(),
+        )
+        return False
+
+    self.hot_storage[key] = value
+    self.hot_storage_size = projected
+    node_logger.debug(
+        "Stored atom %s in hot storage (bytes=%s, total=%s)",
+        key.hex(),
+        value.size,
+        projected,
+    )
+    return True
+
+
+def _cold_storage_set(self, atom: Atom) -> None:
+    """Persist an atom into the cold storage directory if it already exists."""
+    node_logger = self.logger
+    atom_id = atom.object_id()
+    atom_hex = atom_id.hex()
+    if not self.cold_storage_path:
+        node_logger.debug("Cold storage disabled; skipping atom %s", atom_hex)
+        return
+    atom_bytes = atom.to_bytes()
+    projected = self.cold_storage_size + len(atom_bytes)
+    if self.cold_storage_limit and projected > self.cold_storage_limit:
+        node_logger.warning(
+            "Cold storage limit reached (%s > %s); skipping atom %s",
+            projected,
+            self.cold_storage_limit,
+            atom_hex,
+        )
+        return
+    directory = Path(self.cold_storage_path)
+    if not directory.exists():
+        node_logger.warning(
+            "Cold storage path %s missing; skipping atom %s",
+            directory,
+            atom_hex,
+        )
+        return
+    filename = f"{atom_hex.upper()}.bin"
+    file_path = directory / filename
+    try:
+        file_path.write_bytes(atom_bytes)
+        self.cold_storage_size = projected
+        node_logger.debug("Persisted atom %s to cold storage", atom_hex)
+    except OSError as exc:
+        node_logger.error(
+            "Failed writing atom %s to cold storage %s: %s",
+            atom_hex,
+            file_path,
+            exc,
+        )
+
+
+def _network_set(self, atom: Atom) -> None:
+    """Advertise an atom to the closest known peer so they can fetch it from us."""
+    node_logger = self.logger
+    atom_id = atom.object_id()
+    atom_hex = atom_id.hex()
+    try:
+        from ...communication.models.message import Message, MessageTopic
+    except Exception as exc:
+        node_logger.warning(
+            "Communication module unavailable; cannot advertise atom %s: %s",
+            atom_hex,
+            exc,
+        )
+        return
+
+    try:
+        closest_peer = self.peer_route.closest_peer_for_hash(atom_id)
+    except Exception as exc:
+        node_logger.warning("Peer lookup failed for atom %s: %s", atom_hex, exc)
+        return
+    if closest_peer is None or closest_peer.address is None:
+        node_logger.debug("No peer available to advertise atom %s", atom_hex)
+        return
+    target_addr = closest_peer.address
+
+    try:
+        provider_ip, provider_port = self.incoming_socket.getsockname()[:2]
+    except Exception as exc:
+        node_logger.warning(
+            "Unable to determine provider address for atom %s: %s",
+            atom_hex,
+            exc,
+        )
+        return
+
+    provider_str = f"{provider_ip}:{int(provider_port)}"
+    try:
+        provider_bytes = provider_str.encode("utf-8")
+    except Exception as exc:
+        node_logger.warning("Unable to encode provider info for %s: %s", atom_hex, exc)
+        return
+
+    payload = atom_id + provider_bytes
+    message = Message(topic=MessageTopic.STORAGE_REQUEST, content=payload)
+    try:
+        self.outgoing_queue.put((message.to_bytes(), target_addr))
+        node_logger.debug(
+            "Advertised atom %s to peer at %s:%s",
+            atom_hex,
+            target_addr[0],
+            target_addr[1],
+        )
+    except Exception as exc:
+        node_logger.error(
+            "Failed to queue advertisement for atom %s to %s:%s: %s",
+            atom_hex,
+            target_addr[0],
+            target_addr[1],
+            exc,
+        )

astreum/storage/models/atom.py

@@ -0,0 +1,107 @@
+
+
+from enum import IntEnum
+from typing import List, Optional, Tuple
+
+from blake3 import blake3
+
+ZERO32 = b"\x00"*32
+
+def u64_le(n: int) -> bytes:
+    return int(n).to_bytes(8, "little", signed=False)
+
+def hash_bytes(b: bytes) -> bytes:
+    return blake3(b).digest()
+
+class AtomKind(IntEnum):
+    SYMBOL = 0
+    BYTES = 1
+    LIST = 2
+
+
+class Atom:
+    data: bytes
+    kind: AtomKind
+    next_id: bytes
+    size: int
+    
+    def __init__(self, data: bytes, kind: AtomKind, next_id: bytes = ZERO32):
+        self.data = data
+        self.kind = kind
+        self.next_id = next_id
+        self.size = len(data)
+        
+
+    def generate_id(self) -> bytes:
+        """Compute the object id using this atom's metadata."""
+        kind_bytes = int(self.kind).to_bytes(1, "little", signed=False)
+        return blake3(
+            kind_bytes + self.data_hash() + self.next_id + u64_le(self.size)
+        ).digest()
+
+    def data_hash(self) -> bytes:
+        return hash_bytes(self.data)
+
+    def object_id(self) -> bytes:
+        return self.generate_id()
+
+    @staticmethod
+    def verify_metadata(
+        object_id: bytes,
+        size: int,
+        next_hash: bytes,
+        data_hash: bytes,
+        kind: AtomKind,
+    ) -> bool:
+        kind_bytes = int(kind).to_bytes(1, "little", signed=False)
+        expected = blake3(kind_bytes + data_hash + next_hash + u64_le(size)).digest()
+        return object_id == expected
+
+    def to_bytes(self) -> bytes:
+        """Serialize as next-hash + kind byte + payload."""
+        kind_byte = int(self.kind).to_bytes(1, "little", signed=False)
+        return self.next_id + kind_byte + self.data
+
+    @staticmethod
+    def from_bytes(buf: bytes) -> "Atom":
+        header_len = len(ZERO32)
+        if len(buf) < header_len + 1:
+            raise ValueError("buffer too short for Atom header")
+        next_hash = buf[:header_len]
+        kind_value = buf[header_len]
+        data = buf[header_len + 1 :]
+        try:
+            kind = AtomKind(kind_value)
+        except ValueError as exc:
+            raise ValueError(f"unknown atom kind: {kind_value}") from exc
+        return Atom(data=data, next_id=next_hash, kind=kind)
+
+def bytes_list_to_atoms(values: List[bytes]) -> Tuple[bytes, List[Atom]]:
+    """Build a forward-ordered linked list of atoms from byte payloads.
+
+    Returns the head object's hash (ZERO32 if no values) and the atoms created.
+    """
+    next_hash = ZERO32
+    atoms: List[Atom] = []
+
+    for value in reversed(values):
+        atom = Atom(data=bytes(value), next_id=next_hash, kind=AtomKind.BYTES)
+        atoms.append(atom)
+        next_hash = atom.object_id()
+
+    atoms.reverse()
+    return (next_hash if values else ZERO32), atoms
+
+
+def get_atom_list_from_storage(self, root_hash: bytes) -> Optional[List["Atom"]]:
+    """Follow the list chain starting at root_hash, returning atoms or None on gaps."""
+    next_id: bytes = root_hash
+    atom_list: List["Atom"] = []
+    while next_id != ZERO32:
+        elem = self.storage_get(key=next_id)
+        if elem:
+            atom_list.append(elem)
+            next_id = elem.next_id
+        else:
+            return None
+    return atom_list