astreum 0.2.61__py3-none-any.whl → 0.3.1__py3-none-any.whl

astreum/storage/__init__.py

@@ -0,0 +1,7 @@
+from .models.atom import Atom
+from .setup import storage_setup
+
+__all__ = [
+    "Atom",
+    "storage_setup",
+]

astreum/storage/actions/get.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+from ..models.atom import Atom
+
+
+def _hot_storage_get(self, key: bytes) -> Optional[Atom]:
+    """Retrieve an atom from in-memory cache while tracking hit statistics."""
+    node_logger = self.logger
+    atom = self.hot_storage.get(key)
+    if atom is not None:
+        self.hot_storage_hits[key] = self.hot_storage_hits.get(key, 0) + 1
+        node_logger.debug("Hot storage hit for %s", key.hex())
+    else:
+        node_logger.debug("Hot storage miss for %s", key.hex())
+    return atom
+
+
+def _network_get(self, key: bytes) -> Optional[Atom]:
+    """Attempt to fetch an atom from network peers when local storage misses."""
+    node_logger = self.logger
+    node_logger.debug("Attempting network fetch for %s", key.hex())
+    # locate storage provider
+    # query storage provider
+    node_logger.warning("Network fetch for %s is not implemented", key.hex())
+    return None
+
+
+def storage_get(self, key: bytes) -> Optional[Atom]:
+    """Retrieve an Atom by checking local storage first, then the network."""
+    node_logger = self.logger
+    node_logger.debug("Fetching atom %s", key.hex())
+    atom = self._hot_storage_get(key)
+    if atom is not None:
+        node_logger.debug("Returning atom %s from hot storage", key.hex())
+        return atom
+    atom = self._cold_storage_get(key)
+    if atom is not None:
+        node_logger.debug("Returning atom %s from cold storage", key.hex())
+        return atom
+    node_logger.debug("Falling back to network fetch for %s", key.hex())
+    return self._network_get(key)
+
+
+def _cold_storage_get(self, key: bytes) -> Optional[Atom]:
+    """Read an atom from the cold storage directory if configured."""
+    node_logger = self.logger
+    if not self.cold_storage_path:
+        node_logger.debug("Cold storage disabled; cannot fetch %s", key.hex())
+        return None
+    filename = f"{key.hex().upper()}.bin"
+    file_path = Path(self.cold_storage_path) / filename
+    try:
+        data = file_path.read_bytes()
+    except FileNotFoundError:
+        node_logger.debug("Cold storage miss for %s", key.hex())
+        return None
+    except OSError as exc:
+        node_logger.warning("Error reading cold storage file %s: %s", file_path, exc)
+        return None
+    try:
+        atom = Atom.from_bytes(data)
+        node_logger.debug("Loaded atom %s from cold storage", key.hex())
+        return atom
+    except ValueError as exc:
+        node_logger.warning("Cold storage data corrupted for %s: %s", file_path, exc)
+        return None

astreum/storage/actions/set.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from ..models.atom import Atom
+
+
+def _hot_storage_set(self, key: bytes, value: Atom) -> bool:
+    """Store atom in hot storage without exceeding the configured limit."""
+    node_logger = self.logger
+    projected = self.hot_storage_size + value.size
+    if projected > self.hot_storage_limit:
+        node_logger.warning(
+            "Hot storage limit reached (%s > %s); skipping atom %s",
+            projected,
+            self.hot_storage_limit,
+            key.hex(),
+        )
+        return False
+
+    self.hot_storage[key] = value
+    self.hot_storage_size = projected
+    node_logger.debug(
+        "Stored atom %s in hot storage (bytes=%s, total=%s)",
+        key.hex(),
+        value.size,
+        projected,
+    )
+    return True
+
+
+def _cold_storage_set(self, atom: Atom) -> None:
+    """Persist an atom into the cold storage directory if it already exists."""
+    node_logger = self.logger
+    atom_id = atom.object_id()
+    atom_hex = atom_id.hex()
+    if not self.cold_storage_path:
+        node_logger.debug("Cold storage disabled; skipping atom %s", atom_hex)
+        return
+    atom_bytes = atom.to_bytes()
+    projected = self.cold_storage_size + len(atom_bytes)
+    if self.cold_storage_limit and projected > self.cold_storage_limit:
+        node_logger.warning(
+            "Cold storage limit reached (%s > %s); skipping atom %s",
+            projected,
+            self.cold_storage_limit,
+            atom_hex,
+        )
+        return
+    directory = Path(self.cold_storage_path)
+    if not directory.exists():
+        node_logger.warning(
+            "Cold storage path %s missing; skipping atom %s",
+            directory,
+            atom_hex,
+        )
+        return
+    filename = f"{atom_hex.upper()}.bin"
+    file_path = directory / filename
+    try:
+        file_path.write_bytes(atom_bytes)
+        self.cold_storage_size = projected
+        node_logger.debug("Persisted atom %s to cold storage", atom_hex)
+    except OSError as exc:
+        node_logger.error(
+            "Failed writing atom %s to cold storage %s: %s",
+            atom_hex,
+            file_path,
+            exc,
+        )
+
+
+def _network_set(self, atom: Atom) -> None:
+    """Advertise an atom to the closest known peer so they can fetch it from us."""
+    node_logger = self.logger
+    atom_id = atom.object_id()
+    atom_hex = atom_id.hex()
+    try:
+        from ...communication.models.message import Message, MessageTopic
+    except Exception as exc:
+        node_logger.warning(
+            "Communication module unavailable; cannot advertise atom %s: %s",
+            atom_hex,
+            exc,
+        )
+        return
+
+    try:
+        closest_peer = self.peer_route.closest_peer_for_hash(atom_id)
+    except Exception as exc:
+        node_logger.warning("Peer lookup failed for atom %s: %s", atom_hex, exc)
+        return
+    if closest_peer is None or closest_peer.address is None:
+        node_logger.debug("No peer available to advertise atom %s", atom_hex)
+        return
+    target_addr = closest_peer.address
+
+    try:
+        provider_ip, provider_port = self.incoming_socket.getsockname()[:2]
+    except Exception as exc:
+        node_logger.warning(
+            "Unable to determine provider address for atom %s: %s",
+            atom_hex,
+            exc,
+        )
+        return
+
+    provider_str = f"{provider_ip}:{int(provider_port)}"
+    try:
+        provider_bytes = provider_str.encode("utf-8")
+    except Exception as exc:
+        node_logger.warning("Unable to encode provider info for %s: %s", atom_hex, exc)
+        return
+
+    payload = atom_id + provider_bytes
+    message = Message(topic=MessageTopic.STORAGE_REQUEST, content=payload)
+    try:
+        self.outgoing_queue.put((message.to_bytes(), target_addr))
+        node_logger.debug(
+            "Advertised atom %s to peer at %s:%s",
+            atom_hex,
+            target_addr[0],
+            target_addr[1],
+        )
+    except Exception as exc:
+        node_logger.error(
+            "Failed to queue advertisement for atom %s to %s:%s: %s",
+            atom_hex,
+            target_addr[0],
+            target_addr[1],
+            exc,
+        )

astreum/{_storage → storage/models}/atom.py

@@ -1,18 +1,18 @@
-
-
+
+
 from enum import IntEnum
 from typing import List, Optional, Tuple
-
+
 from blake3 import blake3
-
-ZERO32 = b"\x00"*32
-
-def u64_le(n: int) -> bytes:
-    return int(n).to_bytes(8, "little", signed=False)
-
-def hash_bytes(b: bytes) -> bytes:
-    return blake3(b).digest()
-
+
+ZERO32 = b"\x00"*32
+
+def u64_le(n: int) -> bytes:
+    return int(n).to_bytes(8, "little", signed=False)
+
+def hash_bytes(b: bytes) -> bytes:
+    return blake3(b).digest()
+
 class AtomKind(IntEnum):
     SYMBOL = 0
     BYTES = 1
@@ -20,35 +20,23 @@ class AtomKind(IntEnum):
 
 
 class Atom:
-    data: bytes
-    next: bytes
-    size: int
-
-    def __init__(
-        self,
-        data: bytes,
-        next: bytes = ZERO32,
-        size: Optional[int] = None,
-        kind: AtomKind = AtomKind.BYTES,
-    ):
+    data: bytes
+    kind: AtomKind
+    next_id: bytes
+    size: int
+    
+    def __init__(self, data: bytes, kind: AtomKind, next_id: bytes = ZERO32):
         self.data = data
-        self.next = next
-        self.size = len(data) if size is None else size
         self.kind = kind
+        self.next_id = next_id
+        self.size = len(data)
+        
 
-    @staticmethod
-    def from_data(
-        data: bytes,
-        next_hash: bytes = ZERO32,
-        kind: AtomKind = AtomKind.BYTES,
-    ) -> "Atom":
-        return Atom(data=data, next=next_hash, size=len(data), kind=kind)
-
     def generate_id(self) -> bytes:
         """Compute the object id using this atom's metadata."""
         kind_bytes = int(self.kind).to_bytes(1, "little", signed=False)
         return blake3(
-            kind_bytes + self.data_hash() + self.next + u64_le(self.size)
+            kind_bytes + self.data_hash() + self.next_id + u64_le(self.size)
         ).digest()
 
     def data_hash(self) -> bytes:
@@ -68,11 +56,11 @@ class Atom:
         kind_bytes = int(kind).to_bytes(1, "little", signed=False)
         expected = blake3(kind_bytes + data_hash + next_hash + u64_le(size)).digest()
         return object_id == expected
-
+
     def to_bytes(self) -> bytes:
         """Serialize as next-hash + kind byte + payload."""
         kind_byte = int(self.kind).to_bytes(1, "little", signed=False)
-        return self.next + kind_byte + self.data
+        return self.next_id + kind_byte + self.data
 
     @staticmethod
     def from_bytes(buf: bytes) -> "Atom":
@@ -86,24 +74,34 @@ class Atom:
             kind = AtomKind(kind_value)
         except ValueError as exc:
             raise ValueError(f"unknown atom kind: {kind_value}") from exc
-        return Atom(data=data, next=next_hash, size=len(data), kind=kind)
-
-def bytes_list_to_atoms(values: List[bytes]) -> Tuple[bytes, List[Atom]]:
-    """Build a forward-ordered linked list of atoms from byte payloads.
-
-    Returns the head object's hash (ZERO32 if no values) and the atoms created.
-    """
-    next_hash = ZERO32
-    atoms: List[Atom] = []
-
-    for value in reversed(values):
-        atom = Atom.from_data(
-            data=bytes(value),
-            next_hash=next_hash,
-            kind=AtomKind.BYTES,
-        )
-        atoms.append(atom)
-        next_hash = atom.object_id()
-
-    atoms.reverse()
-    return (next_hash if values else ZERO32), atoms
+        return Atom(data=data, next_id=next_hash, kind=kind)
+
+def bytes_list_to_atoms(values: List[bytes]) -> Tuple[bytes, List[Atom]]:
+    """Build a forward-ordered linked list of atoms from byte payloads.
+
+    Returns the head object's hash (ZERO32 if no values) and the atoms created.
+    """
+    next_hash = ZERO32
+    atoms: List[Atom] = []
+
+    for value in reversed(values):
+        atom = Atom(data=bytes(value), next_id=next_hash, kind=AtomKind.BYTES)
+        atoms.append(atom)
+        next_hash = atom.object_id()
+
+    atoms.reverse()
+    return (next_hash if values else ZERO32), atoms
+
+
+def get_atom_list_from_storage(self, root_hash: bytes) -> Optional[List["Atom"]]:
+    """Follow the list chain starting at root_hash, returning atoms or None on gaps."""
+    next_id: bytes = root_hash
+    atom_list: List["Atom"] = []
+    while next_id != ZERO32:
+        elem = self.storage_get(key=next_id)
+        if elem:
+            atom_list.append(elem)
+            next_id = elem.next_id
+        else:
+            return None
+    return atom_list