astreum 0.2.61__py3-none-any.whl → 0.3.9__py3-none-any.whl

astreum/storage/__init__.py

@@ -0,0 +1,7 @@
+from .models.atom import Atom
+from .setup import storage_setup
+
+__all__ = [
+    "Atom",
+    "storage_setup",
+]

astreum/storage/actions/get.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+from ..models.atom import Atom
+
+
+def _hot_storage_get(self, key: bytes) -> Optional[Atom]:
+    """Retrieve an atom from in-memory cache while tracking hit statistics."""
+    atom = self.hot_storage.get(key)
+    if atom is not None:
+        self.hot_storage_hits[key] = self.hot_storage_hits.get(key, 0) + 1
+        self.logger.debug("Hot storage hit for %s", key.hex())
+    else:
+        self.logger.debug("Hot storage miss for %s", key.hex())
+    return atom
+
+
+def _network_get(self, key: bytes) -> Optional[Atom]:
+    """Attempt to fetch an atom from network peers when local storage misses."""
+    if not getattr(self, "is_connected", False):
+        self.logger.debug("Network fetch skipped for %s; node not connected", key.hex())
+        return None
+    self.logger.debug("Attempting network fetch for %s", key.hex())
+    try:
+        from ...communication.handlers.object_request import (
+            ObjectRequest,
+            ObjectRequestType,
+        )
+        from ...communication.models.message import Message, MessageTopic
+    except Exception as exc:
+        self.logger.warning(
+            "Communication module unavailable; cannot fetch %s: %s",
+            key.hex(),
+            exc,
+        )
+        return None
+
+    try:
+        closest_peer = self.peer_route.closest_peer_for_hash(key)
+    except Exception as exc:
+        self.logger.warning("Peer lookup failed for %s: %s", key.hex(), exc)
+        return None
+
+    if closest_peer is None or closest_peer.address is None:
+        self.logger.debug("No peer available to fetch %s", key.hex())
+        return None
+
+    obj_req = ObjectRequest(
+        type=ObjectRequestType.OBJECT_GET,
+        data=b"",
+        atom_id=key,
+    )
+    try:
+        message = Message(
+            topic=MessageTopic.OBJECT_REQUEST,
+            content=obj_req.to_bytes(),
+            sender=self.relay_public_key,
+        )
+    except Exception as exc:
+        self.logger.warning("Failed to build object request for %s: %s", key.hex(), exc)
+        return None
+
+    # encrypt the outbound request for the target peer
+    message.encrypt(closest_peer.shared_key_bytes)
+
+    try:
+        self.add_atom_req(key)
+    except Exception as exc:
+        self.logger.warning("Failed to track object request for %s: %s", key.hex(), exc)
+
+    try:
+        self.outgoing_queue.put((message.to_bytes(), closest_peer.address))
+        self.logger.debug(
+            "Queued OBJECT_GET for %s to peer %s",
+            key.hex(),
+            closest_peer.address,
+        )
+    except Exception as exc:
+        self.logger.warning(
+            "Failed to queue OBJECT_GET for %s to %s: %s",
+            key.hex(),
+            closest_peer.address,
+            exc,
+        )
+    return None
+
+
+def storage_get(self, key: bytes) -> Optional[Atom]:
+    """Retrieve an Atom by checking local storage first, then the network."""
+    self.logger.debug("Fetching atom %s", key.hex())
+    atom = self._hot_storage_get(key)
+    if atom is not None:
+        self.logger.debug("Returning atom %s from hot storage", key.hex())
+        return atom
+    atom = self._cold_storage_get(key)
+    if atom is not None:
+        self.logger.debug("Returning atom %s from cold storage", key.hex())
+        return atom
+    
+    if not self.is_connected:
+        return None
+    
+    provider_payload = self.storage_index.get(key)
+    if provider_payload is not None:
+        try:
+            from ...communication.handlers.object_response import decode_object_provider
+            from ...communication.handlers.object_request import (
+                ObjectRequest,
+                ObjectRequestType,
+            )
+            from ...communication.models.message import Message, MessageTopic
+            from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PublicKey
+
+            provider_key, provider_address, provider_port = decode_object_provider(provider_payload)
+            provider_public_key = X25519PublicKey.from_public_bytes(provider_key)
+            shared_key_bytes = self.relay_secret_key.exchange(provider_public_key)
+
+            obj_req = ObjectRequest(
+                type=ObjectRequestType.OBJECT_GET,
+                data=b"",
+                atom_id=key,
+            )
+            message = Message(
+                topic=MessageTopic.OBJECT_REQUEST,
+                content=obj_req.to_bytes(),
+                sender=self.relay_public_key,
+            )
+            message.encrypt(shared_key_bytes)
+            self.add_atom_req(key)
+            self.outgoing_queue.put((message.to_bytes(), (provider_address, provider_port)))
+            self.logger.debug(
+                "Requested atom %s from indexed provider %s:%s",
+                key.hex(),
+                provider_address,
+                provider_port,
+            )
+        except Exception as exc:
+            self.logger.warning("Failed indexed fetch for %s: %s", key.hex(), exc)
+        return None
+
+    self.logger.debug("Falling back to network fetch for %s", key.hex())
+    return self._network_get(key)
+
+
+def local_get(self, key: bytes) -> Optional[Atom]:
+    """Retrieve an Atom by checking only local hot and cold storage."""
+    self.logger.debug("Fetching atom %s (local only)", key.hex())
+    atom = self._hot_storage_get(key)
+    if atom is not None:
+        self.logger.debug("Returning atom %s from hot storage", key.hex())
+        return atom
+    atom = self._cold_storage_get(key)
+    if atom is not None:
+        self.logger.debug("Returning atom %s from cold storage", key.hex())
+        return atom
+    self.logger.debug("Local storage miss for %s", key.hex())
+    return None
+
+
+def _cold_storage_get(self, key: bytes) -> Optional[Atom]:
+    """Read an atom from the cold storage directory if configured."""
+    if not self.config["cold_storage_path"]:
+        self.logger.debug("Cold storage disabled; cannot fetch %s", key.hex())
+        return None
+    filename = f"{key.hex().upper()}.bin"
+    file_path = Path(self.config["cold_storage_path"]) / filename
+    try:
+        data = file_path.read_bytes()
+    except FileNotFoundError:
+        self.logger.debug("Cold storage miss for %s", key.hex())
+        return None
+    except OSError as exc:
+        self.logger.warning("Error reading cold storage file %s: %s", file_path, exc)
+        return None
+    try:
+        atom = Atom.from_bytes(data)
+        self.logger.debug("Loaded atom %s from cold storage", key.hex())
+        return atom
+    except ValueError as exc:
+        self.logger.warning("Cold storage data corrupted for %s: %s", file_path, exc)
+        return None

astreum/storage/actions/set.py

@@ -0,0 +1,178 @@
+from __future__ import annotations
+
+import socket
+from pathlib import Path
+
+from cryptography.hazmat.primitives import serialization
+
+from ..models.atom import Atom
+
+
+def _hot_storage_set(self, key: bytes, value: Atom) -> bool:
+    """Store atom in hot storage without exceeding the configured limit."""
+    node_logger = self.logger
+    projected = self.hot_storage_size + value.size
+    hot_limit = self.config["hot_storage_default_limit"]
+    if projected > hot_limit:
+        node_logger.warning(
+            "Hot storage limit reached (%s > %s); skipping atom %s",
+            projected,
+            hot_limit,
+            key.hex(),
+        )
+        return False
+
+    self.hot_storage[key] = value
+    self.hot_storage_size = projected
+    node_logger.debug(
+        "Stored atom %s in hot storage (bytes=%s, total=%s)",
+        key.hex(),
+        value.size,
+        projected,
+    )
+    return True
+
+
+def _cold_storage_set(self, atom: Atom) -> None:
+    """Persist an atom into the cold storage directory if it already exists."""
+    node_logger = self.logger
+    atom_id = atom.object_id()
+    atom_hex = atom_id.hex()
+    if not self.config["cold_storage_path"]:
+        node_logger.debug("Cold storage disabled; skipping atom %s", atom_hex)
+        return
+    atom_bytes = atom.to_bytes()
+    projected = self.cold_storage_size + len(atom_bytes)
+    cold_limit = self.config["cold_storage_limit"]
+    if cold_limit and projected > cold_limit:
+        node_logger.warning(
+            "Cold storage limit reached (%s > %s); skipping atom %s",
+            projected,
+            cold_limit,
+            atom_hex,
+        )
+        return
+    directory = Path(self.config["cold_storage_path"])
+    if not directory.exists():
+        node_logger.warning(
+            "Cold storage path %s missing; skipping atom %s",
+            directory,
+            atom_hex,
+        )
+        return
+    filename = f"{atom_hex.upper()}.bin"
+    file_path = directory / filename
+    try:
+        file_path.write_bytes(atom_bytes)
+        self.cold_storage_size = projected
+        node_logger.debug("Persisted atom %s to cold storage", atom_hex)
+    except OSError as exc:
+        node_logger.error(
+            "Failed writing atom %s to cold storage %s: %s",
+            atom_hex,
+            file_path,
+            exc,
+        )
+
+
+def _network_set(self, atom: Atom) -> None:
+    """Advertise an atom to the closest known peer so they can fetch it from us."""
+    node_logger = self.logger
+    atom_id = atom.object_id()
+    atom_hex = atom_id.hex()
+    try:
+        from ...communication.handlers.object_request import (
+            ObjectRequest,
+            ObjectRequestType,
+        )
+        from ...communication.models.message import Message, MessageTopic
+    except Exception as exc:
+        node_logger.warning(
+            "Communication module unavailable; cannot advertise atom %s: %s",
+            atom_hex,
+            exc,
+        )
+        return
+
+    try:
+        provider_ip, provider_port = self.incoming_socket.getsockname()[:2]
+    except Exception as exc:
+        node_logger.warning(
+            "Unable to determine provider address for atom %s: %s",
+            atom_hex,
+            exc,
+        )
+        return
+
+    try:
+        provider_ip_bytes = socket.inet_aton(provider_ip)
+        provider_port_bytes = int(provider_port).to_bytes(2, "big", signed=False)
+        provider_key_bytes = self.relay_public_key_bytes
+    except Exception as exc:
+        node_logger.warning("Unable to encode provider info for %s: %s", atom_hex, exc)
+        return
+
+    provider_payload = provider_key_bytes + provider_ip_bytes + provider_port_bytes
+
+    try:
+        closest_peer = self.peer_route.closest_peer_for_hash(atom_id)
+    except Exception as exc:
+        node_logger.warning("Peer lookup failed for atom %s: %s", atom_hex, exc)
+        return
+
+    is_self_closest = False
+    if closest_peer is None or closest_peer.address is None:
+        is_self_closest = True
+    else:
+        try:
+            from ...communication.util import xor_distance
+        except Exception as exc:
+            node_logger.warning("Failed to import xor_distance for atom %s: %s", atom_hex, exc)
+            is_self_closest = True
+        else:
+            try:
+                self_distance = xor_distance(atom_id, self.relay_public_key_bytes)
+                peer_distance = xor_distance(atom_id, closest_peer.public_key_bytes)
+            except Exception as exc:
+                node_logger.warning("Failed computing distance for atom %s: %s", atom_hex, exc)
+                is_self_closest = True
+            else:
+                is_self_closest = self_distance <= peer_distance
+
+    if is_self_closest:
+        node_logger.debug("Self is closest; indexing provider for atom %s", atom_hex)
+        self.storage_index[atom_id] = provider_payload
+        return
+
+    target_addr = closest_peer.address
+
+    obj_req = ObjectRequest(
+        type=ObjectRequestType.OBJECT_PUT,
+        data=provider_payload,
+        atom_id=atom_id,
+    )
+    
+    message_body = obj_req.to_bytes()
+
+    message = Message(
+        topic=MessageTopic.OBJECT_REQUEST,
+        content=message_body,
+        sender=self.relay_public_key,
+    )
+    message.encrypt(closest_peer.shared_key_bytes)
+    try:
+        self.outgoing_queue.put((message.to_bytes(), target_addr))
+        node_logger.debug(
+            "Advertised atom %s to peer at %s:%s",
+            atom_hex,
+            target_addr[0],
+            target_addr[1],
+        )
+    except Exception as exc:
+        node_logger.error(
+            "Failed to queue advertisement for atom %s to %s:%s: %s",
+            atom_hex,
+            target_addr[0],
+            target_addr[1],
+            exc,
+        )

astreum/{_storage → storage/models}/atom.py

@@ -1,18 +1,18 @@
-
-
+
+
 from enum import IntEnum
 from typing import List, Optional, Tuple
-
+
 from blake3 import blake3
-
-ZERO32 = b"\x00"*32
-
-def u64_le(n: int) -> bytes:
-    return int(n).to_bytes(8, "little", signed=False)
-
-def hash_bytes(b: bytes) -> bytes:
-    return blake3(b).digest()
-
+
+ZERO32 = b"\x00"*32
+
+def u64_le(n: int) -> bytes:
+    return int(n).to_bytes(8, "little", signed=False)
+
+def hash_bytes(b: bytes) -> bytes:
+    return blake3(b).digest()
+
 class AtomKind(IntEnum):
     SYMBOL = 0
     BYTES = 1
@@ -20,35 +20,23 @@ class AtomKind(IntEnum):
 
 
 class Atom:
-    data: bytes
-    next: bytes
-    size: int
-
-    def __init__(
-        self,
-        data: bytes,
-        next: bytes = ZERO32,
-        size: Optional[int] = None,
-        kind: AtomKind = AtomKind.BYTES,
-    ):
+    data: bytes
+    kind: AtomKind
+    next_id: bytes
+    size: int
+    
+    def __init__(self, data: bytes, kind: AtomKind, next_id: bytes = ZERO32):
         self.data = data
-        self.next = next
-        self.size = len(data) if size is None else size
         self.kind = kind
+        self.next_id = next_id
+        self.size = len(data)
+        
 
-    @staticmethod
-    def from_data(
-        data: bytes,
-        next_hash: bytes = ZERO32,
-        kind: AtomKind = AtomKind.BYTES,
-    ) -> "Atom":
-        return Atom(data=data, next=next_hash, size=len(data), kind=kind)
-
     def generate_id(self) -> bytes:
         """Compute the object id using this atom's metadata."""
         kind_bytes = int(self.kind).to_bytes(1, "little", signed=False)
         return blake3(
-            kind_bytes + self.data_hash() + self.next + u64_le(self.size)
+            kind_bytes + self.data_hash() + self.next_id + u64_le(self.size)
         ).digest()
 
     def data_hash(self) -> bytes:
@@ -68,11 +56,11 @@ class Atom:
         kind_bytes = int(kind).to_bytes(1, "little", signed=False)
         expected = blake3(kind_bytes + data_hash + next_hash + u64_le(size)).digest()
         return object_id == expected
-
+
     def to_bytes(self) -> bytes:
         """Serialize as next-hash + kind byte + payload."""
         kind_byte = int(self.kind).to_bytes(1, "little", signed=False)
-        return self.next + kind_byte + self.data
+        return self.next_id + kind_byte + self.data
 
     @staticmethod
     def from_bytes(buf: bytes) -> "Atom":
@@ -86,24 +74,34 @@ class Atom:
             kind = AtomKind(kind_value)
         except ValueError as exc:
             raise ValueError(f"unknown atom kind: {kind_value}") from exc
-        return Atom(data=data, next=next_hash, size=len(data), kind=kind)
-
-def bytes_list_to_atoms(values: List[bytes]) -> Tuple[bytes, List[Atom]]:
-    """Build a forward-ordered linked list of atoms from byte payloads.
-
-    Returns the head object's hash (ZERO32 if no values) and the atoms created.
-    """
-    next_hash = ZERO32
-    atoms: List[Atom] = []
-
-    for value in reversed(values):
-        atom = Atom.from_data(
-            data=bytes(value),
-            next_hash=next_hash,
-            kind=AtomKind.BYTES,
-        )
-        atoms.append(atom)
-        next_hash = atom.object_id()
-
-    atoms.reverse()
-    return (next_hash if values else ZERO32), atoms
+        return Atom(data=data, next_id=next_hash, kind=kind)
+
+def bytes_list_to_atoms(values: List[bytes]) -> Tuple[bytes, List[Atom]]:
+    """Build a forward-ordered linked list of atoms from byte payloads.
+
+    Returns the head object's hash (ZERO32 if no values) and the atoms created.
+    """
+    next_hash = ZERO32
+    atoms: List[Atom] = []
+
+    for value in reversed(values):
+        atom = Atom(data=bytes(value), next_id=next_hash, kind=AtomKind.BYTES)
+        atoms.append(atom)
+        next_hash = atom.object_id()
+
+    atoms.reverse()
+    return (next_hash if values else ZERO32), atoms
+
+
+def get_atom_list_from_storage(self, root_hash: bytes) -> Optional[List["Atom"]]:
+    """Follow the list chain starting at root_hash, returning atoms or None on gaps."""
+    next_id: bytes = root_hash
+    atom_list: List["Atom"] = []
+    while next_id != ZERO32:
+        elem = self.storage_get(key=next_id)
+        if elem:
+            atom_list.append(elem)
+            next_id = elem.next_id
+        else:
+            return None
+    return atom_list