astreum 0.3.16__py3-none-any.whl → 0.3.48__py3-none-any.whl

astreum/storage/actions/set.py

@@ -1,159 +1,160 @@
-from __future__ import annotations
-
-import socket
-from pathlib import Path
-
-from cryptography.hazmat.primitives import serialization
-
-from ..models.atom import Atom
-
-
-def _hot_storage_set(self, key: bytes, value: Atom) -> bool:
-    """Store atom in hot storage without exceeding the configured limit."""
-    node_logger = self.logger
-    projected = self.hot_storage_size + value.size
-    hot_limit = self.config["hot_storage_default_limit"]
-    if projected > hot_limit:
-        node_logger.warning(
-            "Hot storage limit reached (%s > %s); skipping atom %s",
-            projected,
-            hot_limit,
-            key.hex(),
-        )
-        return False
-
-    self.hot_storage[key] = value
-    self.hot_storage_size = projected
-    node_logger.debug(
-        "Stored atom %s in hot storage (bytes=%s, total=%s)",
-        key.hex(),
-        value.size,
-        projected,
-    )
-    return True
-
-
-def _cold_storage_set(self, atom: Atom) -> None:
-    """Persist an atom into the cold storage directory if it already exists."""
-    node_logger = self.logger
-    atom_id = atom.object_id()
-    atom_hex = atom_id.hex()
-    if not self.config["cold_storage_path"]:
-        node_logger.debug("Cold storage disabled; skipping atom %s", atom_hex)
-        return
-    atom_bytes = atom.to_bytes()
-    projected = self.cold_storage_size + len(atom_bytes)
-    cold_limit = self.config["cold_storage_limit"]
-    if cold_limit and projected > cold_limit:
-        node_logger.warning(
-            "Cold storage limit reached (%s > %s); skipping atom %s",
-            projected,
-            cold_limit,
-            atom_hex,
-        )
-        return
-    directory = Path(self.config["cold_storage_path"])
-    if not directory.exists():
-        node_logger.warning(
-            "Cold storage path %s missing; skipping atom %s",
-            directory,
-            atom_hex,
-        )
-        return
-    filename = f"{atom_hex.upper()}.bin"
-    file_path = directory / filename
-    try:
-        file_path.write_bytes(atom_bytes)
-        self.cold_storage_size = projected
-        node_logger.debug("Persisted atom %s to cold storage", atom_hex)
-    except OSError as exc:
-        node_logger.error(
-            "Failed writing atom %s to cold storage %s: %s",
-            atom_hex,
-            file_path,
-            exc,
-        )
-
-
-def _network_set(self, atom: Atom) -> None:
-    """Advertise an atom to the closest known peer so they can fetch it from us."""
-    node_logger = self.logger
-    atom_id = atom.object_id()
-    atom_hex = atom_id.hex()
+from __future__ import annotations
+
+import socket
+from pathlib import Path
+
+from cryptography.hazmat.primitives import serialization
+
+from ..models.atom import Atom
+from ..providers import provider_id_for_payload
+
+
+def _hot_storage_set(self, key: bytes, value: Atom) -> bool:
+    """Store atom in hot storage without exceeding the configured limit."""
+    node_logger = self.logger
+    projected = self.hot_storage_size + value.size
+    hot_limit = self.config["hot_storage_limit"]
+    if projected > hot_limit:
+        node_logger.warning(
+            "Hot storage limit reached (%s > %s); skipping atom %s",
+            projected,
+            hot_limit,
+            key.hex(),
+        )
+        return False
+
+    self.hot_storage[key] = value
+    self.hot_storage_size = projected
+    node_logger.debug(
+        "Stored atom %s in hot storage (bytes=%s, total=%s)",
+        key.hex(),
+        value.size,
+        projected,
+    )
+    return True
+
+
+def _cold_storage_set(self, key: bytes, atom: Atom) -> None:
+    """Persist an atom into the cold storage directory if it already exists."""
+    node_logger = self.logger
+    atom_hex = key.hex()
+    if not self.config["cold_storage_path"]:
+        node_logger.debug("Cold storage disabled; skipping atom %s", atom_hex)
+        return
+    atom_bytes = atom.to_bytes()
+    projected = self.cold_storage_size + len(atom_bytes)
+    cold_limit = self.config["cold_storage_limit"]
+    if cold_limit and projected > cold_limit:
+        node_logger.warning(
+            "Cold storage limit reached (%s > %s); skipping atom %s",
+            projected,
+            cold_limit,
+            atom_hex,
+        )
+        return
+    directory = Path(self.config["cold_storage_path"])
+    if not directory.exists():
+        node_logger.warning(
+            "Cold storage path %s missing; skipping atom %s",
+            directory,
+            atom_hex,
+        )
+        return
+    filename = f"{atom_hex.upper()}.bin"
+    file_path = directory / filename
+    try:
+        file_path.write_bytes(atom_bytes)
+        self.cold_storage_size = projected
+        node_logger.debug("Persisted atom %s to cold storage", atom_hex)
+    except OSError as exc:
+        node_logger.error(
+            "Failed writing atom %s to cold storage %s: %s",
+            atom_hex,
+            file_path,
+            exc,
+        )
+
+
+def _network_set(self, atom_id: bytes) -> None:
+    """Advertise an atom id to the closest known peer so they can fetch it from us."""
+    node_logger = self.logger
+    atom_hex = atom_id.hex()
     try:
         from ...communication.handlers.object_request import (
             ObjectRequest,
             ObjectRequestType,
         )
         from ...communication.models.message import Message, MessageTopic
+        from ...communication.outgoing_queue import enqueue_outgoing
     except Exception as exc:
         node_logger.warning(
             "Communication module unavailable; cannot advertise atom %s: %s",
             atom_hex,
-            exc,
-        )
-        return
-
-    try:
-        provider_ip, provider_port = self.incoming_socket.getsockname()[:2]
-    except Exception as exc:
-        node_logger.warning(
-            "Unable to determine provider address for atom %s: %s",
-            atom_hex,
-            exc,
-        )
-        return
-
-    try:
-        provider_ip_bytes = socket.inet_aton(provider_ip)
-        provider_port_bytes = int(provider_port).to_bytes(2, "big", signed=False)
-        provider_key_bytes = self.relay_public_key_bytes
-    except Exception as exc:
-        node_logger.warning("Unable to encode provider info for %s: %s", atom_hex, exc)
-        return
-
-    provider_payload = provider_key_bytes + provider_ip_bytes + provider_port_bytes
-
-    try:
-        closest_peer = self.peer_route.closest_peer_for_hash(atom_id)
-    except Exception as exc:
-        node_logger.warning("Peer lookup failed for atom %s: %s", atom_hex, exc)
-        return
-
-    is_self_closest = False
-    if closest_peer is None or closest_peer.address is None:
-        is_self_closest = True
-    else:
-        try:
-            from ...communication.util import xor_distance
-        except Exception as exc:
-            node_logger.warning("Failed to import xor_distance for atom %s: %s", atom_hex, exc)
-            is_self_closest = True
-        else:
-            try:
-                self_distance = xor_distance(atom_id, self.relay_public_key_bytes)
-                peer_distance = xor_distance(atom_id, closest_peer.public_key_bytes)
-            except Exception as exc:
-                node_logger.warning("Failed computing distance for atom %s: %s", atom_hex, exc)
-                is_self_closest = True
-            else:
-                is_self_closest = self_distance <= peer_distance
-
-    if is_self_closest:
-        node_logger.debug("Self is closest; indexing provider for atom %s", atom_hex)
-        self.storage_index[atom_id] = provider_payload
-        return
-
-    target_addr = closest_peer.address
-
-    obj_req = ObjectRequest(
-        type=ObjectRequestType.OBJECT_PUT,
-        data=provider_payload,
-        atom_id=atom_id,
-    )
-    
-    message_body = obj_req.to_bytes()
-
+            exc,
+        )
+        return
+
+    try:
+        provider_ip, provider_port = self.incoming_socket.getsockname()[:2]
+    except Exception as exc:
+        node_logger.warning(
+            "Unable to determine provider address for atom %s: %s",
+            atom_hex,
+            exc,
+        )
+        return
+
+    try:
+        provider_ip_bytes = socket.inet_aton(provider_ip)
+        provider_port_bytes = int(provider_port).to_bytes(2, "big", signed=False)
+        provider_key_bytes = self.relay_public_key_bytes
+    except Exception as exc:
+        node_logger.warning("Unable to encode provider info for %s: %s", atom_hex, exc)
+        return
+
+    provider_payload = provider_key_bytes + provider_ip_bytes + provider_port_bytes
+
+    try:
+        closest_peer = self.peer_route.closest_peer_for_hash(atom_id)
+    except Exception as exc:
+        node_logger.warning("Peer lookup failed for atom %s: %s", atom_hex, exc)
+        return
+
+    is_self_closest = False
+    if closest_peer is None or closest_peer.address is None:
+        is_self_closest = True
+    else:
+        try:
+            from ...communication.util import xor_distance
+        except Exception as exc:
+            node_logger.warning("Failed to import xor_distance for atom %s: %s", atom_hex, exc)
+            is_self_closest = True
+        else:
+            try:
+                self_distance = xor_distance(atom_id, self.relay_public_key_bytes)
+                peer_distance = xor_distance(atom_id, closest_peer.public_key_bytes)
+            except Exception as exc:
+                node_logger.warning("Failed computing distance for atom %s: %s", atom_hex, exc)
+                is_self_closest = True
+            else:
+                is_self_closest = self_distance <= peer_distance
+
+    if is_self_closest:
+        node_logger.debug("Self is closest; indexing provider for atom %s", atom_hex)
+        provider_id = provider_id_for_payload(self, provider_payload)
+        self.storage_index[atom_id] = provider_id
+        return
+
+    target_addr = closest_peer.address
+
+    obj_req = ObjectRequest(
+        type=ObjectRequestType.OBJECT_PUT,
+        data=provider_payload,
+        atom_id=atom_id,
+    )
+    
+    message_body = obj_req.to_bytes()
+
     message = Message(
         topic=MessageTopic.OBJECT_REQUEST,
         content=message_body,
@@ -161,18 +162,32 @@ def _network_set(self, atom: Atom) -> None:
     )
     message.encrypt(closest_peer.shared_key_bytes)
     try:
-        self.outgoing_queue.put((message.to_bytes(), target_addr))
-        node_logger.debug(
-            "Advertised atom %s to peer at %s:%s",
-            atom_hex,
-            target_addr[0],
-            target_addr[1],
+        queued = enqueue_outgoing(
+            self,
+            target_addr,
+            message=message,
+            difficulty=closest_peer.difficulty,
         )
+        if queued:
+            node_logger.debug(
+                "Advertised atom %s to peer at %s:%s",
+                atom_hex,
+                target_addr[0],
+                target_addr[1],
+            )
+        else:
+            node_logger.debug(
+                "Dropped atom advertisement %s to peer at %s:%s",
+                atom_hex,
+                target_addr[0],
+                target_addr[1],
+            )
     except Exception as exc:
         node_logger.error(
             "Failed to queue advertisement for atom %s to %s:%s: %s",
             atom_hex,
-            target_addr[0],
-            target_addr[1],
-            exc,
-        )
+            target_addr[0],
+            target_addr[1],
+            exc,
+        )
+

astreum/storage/models/atom.py

@@ -91,17 +91,3 @@ def bytes_list_to_atoms(values: List[bytes]) -> Tuple[bytes, List[Atom]]:
 
     atoms.reverse()
     return (next_hash if values else ZERO32), atoms
-
-
-def get_atom_list_from_storage(self, root_hash: bytes) -> Optional[List["Atom"]]:
-    """Follow the list chain starting at root_hash, returning atoms or None on gaps."""
-    next_id: bytes = root_hash
-    atom_list: List["Atom"] = []
-    while next_id != ZERO32:
-        elem = self.storage_get(key=next_id)
-        if elem:
-            atom_list.append(elem)
-            next_id = elem.next_id
-        else:
-            return None
-    return atom_list

astreum/storage/models/trie.py

@@ -99,7 +99,7 @@ class TrieNode:
         if head_hash == ZERO32:
             raise ValueError("empty atom chain for Patricia node")
 
-        atom_chain = node.get_atom_list_from_storage(head_hash)
+        atom_chain = node.get_atom_list(head_hash)
         if atom_chain is None or len(atom_chain) != 5:
             raise ValueError("malformed Patricia atom chain")
 
@@ -177,7 +177,7 @@ class Trie:
         if cached is not None:
             return cached
 
-        if storage_node.storage_get(h) is None:
+        if storage_node.get_atom(atom_id=h) is None:
             return None
 
         pat_node = TrieNode.from_atoms(storage_node, h)

astreum/storage/providers.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from typing import Optional
+
+
+
+def provider_id_for_payload(node, payload: bytes) -> int:
+    """Return the provider id for a payload, inserting if new."""
+    for idx, existing in enumerate(node.storage_providers):
+        if existing == payload:
+            return idx
+    node.storage_providers.append(payload)
+    return len(node.storage_providers) - 1
+
+
+
+def provider_payload_for_id(node, provider_id: int) -> Optional[bytes]:
+    """Return the provider payload for a provider id, or None."""
+    if not isinstance(provider_id, int) or provider_id < 0:
+        return None
+    try:
+        return node.storage_providers[provider_id]
+    except IndexError:
+        return None

astreum/storage/requests.py

@@ -1,16 +1,16 @@
 from __future__ import annotations
 
 from threading import RLock
-from typing import TYPE_CHECKING
+from typing import Optional, TYPE_CHECKING
 
 if TYPE_CHECKING:
     from .. import Node
 
 
-def add_atom_req(node: "Node", atom_id: bytes) -> None:
-    """Mark an atom request as pending."""
+def add_atom_req(node: "Node", atom_id: bytes, payload_type: Optional[int] = None) -> None:
+    """Mark an atom request as pending with an optional payload type."""
     with node.atom_requests_lock:
-        node.atom_requests.add(atom_id)
+        node.atom_requests[atom_id] = payload_type
 
 
 def has_atom_req(node: "Node", atom_id: bytes) -> bool:
@@ -19,10 +19,13 @@ def has_atom_req(node: "Node", atom_id: bytes) -> bool:
         return atom_id in node.atom_requests
 
 
-def pop_atom_req(node: "Node", atom_id: bytes) -> bool:
-    """Remove the pending request if present. Returns True when removed."""
+def pop_atom_req(node: "Node", atom_id: bytes) -> Optional[int]:
+    """Remove the pending request if present and return its payload type."""
     with node.atom_requests_lock:
-        if atom_id in node.atom_requests:
-            node.atom_requests.remove(atom_id)
-            return True
-        return False
+        return node.atom_requests.pop(atom_id, None)
+
+
+def get_atom_req_payload(node: "Node", atom_id: bytes) -> Optional[int]:
+    """Return the payload type for a pending request without removing it."""
+    with node.atom_requests_lock:
+        return node.atom_requests.get(atom_id)

astreum/storage/setup.py

@@ -1,22 +1,27 @@
-from __future__ import annotations
-
-from typing import Any
-
-
-def storage_setup(node: Any, config: dict) -> None:
-    """Initialize hot/cold storage helpers on the node."""
-
-    node.logger.info("Setting up node storage")
-
-    node.hot_storage = {}
-    node.hot_storage_hits = {}
-    node.storage_index = {}
+from __future__ import annotations
+
+from typing import Any
+
+
+def storage_setup(node: Any, config: dict) -> None:
+    """Initialize hot/cold storage helpers on the node."""
+
+    node.logger.info("Setting up node storage")
+
+    node.hot_storage = {}
+    node.hot_storage_hits = {}
+    node.storage_index = {}
+    node.storage_providers = []
     node.hot_storage_size = 0
     node.cold_storage_size = 0
+    node.atom_fetch_interval = config["atom_fetch_interval"]
+    node.atom_fetch_retries = config["atom_fetch_retries"]
 
     node.logger.info(
-        "Storage ready (hot_limit=%s bytes, cold_limit=%s bytes, cold_path=%s)",
-        config["hot_storage_default_limit"],
+        "Storage ready (hot_limit=%s bytes, cold_limit=%s bytes, cold_path=%s, atom_fetch_interval=%s, atom_fetch_retries=%s)",
+        config["hot_storage_limit"],
         config["cold_storage_limit"],
         config["cold_storage_path"] or "disabled",
+        config["atom_fetch_interval"],
+        config["atom_fetch_retries"],
     )