astreum 0.3.16__py3-none-any.whl → 0.3.46__py3-none-any.whl

astreum/storage/actions/get.py

@@ -4,6 +4,7 @@ from pathlib import Path
 from typing import Optional
 
 from ..models.atom import Atom
+from ..providers import provider_payload_for_id
 
 
 def _hot_storage_get(self, key: bytes) -> Optional[Atom]:
@@ -24,11 +25,12 @@ def _network_get(self, key: bytes) -> Optional[Atom]:
         return None
     self.logger.debug("Attempting network fetch for %s", key.hex())
     try:
-        from ...communication.handlers.object_request import (
-            ObjectRequest,
-            ObjectRequestType,
-        )
-        from ...communication.models.message import Message, MessageTopic
+        from ...communication.handlers.object_request import (
+            ObjectRequest,
+            ObjectRequestType,
+        )
+        from ...communication.models.message import Message, MessageTopic
+        from ...communication.outgoing_queue import enqueue_outgoing
     except Exception as exc:
         self.logger.warning(
             "Communication module unavailable; cannot fetch %s: %s",
@@ -71,12 +73,24 @@ def _network_get(self, key: bytes) -> Optional[Atom]:
         self.logger.warning("Failed to track object request for %s: %s", key.hex(), exc)
 
     try:
-        self.outgoing_queue.put((message.to_bytes(), closest_peer.address))
-        self.logger.debug(
-            "Queued OBJECT_GET for %s to peer %s",
-            key.hex(),
-            closest_peer.address,
-        )
+        queued = enqueue_outgoing(
+            self,
+            closest_peer.address,
+            message=message,
+            difficulty=closest_peer.difficulty,
+        )
+        if queued:
+            self.logger.debug(
+                "Queued OBJECT_GET for %s to peer %s",
+                key.hex(),
+                closest_peer.address,
+            )
+        else:
+            self.logger.debug(
+                "Dropped OBJECT_GET for %s to peer %s",
+                key.hex(),
+                closest_peer.address,
+            )
     except Exception as exc:
         self.logger.warning(
             "Failed to queue OBJECT_GET for %s to %s: %s",
@@ -102,43 +116,60 @@ def storage_get(self, key: bytes) -> Optional[Atom]:
     if not self.is_connected:
         return None
     
-    provider_payload = self.storage_index.get(key)
-    if provider_payload is not None:
-        try:
-            from ...communication.handlers.object_response import decode_object_provider
-            from ...communication.handlers.object_request import (
-                ObjectRequest,
-                ObjectRequestType,
-            )
-            from ...communication.models.message import Message, MessageTopic
-            from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PublicKey
-
-            provider_key, provider_address, provider_port = decode_object_provider(provider_payload)
-            provider_public_key = X25519PublicKey.from_public_bytes(provider_key)
-            shared_key_bytes = self.relay_secret_key.exchange(provider_public_key)
-
-            obj_req = ObjectRequest(
-                type=ObjectRequestType.OBJECT_GET,
-                data=b"",
-                atom_id=key,
-            )
-            message = Message(
-                topic=MessageTopic.OBJECT_REQUEST,
-                content=obj_req.to_bytes(),
-                sender=self.relay_public_key,
-            )
-            message.encrypt(shared_key_bytes)
-            self.add_atom_req(key)
-            self.outgoing_queue.put((message.to_bytes(), (provider_address, provider_port)))
-            self.logger.debug(
-                "Requested atom %s from indexed provider %s:%s",
-                key.hex(),
-                provider_address,
-                provider_port,
-            )
-        except Exception as exc:
-            self.logger.warning("Failed indexed fetch for %s: %s", key.hex(), exc)
-        return None
+    provider_id = self.storage_index.get(key)
+    if provider_id is not None:
+        provider_payload = provider_payload_for_id(self, provider_id)
+        if provider_payload is not None:
+            try:
+                from ...communication.handlers.object_response import decode_object_provider
+                from ...communication.handlers.object_request import (
+                    ObjectRequest,
+                    ObjectRequestType,
+                )
+                from ...communication.models.message import Message, MessageTopic
+                from ...communication.outgoing_queue import enqueue_outgoing
+                from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PublicKey
+
+                provider_key, provider_address, provider_port = decode_object_provider(provider_payload)
+                provider_public_key = X25519PublicKey.from_public_bytes(provider_key)
+                shared_key_bytes = self.relay_secret_key.exchange(provider_public_key)
+
+                obj_req = ObjectRequest(
+                    type=ObjectRequestType.OBJECT_GET,
+                    data=b"",
+                    atom_id=key,
+                )
+                message = Message(
+                    topic=MessageTopic.OBJECT_REQUEST,
+                    content=obj_req.to_bytes(),
+                    sender=self.relay_public_key,
+                )
+                message.encrypt(shared_key_bytes)
+                self.add_atom_req(key)
+                queued = enqueue_outgoing(
+                    self,
+                    (provider_address, provider_port),
+                    message=message,
+                    difficulty=1,
+                )
+                if queued:
+                    self.logger.debug(
+                        "Requested atom %s from indexed provider %s:%s",
+                        key.hex(),
+                        provider_address,
+                        provider_port,
+                    )
+                else:
+                    self.logger.debug(
+                        "Dropped request for atom %s to indexed provider %s:%s",
+                        key.hex(),
+                        provider_address,
+                        provider_port,
+                    )
+            except Exception as exc:
+                self.logger.warning("Failed indexed fetch for %s: %s", key.hex(), exc)
+            return None
+        self.logger.warning("Unknown provider id %s for %s", provider_id, key.hex())
 
     self.logger.debug("Falling back to network fetch for %s", key.hex())
     return self._network_get(key)

astreum/storage/actions/set.py

@@ -1,159 +1,160 @@
-from __future__ import annotations
-
-import socket
-from pathlib import Path
-
-from cryptography.hazmat.primitives import serialization
-
-from ..models.atom import Atom
-
-
-def _hot_storage_set(self, key: bytes, value: Atom) -> bool:
-    """Store atom in hot storage without exceeding the configured limit."""
-    node_logger = self.logger
-    projected = self.hot_storage_size + value.size
-    hot_limit = self.config["hot_storage_default_limit"]
-    if projected > hot_limit:
-        node_logger.warning(
-            "Hot storage limit reached (%s > %s); skipping atom %s",
-            projected,
-            hot_limit,
-            key.hex(),
-        )
-        return False
-
-    self.hot_storage[key] = value
-    self.hot_storage_size = projected
-    node_logger.debug(
-        "Stored atom %s in hot storage (bytes=%s, total=%s)",
-        key.hex(),
-        value.size,
-        projected,
-    )
-    return True
-
-
-def _cold_storage_set(self, atom: Atom) -> None:
-    """Persist an atom into the cold storage directory if it already exists."""
-    node_logger = self.logger
-    atom_id = atom.object_id()
-    atom_hex = atom_id.hex()
-    if not self.config["cold_storage_path"]:
-        node_logger.debug("Cold storage disabled; skipping atom %s", atom_hex)
-        return
-    atom_bytes = atom.to_bytes()
-    projected = self.cold_storage_size + len(atom_bytes)
-    cold_limit = self.config["cold_storage_limit"]
-    if cold_limit and projected > cold_limit:
-        node_logger.warning(
-            "Cold storage limit reached (%s > %s); skipping atom %s",
-            projected,
-            cold_limit,
-            atom_hex,
-        )
-        return
-    directory = Path(self.config["cold_storage_path"])
-    if not directory.exists():
-        node_logger.warning(
-            "Cold storage path %s missing; skipping atom %s",
-            directory,
-            atom_hex,
-        )
-        return
-    filename = f"{atom_hex.upper()}.bin"
-    file_path = directory / filename
-    try:
-        file_path.write_bytes(atom_bytes)
-        self.cold_storage_size = projected
-        node_logger.debug("Persisted atom %s to cold storage", atom_hex)
-    except OSError as exc:
-        node_logger.error(
-            "Failed writing atom %s to cold storage %s: %s",
-            atom_hex,
-            file_path,
-            exc,
-        )
-
-
-def _network_set(self, atom: Atom) -> None:
-    """Advertise an atom to the closest known peer so they can fetch it from us."""
-    node_logger = self.logger
-    atom_id = atom.object_id()
-    atom_hex = atom_id.hex()
+from __future__ import annotations
+
+import socket
+from pathlib import Path
+
+from cryptography.hazmat.primitives import serialization
+
+from ..models.atom import Atom
+from ..providers import provider_id_for_payload
+
+
+def _hot_storage_set(self, key: bytes, value: Atom) -> bool:
+    """Store atom in hot storage without exceeding the configured limit."""
+    node_logger = self.logger
+    projected = self.hot_storage_size + value.size
+    hot_limit = self.config["hot_storage_limit"]
+    if projected > hot_limit:
+        node_logger.warning(
+            "Hot storage limit reached (%s > %s); skipping atom %s",
+            projected,
+            hot_limit,
+            key.hex(),
+        )
+        return False
+
+    self.hot_storage[key] = value
+    self.hot_storage_size = projected
+    node_logger.debug(
+        "Stored atom %s in hot storage (bytes=%s, total=%s)",
+        key.hex(),
+        value.size,
+        projected,
+    )
+    return True
+
+
+def _cold_storage_set(self, key: bytes, atom: Atom) -> None:
+    """Persist an atom into the cold storage directory if it already exists."""
+    node_logger = self.logger
+    atom_hex = key.hex()
+    if not self.config["cold_storage_path"]:
+        node_logger.debug("Cold storage disabled; skipping atom %s", atom_hex)
+        return
+    atom_bytes = atom.to_bytes()
+    projected = self.cold_storage_size + len(atom_bytes)
+    cold_limit = self.config["cold_storage_limit"]
+    if cold_limit and projected > cold_limit:
+        node_logger.warning(
+            "Cold storage limit reached (%s > %s); skipping atom %s",
+            projected,
+            cold_limit,
+            atom_hex,
+        )
+        return
+    directory = Path(self.config["cold_storage_path"])
+    if not directory.exists():
+        node_logger.warning(
+            "Cold storage path %s missing; skipping atom %s",
+            directory,
+            atom_hex,
+        )
+        return
+    filename = f"{atom_hex.upper()}.bin"
+    file_path = directory / filename
+    try:
+        file_path.write_bytes(atom_bytes)
+        self.cold_storage_size = projected
+        node_logger.debug("Persisted atom %s to cold storage", atom_hex)
+    except OSError as exc:
+        node_logger.error(
+            "Failed writing atom %s to cold storage %s: %s",
+            atom_hex,
+            file_path,
+            exc,
+        )
+
+
+def _network_set(self, atom_id: bytes) -> None:
+    """Advertise an atom id to the closest known peer so they can fetch it from us."""
+    node_logger = self.logger
+    atom_hex = atom_id.hex()
     try:
         from ...communication.handlers.object_request import (
             ObjectRequest,
             ObjectRequestType,
         )
         from ...communication.models.message import Message, MessageTopic
+        from ...communication.outgoing_queue import enqueue_outgoing
     except Exception as exc:
         node_logger.warning(
             "Communication module unavailable; cannot advertise atom %s: %s",
             atom_hex,
-            exc,
-        )
-        return
-
-    try:
-        provider_ip, provider_port = self.incoming_socket.getsockname()[:2]
-    except Exception as exc:
-        node_logger.warning(
-            "Unable to determine provider address for atom %s: %s",
-            atom_hex,
-            exc,
-        )
-        return
-
-    try:
-        provider_ip_bytes = socket.inet_aton(provider_ip)
-        provider_port_bytes = int(provider_port).to_bytes(2, "big", signed=False)
-        provider_key_bytes = self.relay_public_key_bytes
-    except Exception as exc:
-        node_logger.warning("Unable to encode provider info for %s: %s", atom_hex, exc)
-        return
-
-    provider_payload = provider_key_bytes + provider_ip_bytes + provider_port_bytes
-
-    try:
-        closest_peer = self.peer_route.closest_peer_for_hash(atom_id)
-    except Exception as exc:
-        node_logger.warning("Peer lookup failed for atom %s: %s", atom_hex, exc)
-        return
-
-    is_self_closest = False
-    if closest_peer is None or closest_peer.address is None:
-        is_self_closest = True
-    else:
-        try:
-            from ...communication.util import xor_distance
-        except Exception as exc:
-            node_logger.warning("Failed to import xor_distance for atom %s: %s", atom_hex, exc)
-            is_self_closest = True
-        else:
-            try:
-                self_distance = xor_distance(atom_id, self.relay_public_key_bytes)
-                peer_distance = xor_distance(atom_id, closest_peer.public_key_bytes)
-            except Exception as exc:
-                node_logger.warning("Failed computing distance for atom %s: %s", atom_hex, exc)
-                is_self_closest = True
-            else:
-                is_self_closest = self_distance <= peer_distance
-
-    if is_self_closest:
-        node_logger.debug("Self is closest; indexing provider for atom %s", atom_hex)
-        self.storage_index[atom_id] = provider_payload
-        return
-
-    target_addr = closest_peer.address
-
-    obj_req = ObjectRequest(
-        type=ObjectRequestType.OBJECT_PUT,
-        data=provider_payload,
-        atom_id=atom_id,
-    )
-    
-    message_body = obj_req.to_bytes()
-
+            exc,
+        )
+        return
+
+    try:
+        provider_ip, provider_port = self.incoming_socket.getsockname()[:2]
+    except Exception as exc:
+        node_logger.warning(
+            "Unable to determine provider address for atom %s: %s",
+            atom_hex,
+            exc,
+        )
+        return
+
+    try:
+        provider_ip_bytes = socket.inet_aton(provider_ip)
+        provider_port_bytes = int(provider_port).to_bytes(2, "big", signed=False)
+        provider_key_bytes = self.relay_public_key_bytes
+    except Exception as exc:
+        node_logger.warning("Unable to encode provider info for %s: %s", atom_hex, exc)
+        return
+
+    provider_payload = provider_key_bytes + provider_ip_bytes + provider_port_bytes
+
+    try:
+        closest_peer = self.peer_route.closest_peer_for_hash(atom_id)
+    except Exception as exc:
+        node_logger.warning("Peer lookup failed for atom %s: %s", atom_hex, exc)
+        return
+
+    is_self_closest = False
+    if closest_peer is None or closest_peer.address is None:
+        is_self_closest = True
+    else:
+        try:
+            from ...communication.util import xor_distance
+        except Exception as exc:
+            node_logger.warning("Failed to import xor_distance for atom %s: %s", atom_hex, exc)
+            is_self_closest = True
+        else:
+            try:
+                self_distance = xor_distance(atom_id, self.relay_public_key_bytes)
+                peer_distance = xor_distance(atom_id, closest_peer.public_key_bytes)
+            except Exception as exc:
+                node_logger.warning("Failed computing distance for atom %s: %s", atom_hex, exc)
+                is_self_closest = True
+            else:
+                is_self_closest = self_distance <= peer_distance
+
+    if is_self_closest:
+        node_logger.debug("Self is closest; indexing provider for atom %s", atom_hex)
+        provider_id = provider_id_for_payload(self, provider_payload)
+        self.storage_index[atom_id] = provider_id
+        return
+
+    target_addr = closest_peer.address
+
+    obj_req = ObjectRequest(
+        type=ObjectRequestType.OBJECT_PUT,
+        data=provider_payload,
+        atom_id=atom_id,
+    )
+    
+    message_body = obj_req.to_bytes()
+
     message = Message(
         topic=MessageTopic.OBJECT_REQUEST,
         content=message_body,
@@ -161,18 +162,32 @@ def _network_set(self, atom: Atom) -> None:
     )
     message.encrypt(closest_peer.shared_key_bytes)
     try:
-        self.outgoing_queue.put((message.to_bytes(), target_addr))
-        node_logger.debug(
-            "Advertised atom %s to peer at %s:%s",
-            atom_hex,
-            target_addr[0],
-            target_addr[1],
+        queued = enqueue_outgoing(
+            self,
+            target_addr,
+            message=message,
+            difficulty=closest_peer.difficulty,
         )
+        if queued:
+            node_logger.debug(
+                "Advertised atom %s to peer at %s:%s",
+                atom_hex,
+                target_addr[0],
+                target_addr[1],
+            )
+        else:
+            node_logger.debug(
+                "Dropped atom advertisement %s to peer at %s:%s",
+                atom_hex,
+                target_addr[0],
+                target_addr[1],
+            )
     except Exception as exc:
         node_logger.error(
             "Failed to queue advertisement for atom %s to %s:%s: %s",
             atom_hex,
-            target_addr[0],
-            target_addr[1],
-            exc,
-        )
+            target_addr[0],
+            target_addr[1],
+            exc,
+        )
+

astreum/storage/providers.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from typing import Optional
+
+
+
+def provider_id_for_payload(node, payload: bytes) -> int:
+    """Return the provider id for a payload, inserting if new."""
+    for idx, existing in enumerate(node.storage_providers):
+        if existing == payload:
+            return idx
+    node.storage_providers.append(payload)
+    return len(node.storage_providers) - 1
+
+
+
+def provider_payload_for_id(node, provider_id: int) -> Optional[bytes]:
+    """Return the provider payload for a provider id, or None."""
+    if not isinstance(provider_id, int) or provider_id < 0:
+        return None
+    try:
+        return node.storage_providers[provider_id]
+    except IndexError:
+        return None

astreum/storage/setup.py

@@ -1,22 +1,23 @@
-from __future__ import annotations
-
-from typing import Any
-
-
-def storage_setup(node: Any, config: dict) -> None:
-    """Initialize hot/cold storage helpers on the node."""
-
-    node.logger.info("Setting up node storage")
-
-    node.hot_storage = {}
-    node.hot_storage_hits = {}
-    node.storage_index = {}
-    node.hot_storage_size = 0
-    node.cold_storage_size = 0
-
-    node.logger.info(
-        "Storage ready (hot_limit=%s bytes, cold_limit=%s bytes, cold_path=%s)",
-        config["hot_storage_default_limit"],
-        config["cold_storage_limit"],
-        config["cold_storage_path"] or "disabled",
-    )
+from __future__ import annotations
+
+from typing import Any
+
+
+def storage_setup(node: Any, config: dict) -> None:
+    """Initialize hot/cold storage helpers on the node."""
+
+    node.logger.info("Setting up node storage")
+
+    node.hot_storage = {}
+    node.hot_storage_hits = {}
+    node.storage_index = {}
+    node.storage_providers = []
+    node.hot_storage_size = 0
+    node.cold_storage_size = 0
+
+    node.logger.info(
+        "Storage ready (hot_limit=%s bytes, cold_limit=%s bytes, cold_path=%s)",
+        config["hot_storage_limit"],
+        config["cold_storage_limit"],
+        config["cold_storage_path"] or "disabled",
+    )