koi-net 1.0.0b19__py3-none-any.whl → 1.1.0b1__py3-none-any.whl

koi_net/network/{interface.py → event_queue.py}

@@ -1,21 +1,19 @@
 import logging
 from queue import Queue
-from typing import Generic
 import httpx
 from pydantic import BaseModel
 from rid_lib import RID
-from rid_lib.core import RIDType
 from rid_lib.ext import Cache
 from rid_lib.types import KoiNetNode
 
 from .graph import NetworkGraph
 from .request_handler import RequestHandler
-from .response_handler import ResponseHandler
-from ..protocol.node import NodeType
-from ..protocol.edge import EdgeType
+from ..protocol.node import NodeProfile, NodeType
+from ..protocol.edge import EdgeProfile, EdgeType
 from ..protocol.event import Event
 from ..identity import NodeIdentity
-from ..config import ConfigType
+from ..config import NodeConfig
+from ..effector import Effector
 
 logger = logging.getLogger(__name__)
 
@@ -26,34 +24,36 @@ class EventQueueModel(BaseModel):
 
 type EventQueue = dict[RID, Queue[Event]]
 
-class NetworkInterface(Generic[ConfigType]):
+class NetworkEventQueue:
     """A collection of functions and classes to interact with the KOI network."""
     
-    config: ConfigType    
+    config: NodeConfig    
     identity: NodeIdentity
+    effector: Effector
     cache: Cache
     graph: NetworkGraph
     request_handler: RequestHandler
-    response_handler: ResponseHandler
     poll_event_queue: EventQueue
     webhook_event_queue: EventQueue
     
     def __init__(
         self, 
-        config: ConfigType,
+        config: NodeConfig,
         cache: Cache, 
-        identity: NodeIdentity
+        identity: NodeIdentity,
+        effector: Effector,
+        graph: NetworkGraph,
+        request_handler: RequestHandler,
     ):
         self.config = config
         self.identity = identity
         self.cache = cache
-        self.graph = NetworkGraph(cache, identity)
-        self.request_handler = RequestHandler(cache, self.graph)
-        self.response_handler = ResponseHandler(cache)
+        self.graph = graph
+        self.request_handler = request_handler
+        self.effector = effector
         
         self.poll_event_queue = dict()
         self.webhook_event_queue = dict()
-        self._load_event_queues()
     
     def _load_event_queues(self):
         """Loads event queues from storage."""
@@ -100,29 +100,43 @@ class NetworkInterface(Generic[ConfigType]):
         
         Event will be sent to webhook or poll queue depending on the node type and edge type of the specified node. If `flush` is set to `True`, the webhook queued will be flushed after pushing the event.
         """
-        logger.debug(f"Pushing event {event.event_type} {event.rid} to {node}")
-            
-        node_profile = self.graph.get_node_profile(node)
-        if not node_profile:
-            logger.warning(f"Node {node!r} unknown to me")
+        logger.debug(f"Pushing event {event.event_type} {event.rid!r} to {node}")
+                        
+        node_bundle = self.effector.deref(node)
         
         # if there's an edge from me to the target node, override broadcast type
-        edge_profile = self.graph.get_edge_profile(
+        edge_rid = self.graph.get_edge(
             source=self.identity.rid,
             target=node
         )
         
-        if edge_profile:
+        edge_bundle = self.effector.deref(edge_rid) if edge_rid else None
+        
+        if edge_bundle:
+            logger.debug(f"Found edge from me to {node!r}")
+            edge_profile = edge_bundle.validate_contents(EdgeProfile)
             if edge_profile.edge_type == EdgeType.WEBHOOK:
                 event_queue = self.webhook_event_queue
             elif edge_profile.edge_type == EdgeType.POLL:
                 event_queue = self.poll_event_queue
-        else:
+                
+        elif node_bundle:
+            logger.debug(f"Found bundle for {node!r}")
+            node_profile = node_bundle.validate_contents(NodeProfile)
             if node_profile.node_type == NodeType.FULL:
                 event_queue = self.webhook_event_queue
             elif node_profile.node_type == NodeType.PARTIAL:
                 event_queue = self.poll_event_queue
         
+        elif node == self.config.koi_net.first_contact.rid:
+            logger.debug(f"Node {node!r} is my first contact")
+            # first contact node is always a webhook node
+            event_queue = self.webhook_event_queue
+        
+        else:
+            logger.warning(f"Node {node!r} unknown to me")
+            return
+        
         queue = event_queue.setdefault(node, Queue())
         queue.put(event)
                 
@@ -136,7 +150,7 @@ class NetworkInterface(Generic[ConfigType]):
         if queue:
             while not queue.empty():
                 event = queue.get()
-                logger.debug(f"Dequeued {event.event_type} '{event.rid}'")
+                logger.debug(f"Dequeued {event.event_type} {event.rid!r}")
                 events.append(event)
         
         return events
@@ -146,7 +160,7 @@ class NetworkInterface(Generic[ConfigType]):
         logger.debug(f"Flushing poll queue for {node}")
         return self._flush_queue(self.poll_event_queue, node)
     
-    def flush_webhook_queue(self, node: KoiNetNode):
+    def flush_webhook_queue(self, node: KoiNetNode, requeue_on_fail: bool = True):
         """Flushes a node's webhook queue, and broadcasts events.
         
         If node profile is unknown, or node type is not `FULL`, this operation will fail silently. If the remote node cannot be reached, all events will be requeued.
@@ -154,15 +168,17 @@ class NetworkInterface(Generic[ConfigType]):
         
         logger.debug(f"Flushing webhook queue for {node}")
         
-        node_profile = self.graph.get_node_profile(node)
+        # node_bundle = self.effector.deref(node)
         
-        if not node_profile:
-            logger.warning(f"{node!r} not found")
-            return
+        # if not node_bundle:
+        #     logger.warning(f"{node!r} not found")
+        #     return
         
-        if node_profile.node_type != NodeType.FULL:
-            logger.warning(f"{node!r} is a partial node!")
-            return
+        # node_profile = node_bundle.validate_contents(NodeProfile)
+        
+        # if node_profile.node_type != NodeType.FULL:
+        #     logger.warning(f"{node!r} is a partial node!")
+        #     return
         
         events = self._flush_queue(self.webhook_event_queue, node)
         if not events: return
@@ -174,103 +190,8 @@ class NetworkInterface(Generic[ConfigType]):
             return True
         except httpx.ConnectError:
             logger.warning("Broadcast failed")
-            for event in events:
-                self.push_event_to(event, node)
-            return False
-            
-    def get_state_providers(self, rid_type: RIDType) -> list[KoiNetNode]:
-        """Returns list of node RIDs which provide state for the specified RID type."""
-        
-        logger.debug(f"Looking for state providers of '{rid_type}'")
-        provider_nodes = []
-        for node_rid in self.cache.list_rids(rid_types=[KoiNetNode]):
-            node = self.graph.get_node_profile(node_rid)
-                        
-            if node.node_type == NodeType.FULL and rid_type in node.provides.state:
-                logger.debug(f"Found provider '{node_rid}'")
-                provider_nodes.append(node_rid)
-        
-        if not provider_nodes:
-            logger.debug("Failed to find providers")
-        return provider_nodes
             
-    def fetch_remote_bundle(self, rid: RID):
-        """Attempts to fetch a bundle by RID from known peer nodes."""
-        
-        logger.debug(f"Fetching remote bundle '{rid}'")
-        remote_bundle = None
-        for node_rid in self.get_state_providers(type(rid)):
-            payload = self.request_handler.fetch_bundles(
-                node=node_rid, rids=[rid])
-            
-            if payload.bundles:
-                remote_bundle = payload.bundles[0]
-                logger.debug(f"Got bundle from '{node_rid}'")
-                break
-        
-        if not remote_bundle:
-            logger.warning("Failed to fetch remote bundle")
-            
-        return remote_bundle
-    
-    def fetch_remote_manifest(self, rid: RID):
-        """Attempts to fetch a manifest by RID from known peer nodes."""
-        
-        logger.debug(f"Fetching remote manifest '{rid}'")
-        remote_manifest = None
-        for node_rid in self.get_state_providers(type(rid)):
-            payload = self.request_handler.fetch_manifests(
-                node=node_rid, rids=[rid])
-            
-            if payload.manifests:
-                remote_manifest = payload.manifests[0]
-                logger.debug(f"Got bundle from '{node_rid}'")
-                break
-        
-        if not remote_manifest:
-            logger.warning("Failed to fetch remote bundle")
-            
-        return remote_manifest
-    
-    def poll_neighbors(self) -> list[Event]:
-        """Polls all neighboring nodes and returns compiled list of events.
-        
-        If this node has no neighbors, it will instead attempt to poll the provided first contact URL.
-        """
-        
-        neighbors = self.graph.get_neighbors()
-        
-        if not neighbors and self.config.koi_net.first_contact:
-            logger.debug("No neighbors found, polling first contact")
-            try:
-                payload = self.request_handler.poll_events(
-                    url=self.config.koi_net.first_contact, 
-                    rid=self.identity.rid
-                )
-                if payload.events:
-                    logger.debug(f"Received {len(payload.events)} events from '{self.config.koi_net.first_contact}'")
-                return payload.events
-            except httpx.ConnectError:
-                logger.debug(f"Failed to reach first contact '{self.config.koi_net.first_contact}'")
-        
-        events = []
-        for node_rid in neighbors:
-            node = self.graph.get_node_profile(node_rid)
-            if not node: continue
-            if node.node_type != NodeType.FULL: continue
-            
-            try:
-                payload = self.request_handler.poll_events(
-                    node=node_rid, 
-                    rid=self.identity.rid
-                )
-                if payload.events:
-                    logger.debug(f"Received {len(payload.events)} events from {node_rid!r}")
-                events.extend(payload.events)
-            except httpx.ConnectError:
-                logger.debug(f"Failed to reach node '{node_rid}'")
-                continue
-            
-        return events                
-        
-        
+            if requeue_on_fail:
+                for event in events:
+                    self.push_event_to(event, node)
+            return False

koi_net/network/graph.py

@@ -6,7 +6,6 @@ from rid_lib.ext import Cache
 from rid_lib.types import KoiNetEdge, KoiNetNode
 from ..identity import NodeIdentity
 from ..protocol.edge import EdgeProfile, EdgeStatus
-from ..protocol.node import NodeProfile
 
 logger = logging.getLogger(__name__)
 
@@ -30,43 +29,27 @@ class NetworkGraph:
         for rid in self.cache.list_rids():
             if type(rid) == KoiNetNode:                
                 self.dg.add_node(rid)
-                logger.debug(f"Added node {rid}")
+                logger.debug(f"Added node {rid!r}")
                 
             elif type(rid) == KoiNetEdge:
-                edge_profile = self.get_edge_profile(rid)
-                if not edge_profile:
+                edge_bundle = self.cache.read(rid)
+                if not edge_bundle:
                     logger.warning(f"Failed to load {rid!r}")
                     continue
+                edge_profile = edge_bundle.validate_contents(EdgeProfile)
                 self.dg.add_edge(edge_profile.source, edge_profile.target, rid=rid)
-                logger.debug(f"Added edge {rid} ({edge_profile.source} -> {edge_profile.target})")
+                logger.debug(f"Added edge {rid!r} ({edge_profile.source} -> {edge_profile.target})")
         logger.debug("Done")
         
-    def get_node_profile(self, rid: KoiNetNode) -> NodeProfile | None:
-        """Returns node profile given its RID."""
-        bundle = self.cache.read(rid)
-        if bundle:
-            return bundle.validate_contents(NodeProfile)
-        
-    def get_edge_profile(
-        self, 
-        rid: KoiNetEdge | None = None,
-        source: KoiNetNode | None = None, 
-        target: KoiNetNode | None = None,
-    ) -> EdgeProfile | None:
-        """Returns edge profile given its RID, or source and target node RIDs."""
-        if source and target:
-            if (source, target) not in self.dg.edges: return
+    def get_edge(self, source: KoiNetNode, target: KoiNetNode,) -> EdgeProfile | None:
+        """Returns edge RID given the RIDs of a source and target node."""
+        if (source, target) in self.dg.edges:
             edge_data = self.dg.get_edge_data(source, target)
-            if not edge_data: return
-            rid = edge_data.get("rid")
-            if not rid: return
-        elif not rid:
-            raise ValueError("Either 'rid' or 'source' and 'target' must be provided")
-        
-        bundle = self.cache.read(rid)
-        if bundle:
-            return bundle.validate_contents(EdgeProfile)
-        
+            if edge_data:
+                return edge_data.get("rid")
+
+        return None
+
     def get_edges(
         self,
         direction: Literal["in", "out"] | None = None,
@@ -106,12 +89,14 @@ class NetworkGraph:
         
         neighbors = []
         for edge_rid in self.get_edges(direction):
-            edge_profile = self.get_edge_profile(edge_rid)
+            edge_bundle = self.cache.read(edge_rid)
             
-            if not edge_profile: 
+            if not edge_bundle: 
                 logger.warning(f"Failed to find edge {edge_rid!r} in cache")
                 continue
-                        
+            
+            edge_profile = edge_bundle.validate_contents(EdgeProfile)
+            
             if status and edge_profile.status != status:
                 continue
             

koi_net/network/request_handler.py

@@ -1,8 +1,9 @@
 import logging
 import httpx
 from rid_lib import RID
-from rid_lib.ext import Cache
 from rid_lib.types.koi_net_node import KoiNetNode
+
+from ..identity import NodeIdentity
 from ..protocol.api_models import (
     RidsPayload,
     ManifestsPayload,
@@ -13,8 +14,10 @@ from ..protocol.api_models import (
     FetchBundles,
     PollEvents,
     RequestModels,
-    ResponseModels
+    ResponseModels,
+    ErrorResponse
 )
+from ..protocol.envelope import SignedEnvelope
 from ..protocol.consts import (
     BROADCAST_EVENTS_PATH,
     POLL_EVENTS_PATH,
@@ -22,8 +25,13 @@ from ..protocol.consts import (
     FETCH_MANIFESTS_PATH,
     FETCH_BUNDLES_PATH
 )
-from ..protocol.node import NodeType
-from .graph import NetworkGraph
+from ..protocol.node import NodeProfile, NodeType
+from ..secure import Secure
+from ..effector import Effector
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from .error_handler import ErrorHandler
 
 
 logger = logging.getLogger(__name__)
@@ -32,118 +40,156 @@ logger = logging.getLogger(__name__)
 class RequestHandler:
     """Handles making requests to other KOI nodes."""
     
-    cache: Cache
-    graph: NetworkGraph
+    effector: Effector
+    identity: NodeIdentity
+    secure: Secure
+    error_handler: "ErrorHandler"
     
-    def __init__(self, cache: Cache, graph: NetworkGraph):
-        self.cache = cache
-        self.graph = graph
+    def __init__(
+        self, 
+        effector: Effector,
+        identity: NodeIdentity,
+        secure: Secure
+    ):
+        self.effector = effector
+        self.identity = identity
+        self.secure = secure
+        
+    def set_error_handler(self, error_handler: "ErrorHandler"):
+        self.error_handler = error_handler
+    
+    def get_url(self, node_rid: KoiNetNode) -> str:
+        """Retrieves URL of a node."""
+        
+        logger.debug(f"Getting URL for {node_rid!r}")
+        node_url = None
+        
+        if node_rid == self.identity.rid:
+            raise Exception("Don't talk to yourself")
+        
+        node_bundle = self.effector.deref(node_rid)
                 
+        if node_bundle:
+            node_profile = node_bundle.validate_contents(NodeProfile)
+            logger.debug(f"Found node profile: {node_profile}")
+            if node_profile.node_type != NodeType.FULL:
+                raise Exception("Can't query partial node")
+            node_url = node_profile.base_url
+        
+        else:
+            if node_rid == self.identity.config.koi_net.first_contact.rid:
+                logger.debug("Found URL of first contact")
+                node_url = self.identity.config.koi_net.first_contact.url
+        
+        if not node_url:
+            raise Exception("Node not found")
+        
+        logger.debug(f"Resolved {node_rid!r} to {node_url}")
+        return node_url
+    
     def make_request(
-        self, 
-        url: str, 
+        self,
+        node: KoiNetNode,
+        path: str, 
         request: RequestModels,
-        response_model: type[ResponseModels] | None = None
     ) -> ResponseModels | None:
-        logger.debug(f"Making request to {url}")
-        resp = httpx.post(
-            url=url,
-            data=request.model_dump_json()
+        url = self.get_url(node) + path
+        logger.info(f"Making request to {url}")
+    
+        signed_envelope = self.secure.create_envelope(
+            payload=request,
+            target=node
         )
-        if response_model:
-            return response_model.model_validate_json(resp.text)
-            
-    def get_url(self, node_rid: KoiNetNode, url: str) -> str:
-        """Retrieves URL of a node, or returns provided URL."""
         
-        if not node_rid and not url:
-            raise ValueError("One of 'node_rid' and 'url' must be provided")
+        try:
+            result = httpx.post(url, data=signed_envelope.model_dump_json())
+        except httpx.ConnectError as err:
+            logger.debug("Failed to connect")
+            self.error_handler.handle_connection_error(node)
+            raise err
         
-        if node_rid:
-            node_profile = self.graph.get_node_profile(node_rid)
-            if not node_profile:
-                raise Exception("Node not found")
-            if node_profile.node_type != NodeType.FULL:
-                raise Exception("Can't query partial node")
-            logger.debug(f"Resolved {node_rid!r} to {node_profile.base_url}")
-            return node_profile.base_url
+        if result.status_code != 200:
+            resp = ErrorResponse.model_validate_json(result.text)
+            self.error_handler.handle_protocol_error(resp.error, node)
+            return resp
+        
+        if path == BROADCAST_EVENTS_PATH:
+            return None
+        elif path == POLL_EVENTS_PATH:
+            EnvelopeModel = SignedEnvelope[EventsPayload]
+        elif path == FETCH_RIDS_PATH:
+            EnvelopeModel = SignedEnvelope[RidsPayload]
+        elif path == FETCH_MANIFESTS_PATH:
+            EnvelopeModel = SignedEnvelope[ManifestsPayload]
+        elif path == FETCH_BUNDLES_PATH:
+            EnvelopeModel = SignedEnvelope[BundlesPayload]
         else:
-            return url
+            raise Exception(f"Unknown path '{path}'")
+        
+        resp_envelope = EnvelopeModel.model_validate_json(result.text)
+        self.secure.validate_envelope(resp_envelope)
+        
+        return resp_envelope.payload
     
     def broadcast_events(
         self, 
-        node: RID = None, 
-        url: str = None, 
+        node: RID, 
         req: EventsPayload | None = None,
         **kwargs
     ) -> None:
         """See protocol.api_models.EventsPayload for available kwargs."""
         request = req or EventsPayload.model_validate(kwargs)
-        self.make_request(
-            self.get_url(node, url) + BROADCAST_EVENTS_PATH, request
-        )
-        logger.info(f"Broadcasted {len(request.events)} event(s) to {node or url!r}")
+        self.make_request(node, BROADCAST_EVENTS_PATH, request)
+        logger.info(f"Broadcasted {len(request.events)} event(s) to {node!r}")
         
     def poll_events(
         self, 
-        node: RID = None, 
-        url: str = None, 
+        node: RID, 
         req: PollEvents | None = None,
         **kwargs
     ) -> EventsPayload:
         """See protocol.api_models.PollEvents for available kwargs."""
         request = req or PollEvents.model_validate(kwargs)
-        resp = self.make_request(
-            self.get_url(node, url) + POLL_EVENTS_PATH, request,
-            response_model=EventsPayload
-        )
-        logger.info(f"Polled {len(resp.events)} events from {node or url!r}")
+        resp = self.make_request(node, POLL_EVENTS_PATH, request)
+        if type(resp) != ErrorResponse:
+            logger.info(f"Polled {len(resp.events)} events from {node!r}")
         return resp
         
     def fetch_rids(
         self, 
-        node: RID = None, 
-        url: str = None, 
+        node: RID, 
         req: FetchRids | None = None,
         **kwargs
     ) -> RidsPayload:
         """See protocol.api_models.FetchRids for available kwargs."""
         request = req or FetchRids.model_validate(kwargs)
-        resp = self.make_request(
-            self.get_url(node, url) + FETCH_RIDS_PATH, request,
-            response_model=RidsPayload
-        )
-        logger.info(f"Fetched {len(resp.rids)} RID(s) from {node or url!r}")
+        resp = self.make_request(node, FETCH_RIDS_PATH, request)
+        if type(resp) != ErrorResponse:
+            logger.info(f"Fetched {len(resp.rids)} RID(s) from {node!r}")
         return resp
                 
     def fetch_manifests(
         self, 
-        node: RID = None, 
-        url: str = None, 
+        node: RID, 
         req: FetchManifests | None = None,
         **kwargs
     ) -> ManifestsPayload:
         """See protocol.api_models.FetchManifests for available kwargs."""
         request = req or FetchManifests.model_validate(kwargs)
-        resp = self.make_request(
-            self.get_url(node, url) + FETCH_MANIFESTS_PATH, request,
-            response_model=ManifestsPayload
-        )
-        logger.info(f"Fetched {len(resp.manifests)} manifest(s) from {node or url!r}")
+        resp = self.make_request(node, FETCH_MANIFESTS_PATH, request)
+        if type(resp) != ErrorResponse:
+            logger.info(f"Fetched {len(resp.manifests)} manifest(s) from {node!r}")
         return resp
                 
     def fetch_bundles(
         self, 
-        node: RID = None, 
-        url: str = None, 
+        node: RID, 
         req: FetchBundles | None = None,
         **kwargs
     ) -> BundlesPayload:
         """See protocol.api_models.FetchBundles for available kwargs."""
         request = req or FetchBundles.model_validate(kwargs)
-        resp = self.make_request(
-            self.get_url(node, url) + FETCH_BUNDLES_PATH, request,
-            response_model=BundlesPayload
-        )
-        logger.info(f"Fetched {len(resp.bundles)} bundle(s) from {node or url!r}")
+        resp = self.make_request(node, FETCH_BUNDLES_PATH, request)
+        if type(resp) != ErrorResponse:
+            logger.info(f"Fetched {len(resp.bundles)} bundle(s) from {node!r}")
         return resp