koi-net 1.1.0b8__py3-none-any.whl → 1.2.0b2__py3-none-any.whl

koi_net/network/error_handler.py

@@ -1,36 +1,37 @@
-from logging import getLogger
+import structlog
+from koi_net.handshaker import Handshaker
 from koi_net.protocol.errors import ErrorType
 from koi_net.protocol.event import EventType
 from rid_lib.types import KoiNetNode
-from ..processor.interface import ProcessorInterface
-from ..actor import Actor
+from ..processor.kobj_queue import KobjQueue
 
-logger = getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class ErrorHandler:
+    """Handles network errors that may occur during requests."""
     timeout_counter: dict[KoiNetNode, int]
-    processor: ProcessorInterface
-    actor: Actor
+    kobj_queue: KobjQueue
     
     def __init__(
         self, 
-        processor: ProcessorInterface,
-        actor: Actor
+        kobj_queue: KobjQueue,
+        handshaker: Handshaker
     ):
-        self.processor = processor
-        self.actor = actor
+        self.kobj_queue = kobj_queue
+        self.handshaker = handshaker
         self.timeout_counter = {}
         
     def handle_connection_error(self, node: KoiNetNode):
+        """Drops nodes after timing out three times."""
         self.timeout_counter.setdefault(node, 0)
         self.timeout_counter[node] += 1
         
-        logger.debug(f"{node} has timed out {self.timeout_counter[node]} time(s)")
+        log.debug(f"{node} has timed out {self.timeout_counter[node]} time(s)")
         
         if self.timeout_counter[node] > 3:
-            logger.debug(f"Exceeded time out limit, forgetting node")
-            self.processor.handle(rid=node, event_type=EventType.FORGET)
+            log.debug(f"Exceeded time out limit, forgetting node")
+            self.kobj_queue.put_kobj(rid=node, event_type=EventType.FORGET)
             # do something
         
         
@@ -39,11 +40,12 @@ class ErrorHandler:
         error_type: ErrorType, 
         node: KoiNetNode
     ):
-        logger.info(f"Handling protocol error {error_type} for node {node!r}")
+        """Attempts handshake when this node is unknown to target."""
+        log.info(f"Handling protocol error {error_type} for node {node!r}")
         match error_type:
             case ErrorType.UnknownNode:
-                logger.info("Peer doesn't know me, attempting handshake...")
-                self.actor.handshake_with(node)
+                log.info("Peer doesn't know me, attempting handshake...")
+                self.handshaker.handshake_with(node)
                 
             case ErrorType.InvalidKey: ...
             case ErrorType.InvalidSignature: ...

koi_net/network/event_queue.py

@@ -1,199 +1,31 @@
-import logging
+import structlog
 from queue import Queue
-import httpx
-from pydantic import BaseModel
-from rid_lib import RID
-from rid_lib.ext import Cache
+
 from rid_lib.types import KoiNetNode
+from pydantic import BaseModel
 
-from .graph import NetworkGraph
-from .request_handler import NodeNotFoundError, RequestHandler
-from ..protocol.node import NodeProfile, NodeType
-from ..protocol.edge import EdgeProfile, EdgeType
 from ..protocol.event import Event
-from ..identity import NodeIdentity
-from ..config import NodeConfig
-from ..effector import Effector
-
-logger = logging.getLogger(__name__)
 
+log = structlog.stdlib.get_logger()
 
-class EventQueueModel(BaseModel):
-    webhook: dict[KoiNetNode, list[Event]]
-    poll: dict[KoiNetNode, list[Event]]
 
-type EventQueue = dict[RID, Queue[Event]]
+class QueuedEvent(BaseModel):
+    event: Event
+    target: KoiNetNode
 
-class NetworkEventQueue:
-    """A collection of functions and classes to interact with the KOI network."""
+class EventQueue:
+    """Handles out going network event queues."""
+    q: Queue[QueuedEvent]
     
-    config: NodeConfig    
-    identity: NodeIdentity
-    effector: Effector
-    cache: Cache
-    graph: NetworkGraph
-    request_handler: RequestHandler
-    poll_event_queue: EventQueue
-    webhook_event_queue: EventQueue
+    def __init__(self):
+        self.q = Queue()
     
-    def __init__(
-        self, 
-        config: NodeConfig,
-        cache: Cache, 
-        identity: NodeIdentity,
-        effector: Effector,
-        graph: NetworkGraph,
-        request_handler: RequestHandler,
-    ):
-        self.config = config
-        self.identity = identity
-        self.cache = cache
-        self.graph = graph
-        self.request_handler = request_handler
-        self.effector = effector
-        
-        self.poll_event_queue = dict()
-        self.webhook_event_queue = dict()
-    
-    def _load_event_queues(self):
-        """Loads event queues from storage."""
-        try:
-            with open(self.config.koi_net.event_queues_path, "r") as f:
-                queues = EventQueueModel.model_validate_json(f.read())
-            
-            for node in queues.poll.keys():
-                for event in queues.poll[node]:
-                    queue = self.poll_event_queue.setdefault(node, Queue())
-                    queue.put(event)
-            
-            for node in queues.webhook.keys():
-                for event in queues.webhook[node]:
-                    queue = self.webhook_event_queue.setdefault(node, Queue())
-                    queue.put(event)
-                                
-        except FileNotFoundError:
-            return
-        
-    def _save_event_queues(self):
-        """Writes event queues to storage."""
-        events_model = EventQueueModel(
-            poll={
-                node: list(queue.queue) 
-                for node, queue in self.poll_event_queue.items()
-                if not queue.empty()
-            },
-            webhook={
-                node: list(queue.queue) 
-                for node, queue in self.webhook_event_queue.items()
-                if not queue.empty()
-            }
-        )
-        
-        if len(events_model.poll) == 0 and len(events_model.webhook) == 0:
-            return
-        
-        with open(self.config.koi_net.event_queues_path, "w") as f:
-            f.write(events_model.model_dump_json(indent=2))
-    
-    def push_event_to(self, event: Event, node: KoiNetNode, flush=False):
+    def push_event_to(self, event: Event, target: KoiNetNode):
         """Pushes event to queue of specified node.
         
-        Event will be sent to webhook or poll queue depending on the node type and edge type of the specified node. If `flush` is set to `True`, the webhook queued will be flushed after pushing the event.
+        Event will be sent to webhook or poll queue depending on the 
+        node type and edge type of the specified node.
         """
-        logger.debug(f"Pushing event {event.event_type} {event.rid!r} to {node}")
-                        
-        node_bundle = self.effector.deref(node)
-        
-        # if there's an edge from me to the target node, override broadcast type
-        edge_rid = self.graph.get_edge(
-            source=self.identity.rid,
-            target=node
-        )
-        
-        edge_bundle = self.effector.deref(edge_rid) if edge_rid else None
-        
-        if edge_bundle:
-            logger.debug(f"Found edge from me to {node!r}")
-            edge_profile = edge_bundle.validate_contents(EdgeProfile)
-            if edge_profile.edge_type == EdgeType.WEBHOOK:
-                event_queue = self.webhook_event_queue
-            elif edge_profile.edge_type == EdgeType.POLL:
-                event_queue = self.poll_event_queue
-                
-        elif node_bundle:
-            logger.debug(f"Found bundle for {node!r}")
-            node_profile = node_bundle.validate_contents(NodeProfile)
-            if node_profile.node_type == NodeType.FULL:
-                event_queue = self.webhook_event_queue
-            elif node_profile.node_type == NodeType.PARTIAL:
-                event_queue = self.poll_event_queue
-        
-        elif node == self.config.koi_net.first_contact.rid:
-            logger.debug(f"Node {node!r} is my first contact")
-            # first contact node is always a webhook node
-            event_queue = self.webhook_event_queue
-        
-        else:
-            logger.warning(f"Node {node!r} unknown to me")
-            return
         
-        queue = event_queue.setdefault(node, Queue())
-        queue.put(event)
-                
-        if flush and event_queue is self.webhook_event_queue:
-            self.flush_webhook_queue(node)
-            
-    def _flush_queue(self, event_queue: EventQueue, node: KoiNetNode) -> list[Event]:
-        """Flushes a node's queue, returning list of events."""
-        queue = event_queue.get(node)
-        events = list()
-        if queue:
-            while not queue.empty():
-                event = queue.get()
-                logger.debug(f"Dequeued {event.event_type} {event.rid!r}")
-                events.append(event)
-        
-        return events
-    
-    def flush_poll_queue(self, node: KoiNetNode) -> list[Event]:
-        """Flushes a node's poll queue, returning list of events."""
-        logger.debug(f"Flushing poll queue for {node}")
-        return self._flush_queue(self.poll_event_queue, node)
-    
-    def flush_webhook_queue(self, node: KoiNetNode, requeue_on_fail: bool = True):
-        """Flushes a node's webhook queue, and broadcasts events.
-        
-        If node profile is unknown, or node type is not `FULL`, this operation will fail silently. If the remote node cannot be reached, all events will be requeued.
-        """
-        
-        logger.debug(f"Flushing webhook queue for {node}")
-        
-        # node_bundle = self.effector.deref(node)
-        
-        # if not node_bundle:
-        #     logger.warning(f"{node!r} not found")
-        #     return
-        
-        # node_profile = node_bundle.validate_contents(NodeProfile)
-        
-        # if node_profile.node_type != NodeType.FULL:
-        #     logger.warning(f"{node!r} is a partial node!")
-        #     return
-        
-        events = self._flush_queue(self.webhook_event_queue, node)
-        if not events: return
-        
-        logger.debug(f"Broadcasting {len(events)} events")
-        
-        try:  
-            self.request_handler.broadcast_events(node, events=events)
-
-        except NodeNotFoundError:
-            logger.warning("Broadcast failed (node not found)")
-            
-        except httpx.ConnectError:
-            logger.warning("Broadcast failed (couldn't connect)")
-            
-            if requeue_on_fail:
-                for event in events:
-                    self.push_event_to(event, node)
+        self.q.put(QueuedEvent(target=target, event=event))
+    

koi_net/network/graph.py

@@ -1,4 +1,4 @@
-import logging
+import structlog
 from typing import Literal
 import networkx as nx
 from rid_lib import RIDType
@@ -7,7 +7,7 @@ from rid_lib.types import KoiNetEdge, KoiNetNode
 from ..identity import NodeIdentity
 from ..protocol.edge import EdgeProfile, EdgeStatus
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class NetworkGraph:
@@ -24,22 +24,22 @@ class NetworkGraph:
         
     def generate(self):
         """Generates directed graph from cached KOI nodes and edges."""
-        logger.debug("Generating network graph")
+        log.debug("Generating network graph")
         self.dg.clear()
         for rid in self.cache.list_rids():
             if type(rid) == KoiNetNode:                
                 self.dg.add_node(rid)
-                logger.debug(f"Added node {rid!r}")
+                log.debug(f"Added node {rid!r}")
                 
             elif type(rid) == KoiNetEdge:
                 edge_bundle = self.cache.read(rid)
                 if not edge_bundle:
-                    logger.warning(f"Failed to load {rid!r}")
+                    log.warning(f"Failed to load {rid!r}")
                     continue
                 edge_profile = edge_bundle.validate_contents(EdgeProfile)
                 self.dg.add_edge(edge_profile.source, edge_profile.target, rid=rid)
-                logger.debug(f"Added edge {rid!r} ({edge_profile.source} -> {edge_profile.target})")
-        logger.debug("Done")
+                log.debug(f"Added edge {rid!r} ({edge_profile.source} -> {edge_profile.target})")
+        log.debug("Done")
         
     def get_edge(self, source: KoiNetNode, target: KoiNetNode,) -> KoiNetEdge | None:
         """Returns edge RID given the RIDs of a source and target node."""
@@ -56,7 +56,9 @@ class NetworkGraph:
     ) -> list[KoiNetEdge]:
         """Returns edges this node belongs to.
         
-        All edges returned by default, specify `direction` to restrict to incoming or outgoing edges only."""
+        All edges returned by default, specify `direction` to restrict 
+        to incoming or outgoing edges only.
+        """
                         
         edges = []
         if direction != "in" and self.dg.out_edges:
@@ -85,14 +87,17 @@ class NetworkGraph:
     ) -> list[KoiNetNode]:
         """Returns neighboring nodes this node shares an edge with.
         
-        All neighboring nodes returned by default, specify `direction` to restrict to neighbors connected by incoming or outgoing edges only."""
+        All neighboring nodes returned by default, specify `direction` 
+        to restrict to neighbors connected by incoming or outgoing edges
+        only.
+        """
         
         neighbors = set()
         for edge_rid in self.get_edges(direction):
             edge_bundle = self.cache.read(edge_rid)
             
             if not edge_bundle: 
-                logger.warning(f"Failed to find edge {edge_rid!r} in cache")
+                log.warning(f"Failed to find edge {edge_rid!r} in cache")
                 continue
             
             edge_profile = edge_bundle.validate_contents(EdgeProfile)

koi_net/network/poll_event_buffer.py

@@ -0,0 +1,26 @@
+from rid_lib.types import KoiNetNode
+
+from koi_net.protocol.event import Event
+
+
+class PollEventBuffer:
+    buffers: dict[KoiNetNode, list[Event]]
+    
+    def __init__(self):
+        self.buffers = dict()
+        
+    def put(self, node: KoiNetNode, event: Event):
+        event_buf = self.buffers.setdefault(node, [])
+        event_buf.append(event)
+        
+    def flush(self, node: KoiNetNode, limit: int = 0):
+        event_buf = self.buffers.get(node, [])
+        
+        if limit and len(event_buf) > limit:
+            to_return = event_buf[:limit]
+            self.buffers[node] = event_buf[limit:]
+        else:
+            to_return = event_buf.copy()
+            self.buffers[node] = []
+        
+        return to_return

koi_net/network/request_handler.py

@@ -1,8 +1,11 @@
-import logging
+import structlog
 import httpx
 from rid_lib import RID
+from rid_lib.ext import Cache
 from rid_lib.types.koi_net_node import KoiNetNode
 
+from koi_net.protocol.model_map import API_MODEL_MAP
+
 from ..identity import NodeIdentity
 from ..protocol.api_models import (
     RidsPayload,
@@ -27,14 +30,9 @@ from ..protocol.consts import (
 )
 from ..protocol.node import NodeProfile, NodeType
 from ..secure import Secure
-from ..effector import Effector
-
-from typing import TYPE_CHECKING
-if TYPE_CHECKING:
-    from .error_handler import ErrorHandler
-
+from .error_handler import ErrorHandler
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 # Custom error types for request handling
@@ -57,51 +55,50 @@ class UnknownPathError(Exception):
 class RequestHandler:
     """Handles making requests to other KOI nodes."""
     
-    effector: Effector
+    cache: Cache
     identity: NodeIdentity
     secure: Secure
-    error_handler: "ErrorHandler"
+    error_handler: ErrorHandler
     
     def __init__(
         self, 
-        effector: Effector,
+        cache: Cache,
         identity: NodeIdentity,
-        secure: Secure
+        secure: Secure,
+        error_handler: ErrorHandler
     ):
-        self.effector = effector
+        self.cache = cache
         self.identity = identity
         self.secure = secure
-        
-    def set_error_handler(self, error_handler: "ErrorHandler"):
         self.error_handler = error_handler
     
     def get_url(self, node_rid: KoiNetNode) -> str:
-        """Retrieves URL of a node."""
+        """Retrieves URL of a node from its RID."""
         
-        logger.debug(f"Getting URL for {node_rid!r}")
+        log.debug(f"Getting URL for {node_rid!r}")
         node_url = None
         
         if node_rid == self.identity.rid:
             raise SelfRequestError("Don't talk to yourself")
         
-        node_bundle = self.effector.deref(node_rid)
+        node_bundle = self.cache.read(node_rid)
                 
         if node_bundle:
             node_profile = node_bundle.validate_contents(NodeProfile)
-            logger.debug(f"Found node profile: {node_profile}")
+            log.debug(f"Found node profile: {node_profile}")
             if node_profile.node_type != NodeType.FULL:
                 raise PartialNodeQueryError("Can't query partial node")
             node_url = node_profile.base_url
         
         else:
             if node_rid == self.identity.config.koi_net.first_contact.rid:
-                logger.debug("Found URL of first contact")
+                log.debug("Found URL of first contact")
                 node_url = self.identity.config.koi_net.first_contact.url
         
         if not node_url:
             raise NodeNotFoundError("Node not found")
         
-        logger.debug(f"Resolved {node_rid!r} to {node_url}")
+        log.debug(f"Resolved {node_rid!r} to {node_url}")
         return node_url
     
     def make_request(
@@ -110,8 +107,9 @@ class RequestHandler:
         path: str, 
         request: RequestModels,
     ) -> ResponseModels | None:
+        """Makes a request to a node."""
         url = self.get_url(node) + path
-        logger.info(f"Making request to {url}")
+        log.info(f"Making request to {url}")
     
         signed_envelope = self.secure.create_envelope(
             payload=request,
@@ -119,9 +117,12 @@ class RequestHandler:
         )
         
         try:
-            result = httpx.post(url, data=signed_envelope.model_dump_json(exclude_none=True))
+            result = httpx.post(
+                url, 
+                data=signed_envelope.model_dump_json(exclude_none=True)
+            )
         except httpx.ConnectError as err:
-            logger.debug("Failed to connect")
+            log.debug("Failed to connect")
             self.error_handler.handle_connection_error(node)
             raise err
         
@@ -130,20 +131,11 @@ class RequestHandler:
             self.error_handler.handle_protocol_error(resp.error, node)
             return resp
         
-        if path == BROADCAST_EVENTS_PATH:
-            return None
-        elif path == POLL_EVENTS_PATH:
-            EnvelopeModel = SignedEnvelope[EventsPayload]
-        elif path == FETCH_RIDS_PATH:
-            EnvelopeModel = SignedEnvelope[RidsPayload]
-        elif path == FETCH_MANIFESTS_PATH:
-            EnvelopeModel = SignedEnvelope[ManifestsPayload]
-        elif path == FETCH_BUNDLES_PATH:
-            EnvelopeModel = SignedEnvelope[BundlesPayload]
-        else:
-            raise UnknownPathError(f"Unknown path '{path}'")
+        resp_env_model = API_MODEL_MAP[path].response_envelope
+        if resp_env_model is None:
+            return
         
-        resp_envelope = EnvelopeModel.model_validate_json(result.text)
+        resp_envelope = resp_env_model.model_validate_json(result.text)
         self.secure.validate_envelope(resp_envelope)
         
         return resp_envelope.payload
@@ -154,10 +146,13 @@ class RequestHandler:
         req: EventsPayload | None = None,
         **kwargs
     ) -> None:
-        """See protocol.api_models.EventsPayload for available kwargs."""
+        """Broadcasts events to a node.
+        
+        Pass `EventsPayload` object, or see `protocol.api_models.EventsPayload` for available kwargs.
+        """
         request = req or EventsPayload.model_validate(kwargs)
         self.make_request(node, BROADCAST_EVENTS_PATH, request)
-        logger.info(f"Broadcasted {len(request.events)} event(s) to {node!r}")
+        log.info(f"Broadcasted {len(request.events)} event(s) to {node!r}")
         
     def poll_events(
         self, 
@@ -165,11 +160,14 @@ class RequestHandler:
         req: PollEvents | None = None,
         **kwargs
     ) -> EventsPayload:
-        """See protocol.api_models.PollEvents for available kwargs."""
+        """Polls events from a node.
+        
+        Pass `PollEvents` object as `req` or fields as kwargs.
+        """
         request = req or PollEvents.model_validate(kwargs)
         resp = self.make_request(node, POLL_EVENTS_PATH, request)
         if type(resp) != ErrorResponse:
-            logger.info(f"Polled {len(resp.events)} events from {node!r}")
+            log.info(f"Polled {len(resp.events)} events from {node!r}")
         return resp
         
     def fetch_rids(
@@ -178,11 +176,14 @@ class RequestHandler:
         req: FetchRids | None = None,
         **kwargs
     ) -> RidsPayload:
-        """See protocol.api_models.FetchRids for available kwargs."""
+        """Fetches RIDs from a node.
+        
+        Pass `FetchRids` object as `req` or fields as kwargs.
+        """
         request = req or FetchRids.model_validate(kwargs)
         resp = self.make_request(node, FETCH_RIDS_PATH, request)
         if type(resp) != ErrorResponse:
-            logger.info(f"Fetched {len(resp.rids)} RID(s) from {node!r}")
+            log.info(f"Fetched {len(resp.rids)} RID(s) from {node!r}")
         return resp
                 
     def fetch_manifests(
@@ -191,11 +192,14 @@ class RequestHandler:
         req: FetchManifests | None = None,
         **kwargs
     ) -> ManifestsPayload:
-        """See protocol.api_models.FetchManifests for available kwargs."""
+        """Fetches manifests from a node.
+        
+        Pass `FetchManifests` object as `req` or fields as kwargs.
+        """
         request = req or FetchManifests.model_validate(kwargs)
         resp = self.make_request(node, FETCH_MANIFESTS_PATH, request)
         if type(resp) != ErrorResponse:
-            logger.info(f"Fetched {len(resp.manifests)} manifest(s) from {node!r}")
+            log.info(f"Fetched {len(resp.manifests)} manifest(s) from {node!r}")
         return resp
                 
     def fetch_bundles(
@@ -204,9 +208,12 @@ class RequestHandler:
         req: FetchBundles | None = None,
         **kwargs
     ) -> BundlesPayload:
-        """See protocol.api_models.FetchBundles for available kwargs."""
+        """Fetches bundles from a node.
+        
+        Pass `FetchBundles` object as `req` or fields as kwargs.
+        """
         request = req or FetchBundles.model_validate(kwargs)
         resp = self.make_request(node, FETCH_BUNDLES_PATH, request)
         if type(resp) != ErrorResponse:
-            logger.info(f"Fetched {len(resp.bundles)} bundle(s) from {node!r}")
+            log.info(f"Fetched {len(resp.bundles)} bundle(s) from {node!r}")
         return resp