koi-net 1.0.0b19__py3-none-any.whl → 1.1.0__py3-none-any.whl

koi_net/identity.py

@@ -1,33 +1,19 @@
 import logging
-from rid_lib.ext.bundle import Bundle
-from rid_lib.ext.cache import Cache
 from rid_lib.types.koi_net_node import KoiNetNode
-
 from .config import NodeConfig
 from .protocol.node import NodeProfile
 
+
 logger = logging.getLogger(__name__)
 
     
 class NodeIdentity:
-    """Represents a node's identity (RID, profile, bundle)."""
+    """Represents a node's identity (RID, profile)."""
     
     config: NodeConfig    
-    cache: Cache
     
-    def __init__(
-        self,
-        config: NodeConfig,
-        cache: Cache
-    ):
-        """Initializes node identity from a name and profile.
-        
-        Attempts to read identity from storage. If it doesn't already exist, a new RID is generated from the provided name, and that RID and profile are written to storage. Changes to the name or profile will update the stored identity.
-        
-        WARNING: If the name is changed, the RID will be overwritten which will have consequences for the rest of the network.
-        """
+    def __init__(self, config: NodeConfig):
         self.config = config
-        self.cache = cache
         
     @property
     def rid(self) -> KoiNetNode:
@@ -35,8 +21,4 @@ class NodeIdentity:
     
     @property
     def profile(self) -> NodeProfile:
-        return self.config.koi_net.node_profile 
-    
-    @property
-    def bundle(self) -> Bundle:
-        return self.cache.read(self.rid)
+        return self.config.koi_net.node_profile

koi_net/lifecycle.py

@@ -0,0 +1,104 @@
+import logging
+from contextlib import contextmanager, asynccontextmanager
+
+from rid_lib.types import KoiNetNode
+
+from .actor import Actor
+from .effector import Effector
+from .config import NodeConfig
+from .processor.interface import ProcessorInterface
+from .network.graph import NetworkGraph
+from .identity import NodeIdentity
+
+logger = logging.getLogger(__name__)
+
+
+class NodeLifecycle:
+    config: NodeConfig
+    graph: NetworkGraph
+    processor: ProcessorInterface
+    effector: Effector
+    actor: Actor
+    
+    def __init__(
+        self,
+        config: NodeConfig,
+        identity: NodeIdentity,
+        graph: NetworkGraph,
+        processor: ProcessorInterface,
+        effector: Effector,
+        actor: Actor,
+        use_kobj_processor_thread: bool
+    ):
+        self.config = config
+        self.identity = identity
+        self.graph = graph
+        self.processor = processor
+        self.effector = effector
+        self.actor = actor
+        self.use_kobj_processor_thread = use_kobj_processor_thread
+        
+    @contextmanager
+    def run(self):
+        try:
+            logger.info("Starting node lifecycle...")
+            self.start()
+            yield
+        except KeyboardInterrupt:
+            logger.info("Keyboard interrupt!")
+        finally:
+            logger.info("Stopping node lifecycle...")
+            self.stop()
+
+    @asynccontextmanager
+    async def async_run(self):
+        try:
+            logger.info("Starting async node lifecycle...")
+            self.start()
+            yield
+        except KeyboardInterrupt:
+            logger.info("Keyboard interrupt!")
+        finally:
+            logger.info("Stopping async node lifecycle...")
+            self.stop()
+    
+    def start(self):
+        """Starts a node, call this method first.
+        
+        Starts the processor thread (if enabled). Loads event queues into memory. Generates network graph from nodes and edges in cache. Processes any state changes of node bundle. Initiates handshake with first contact (if provided) if node doesn't have any neighbors.
+        """
+        if self.use_kobj_processor_thread:
+            logger.info("Starting processor worker thread")
+            self.processor.worker_thread.start()
+        
+        self.graph.generate()
+        
+        # refresh to reflect changes (if any) in config.yaml
+        self.effector.deref(self.identity.rid, refresh_cache=True)
+        
+        logger.debug("Waiting for kobj queue to empty")
+        if self.use_kobj_processor_thread:
+            self.processor.kobj_queue.join()
+        else:
+            self.processor.flush_kobj_queue()
+        logger.debug("Done")
+    
+        if not self.graph.get_neighbors() and self.config.koi_net.first_contact.rid:
+            logger.debug(f"I don't have any neighbors, reaching out to first contact {self.config.koi_net.first_contact.rid!r}")
+            
+            self.actor.handshake_with(self.config.koi_net.first_contact.rid)
+        
+        for coordinator in self.actor.identify_coordinators():
+            self.actor.catch_up_with(coordinator, rid_types=[KoiNetNode])
+        
+
+    def stop(self):
+        """Stops a node, call this method last.
+        
+        Finishes processing knowledge object queue. Saves event queues to storage.
+        """        
+        if self.use_kobj_processor_thread:
+            logger.info(f"Waiting for kobj queue to empty ({self.processor.kobj_queue.unfinished_tasks} tasks remaining)")
+            self.processor.kobj_queue.join()
+        else:
+            self.processor.flush_kobj_queue()

koi_net/network/__init__.py

@@ -1 +0,0 @@
-from .interface import NetworkInterface

koi_net/network/error_handler.py

@@ -0,0 +1,50 @@
+from logging import getLogger
+from koi_net.protocol.errors import ErrorType
+from koi_net.protocol.event import EventType
+from rid_lib.types import KoiNetNode
+from ..processor.interface import ProcessorInterface
+from ..actor import Actor
+
+logger = getLogger(__name__)
+
+
+class ErrorHandler:
+    timeout_counter: dict[KoiNetNode, int]
+    processor: ProcessorInterface
+    actor: Actor
+    
+    def __init__(
+        self, 
+        processor: ProcessorInterface,
+        actor: Actor
+    ):
+        self.processor = processor
+        self.actor = actor
+        self.timeout_counter = {}
+        
+    def handle_connection_error(self, node: KoiNetNode):
+        self.timeout_counter.setdefault(node, 0)
+        self.timeout_counter[node] += 1
+        
+        logger.debug(f"{node} has timed out {self.timeout_counter[node]} time(s)")
+        
+        if self.timeout_counter[node] > 3:
+            logger.debug(f"Exceeded time out limit, forgetting node")
+            self.processor.handle(rid=node, event_type=EventType.FORGET)
+            # do something
+        
+        
+    def handle_protocol_error(
+        self, 
+        error_type: ErrorType, 
+        node: KoiNetNode
+    ):
+        logger.info(f"Handling protocol error {error_type} for node {node!r}")
+        match error_type:
+            case ErrorType.UnknownNode:
+                logger.info("Peer doesn't know me, attempting handshake...")
+                self.actor.handshake_with(node)
+                
+            case ErrorType.InvalidKey: ...
+            case ErrorType.InvalidSignature: ...
+            case ErrorType.InvalidTarget: ...

koi_net/network/event_queue.py

@@ -0,0 +1,199 @@
+import logging
+from queue import Queue
+import httpx
+from pydantic import BaseModel
+from rid_lib import RID
+from rid_lib.ext import Cache
+from rid_lib.types import KoiNetNode
+
+from .graph import NetworkGraph
+from .request_handler import NodeNotFoundError, RequestHandler
+from ..protocol.node import NodeProfile, NodeType
+from ..protocol.edge import EdgeProfile, EdgeType
+from ..protocol.event import Event
+from ..identity import NodeIdentity
+from ..config import NodeConfig
+from ..effector import Effector
+
+logger = logging.getLogger(__name__)
+
+
+class EventQueueModel(BaseModel):
+    webhook: dict[KoiNetNode, list[Event]]
+    poll: dict[KoiNetNode, list[Event]]
+
+type EventQueue = dict[RID, Queue[Event]]
+
+class NetworkEventQueue:
+    """A collection of functions and classes to interact with the KOI network."""
+    
+    config: NodeConfig    
+    identity: NodeIdentity
+    effector: Effector
+    cache: Cache
+    graph: NetworkGraph
+    request_handler: RequestHandler
+    poll_event_queue: EventQueue
+    webhook_event_queue: EventQueue
+    
+    def __init__(
+        self, 
+        config: NodeConfig,
+        cache: Cache, 
+        identity: NodeIdentity,
+        effector: Effector,
+        graph: NetworkGraph,
+        request_handler: RequestHandler,
+    ):
+        self.config = config
+        self.identity = identity
+        self.cache = cache
+        self.graph = graph
+        self.request_handler = request_handler
+        self.effector = effector
+        
+        self.poll_event_queue = dict()
+        self.webhook_event_queue = dict()
+    
+    def _load_event_queues(self):
+        """Loads event queues from storage."""
+        try:
+            with open(self.config.koi_net.event_queues_path, "r") as f:
+                queues = EventQueueModel.model_validate_json(f.read())
+            
+            for node in queues.poll.keys():
+                for event in queues.poll[node]:
+                    queue = self.poll_event_queue.setdefault(node, Queue())
+                    queue.put(event)
+            
+            for node in queues.webhook.keys():
+                for event in queues.webhook[node]:
+                    queue = self.webhook_event_queue.setdefault(node, Queue())
+                    queue.put(event)
+                                
+        except FileNotFoundError:
+            return
+        
+    def _save_event_queues(self):
+        """Writes event queues to storage."""
+        events_model = EventQueueModel(
+            poll={
+                node: list(queue.queue) 
+                for node, queue in self.poll_event_queue.items()
+                if not queue.empty()
+            },
+            webhook={
+                node: list(queue.queue) 
+                for node, queue in self.webhook_event_queue.items()
+                if not queue.empty()
+            }
+        )
+        
+        if len(events_model.poll) == 0 and len(events_model.webhook) == 0:
+            return
+        
+        with open(self.config.koi_net.event_queues_path, "w") as f:
+            f.write(events_model.model_dump_json(indent=2))
+    
+    def push_event_to(self, event: Event, node: KoiNetNode, flush=False):
+        """Pushes event to queue of specified node.
+        
+        Event will be sent to webhook or poll queue depending on the node type and edge type of the specified node. If `flush` is set to `True`, the webhook queued will be flushed after pushing the event.
+        """
+        logger.debug(f"Pushing event {event.event_type} {event.rid!r} to {node}")
+                        
+        node_bundle = self.effector.deref(node)
+        
+        # if there's an edge from me to the target node, override broadcast type
+        edge_rid = self.graph.get_edge(
+            source=self.identity.rid,
+            target=node
+        )
+        
+        edge_bundle = self.effector.deref(edge_rid) if edge_rid else None
+        
+        if edge_bundle:
+            logger.debug(f"Found edge from me to {node!r}")
+            edge_profile = edge_bundle.validate_contents(EdgeProfile)
+            if edge_profile.edge_type == EdgeType.WEBHOOK:
+                event_queue = self.webhook_event_queue
+            elif edge_profile.edge_type == EdgeType.POLL:
+                event_queue = self.poll_event_queue
+                
+        elif node_bundle:
+            logger.debug(f"Found bundle for {node!r}")
+            node_profile = node_bundle.validate_contents(NodeProfile)
+            if node_profile.node_type == NodeType.FULL:
+                event_queue = self.webhook_event_queue
+            elif node_profile.node_type == NodeType.PARTIAL:
+                event_queue = self.poll_event_queue
+        
+        elif node == self.config.koi_net.first_contact.rid:
+            logger.debug(f"Node {node!r} is my first contact")
+            # first contact node is always a webhook node
+            event_queue = self.webhook_event_queue
+        
+        else:
+            logger.warning(f"Node {node!r} unknown to me")
+            return
+        
+        queue = event_queue.setdefault(node, Queue())
+        queue.put(event)
+                
+        if flush and event_queue is self.webhook_event_queue:
+            self.flush_webhook_queue(node)
+            
+    def _flush_queue(self, event_queue: EventQueue, node: KoiNetNode) -> list[Event]:
+        """Flushes a node's queue, returning list of events."""
+        queue = event_queue.get(node)
+        events = list()
+        if queue:
+            while not queue.empty():
+                event = queue.get()
+                logger.debug(f"Dequeued {event.event_type} {event.rid!r}")
+                events.append(event)
+        
+        return events
+    
+    def flush_poll_queue(self, node: KoiNetNode) -> list[Event]:
+        """Flushes a node's poll queue, returning list of events."""
+        logger.debug(f"Flushing poll queue for {node}")
+        return self._flush_queue(self.poll_event_queue, node)
+    
+    def flush_webhook_queue(self, node: KoiNetNode, requeue_on_fail: bool = True):
+        """Flushes a node's webhook queue, and broadcasts events.
+        
+        If node profile is unknown, or node type is not `FULL`, this operation will fail silently. If the remote node cannot be reached, all events will be requeued.
+        """
+        
+        logger.debug(f"Flushing webhook queue for {node}")
+        
+        # node_bundle = self.effector.deref(node)
+        
+        # if not node_bundle:
+        #     logger.warning(f"{node!r} not found")
+        #     return
+        
+        # node_profile = node_bundle.validate_contents(NodeProfile)
+        
+        # if node_profile.node_type != NodeType.FULL:
+        #     logger.warning(f"{node!r} is a partial node!")
+        #     return
+        
+        events = self._flush_queue(self.webhook_event_queue, node)
+        if not events: return
+        
+        logger.debug(f"Broadcasting {len(events)} events")
+        
+        try:  
+            self.request_handler.broadcast_events(node, events=events)
+
+        except NodeNotFoundError:
+            logger.warning("Broadcast failed (node not found)")
+            
+        except httpx.ConnectError:
+            logger.warning("Broadcast failed (couldn't connect)")
+            
+            if requeue_on_fail:
+                for event in events:
+                    self.push_event_to(event, node)

koi_net/network/graph.py

@@ -6,7 +6,6 @@ from rid_lib.ext import Cache
 from rid_lib.types import KoiNetEdge, KoiNetNode
 from ..identity import NodeIdentity
 from ..protocol.edge import EdgeProfile, EdgeStatus
-from ..protocol.node import NodeProfile
 
 logger = logging.getLogger(__name__)
 
@@ -30,43 +29,27 @@ class NetworkGraph:
         for rid in self.cache.list_rids():
             if type(rid) == KoiNetNode:                
                 self.dg.add_node(rid)
-                logger.debug(f"Added node {rid}")
+                logger.debug(f"Added node {rid!r}")
                 
             elif type(rid) == KoiNetEdge:
-                edge_profile = self.get_edge_profile(rid)
-                if not edge_profile:
+                edge_bundle = self.cache.read(rid)
+                if not edge_bundle:
                     logger.warning(f"Failed to load {rid!r}")
                     continue
+                edge_profile = edge_bundle.validate_contents(EdgeProfile)
                 self.dg.add_edge(edge_profile.source, edge_profile.target, rid=rid)
-                logger.debug(f"Added edge {rid} ({edge_profile.source} -> {edge_profile.target})")
+                logger.debug(f"Added edge {rid!r} ({edge_profile.source} -> {edge_profile.target})")
         logger.debug("Done")
         
-    def get_node_profile(self, rid: KoiNetNode) -> NodeProfile | None:
-        """Returns node profile given its RID."""
-        bundle = self.cache.read(rid)
-        if bundle:
-            return bundle.validate_contents(NodeProfile)
-        
-    def get_edge_profile(
-        self, 
-        rid: KoiNetEdge | None = None,
-        source: KoiNetNode | None = None, 
-        target: KoiNetNode | None = None,
-    ) -> EdgeProfile | None:
-        """Returns edge profile given its RID, or source and target node RIDs."""
-        if source and target:
-            if (source, target) not in self.dg.edges: return
+    def get_edge(self, source: KoiNetNode, target: KoiNetNode,) -> KoiNetEdge | None:
+        """Returns edge RID given the RIDs of a source and target node."""
+        if (source, target) in self.dg.edges:
             edge_data = self.dg.get_edge_data(source, target)
-            if not edge_data: return
-            rid = edge_data.get("rid")
-            if not rid: return
-        elif not rid:
-            raise ValueError("Either 'rid' or 'source' and 'target' must be provided")
-        
-        bundle = self.cache.read(rid)
-        if bundle:
-            return bundle.validate_contents(EdgeProfile)
-        
+            if edge_data:
+                return edge_data.get("rid")
+
+        return None
+
     def get_edges(
         self,
         direction: Literal["in", "out"] | None = None,
@@ -74,15 +57,15 @@ class NetworkGraph:
         """Returns edges this node belongs to.
         
         All edges returned by default, specify `direction` to restrict to incoming or outgoing edges only."""
-                
+                        
         edges = []
         if direction != "in" and self.dg.out_edges:
             out_edges = self.dg.out_edges(self.identity.rid)
-            edges.extend([e for e in out_edges])
+            edges.extend(out_edges)
                 
         if direction != "out" and self.dg.in_edges:
             in_edges = self.dg.in_edges(self.identity.rid)
-            edges.extend([e for e in in_edges])
+            edges.extend(in_edges)
                     
         edge_rids = []
         for edge in edges:
@@ -104,13 +87,15 @@ class NetworkGraph:
         
         All neighboring nodes returned by default, specify `direction` to restrict to neighbors connected by incoming or outgoing edges only."""
         
-        neighbors = []
+        neighbors = set()
         for edge_rid in self.get_edges(direction):
-            edge_profile = self.get_edge_profile(edge_rid)
+            edge_bundle = self.cache.read(edge_rid)
             
-            if not edge_profile: 
+            if not edge_bundle: 
                 logger.warning(f"Failed to find edge {edge_rid!r} in cache")
                 continue
+            
+            edge_profile = edge_bundle.validate_contents(EdgeProfile)
                         
             if status and edge_profile.status != status:
                 continue
@@ -119,9 +104,9 @@ class NetworkGraph:
                 continue
             
             if edge_profile.target == self.identity.rid:
-                neighbors.append(edge_profile.source)
+                neighbors.add(edge_profile.source)
             elif edge_profile.source == self.identity.rid:
-                neighbors.append(edge_profile.target)
+                neighbors.add(edge_profile.target)
                 
         return list(neighbors)