koi-net 1.1.0b8__py3-none-any.whl → 1.2.0b1__py3-none-any.whl

koi_net/lifecycle.py

@@ -1,12 +1,18 @@
 import logging
 from contextlib import contextmanager, asynccontextmanager
 
+from rid_lib.ext import Bundle, Cache
 from rid_lib.types import KoiNetNode
 
-from .actor import Actor
-from .effector import Effector
+from koi_net.behaviors import Behaviors
+from koi_net.handshaker import Handshaker
+from koi_net.kobj_worker import KnowledgeProcessingWorker
+from koi_net.models import END
+from koi_net.network.event_queue import EventQueue
+from koi_net.processor.event_worker import EventProcessingWorker
+
 from .config import NodeConfig
-from .processor.interface import ProcessorInterface
+from .processor.kobj_queue import KobjQueue
 from .network.graph import NetworkGraph
 from .identity import NodeIdentity
 
@@ -14,32 +20,46 @@ logger = logging.getLogger(__name__)
 
 
 class NodeLifecycle:
+    """Manages node startup and shutdown processes."""
+    
     config: NodeConfig
+    identity: NodeIdentity
     graph: NetworkGraph
-    processor: ProcessorInterface
-    effector: Effector
-    actor: Actor
+    kobj_queue: KobjQueue
+    kobj_worker: KnowledgeProcessingWorker
+    event_queue: EventQueue
+    event_worker: EventProcessingWorker
+    cache: Cache
+    handshaker: Handshaker
+    behaviors: Behaviors
     
     def __init__(
         self,
         config: NodeConfig,
         identity: NodeIdentity,
         graph: NetworkGraph,
-        processor: ProcessorInterface,
-        effector: Effector,
-        actor: Actor,
-        use_kobj_processor_thread: bool
+        kobj_queue: KobjQueue,
+        kobj_worker: KnowledgeProcessingWorker,
+        event_queue: EventQueue,
+        event_worker: EventProcessingWorker,
+        cache: Cache,
+        handshaker: Handshaker,
+        behaviors: Behaviors
     ):
         self.config = config
         self.identity = identity
         self.graph = graph
-        self.processor = processor
-        self.effector = effector
-        self.actor = actor
-        self.use_kobj_processor_thread = use_kobj_processor_thread
+        self.kobj_queue = kobj_queue
+        self.kobj_worker = kobj_worker
+        self.event_queue = event_queue
+        self.event_worker = event_worker
+        self.cache = cache
+        self.handshaker = handshaker
+        self.behaviors = behaviors
         
     @contextmanager
     def run(self):
+        """Synchronous context manager for node startup and shutdown."""
         try:
             logger.info("Starting node lifecycle...")
             self.start()
@@ -52,6 +72,7 @@ class NodeLifecycle:
 
     @asynccontextmanager
     async def async_run(self):
+        """Asynchronous context manager for node startup and shutdown."""
         try:
             logger.info("Starting async node lifecycle...")
             self.start()
@@ -63,42 +84,50 @@ class NodeLifecycle:
             self.stop()
     
     def start(self):
-        """Starts a node, call this method first.
+        """Starts a node.
         
-        Starts the processor thread (if enabled). Loads event queues into memory. Generates network graph from nodes and edges in cache. Processes any state changes of node bundle. Initiates handshake with first contact (if provided) if node doesn't have any neighbors.
+        Starts the processor thread (if enabled). Generates network 
+        graph from nodes and edges in cache. Processes any state changes 
+        of node bundle. Initiates handshake with first contact if node 
+        doesn't have any neighbors. Catches up with coordinator state.
         """
-        if self.use_kobj_processor_thread:
-            logger.info("Starting processor worker thread")
-            self.processor.worker_thread.start()
+        logger.info("Starting processor worker thread")
         
+        self.kobj_worker.thread.start()
+        self.event_worker.thread.start()
         self.graph.generate()
         
         # refresh to reflect changes (if any) in config.yaml
-        self.effector.deref(self.identity.rid, refresh_cache=True)
+        
+        self.kobj_queue.put_kobj(bundle=Bundle.generate(
+            rid=self.identity.rid,
+            contents=self.identity.profile.model_dump()
+        ))
         
         logger.debug("Waiting for kobj queue to empty")
-        if self.use_kobj_processor_thread:
-            self.processor.kobj_queue.join()
-        else:
-            self.processor.flush_kobj_queue()
-        logger.debug("Done")
-    
-        if not self.graph.get_neighbors() and self.config.koi_net.first_contact.rid:
-            logger.debug(f"I don't have any neighbors, reaching out to first contact {self.config.koi_net.first_contact.rid!r}")
+        self.kobj_queue.q.join()
+        
+        # TODO: FACTOR OUT BEHAVIOR
+        
+        coordinators = self.graph.get_neighbors(direction="in", allowed_type=KoiNetNode)
+        
+        if len(coordinators) == 0 and self.config.koi_net.first_contact.rid:
+            logger.debug(f"I don't have any edges with coordinators, reaching out to first contact {self.config.koi_net.first_contact.rid!r}")
             
-            self.actor.handshake_with(self.config.koi_net.first_contact.rid)
+            self.handshaker.handshake_with(self.config.koi_net.first_contact.rid)
+        
         
-        for coordinator in self.actor.identify_coordinators():
-            self.actor.catch_up_with(coordinator, rid_types=[KoiNetNode])
+        
+        for coordinator in self.behaviors.identify_coordinators():
+            self.behaviors.catch_up_with(coordinator, rid_types=[KoiNetNode])
         
 
     def stop(self):
-        """Stops a node, call this method last.
+        """Stops a node.
         
-        Finishes processing knowledge object queue. Saves event queues to storage.
+        Finishes processing knowledge object queue.
         """        
-        if self.use_kobj_processor_thread:
-            logger.info(f"Waiting for kobj queue to empty ({self.processor.kobj_queue.unfinished_tasks} tasks remaining)")
-            self.processor.kobj_queue.join()
-        else:
-            self.processor.flush_kobj_queue()
+        logger.info(f"Waiting for kobj queue to empty ({self.kobj_queue.q.unfinished_tasks} tasks remaining)")
+        
+        self.kobj_queue.q.put(END)
+        self.event_queue.q.put(END)

koi_net/models.py

@@ -0,0 +1,14 @@
+from pydantic import BaseModel
+from rid_lib.types import KoiNetNode
+from koi_net.protocol.event import Event
+
+class End:
+    """Class for a sentinel value by knowledge handlers."""
+    pass
+
+END = End()
+
+
+class QueuedEvent(BaseModel):
+    event: Event
+    target: KoiNetNode

koi_net/network/error_handler.py

@@ -1,28 +1,29 @@
 from logging import getLogger
+from koi_net.handshaker import Handshaker
 from koi_net.protocol.errors import ErrorType
 from koi_net.protocol.event import EventType
 from rid_lib.types import KoiNetNode
-from ..processor.interface import ProcessorInterface
-from ..actor import Actor
+from ..processor.kobj_queue import KobjQueue
 
 logger = getLogger(__name__)
 
 
 class ErrorHandler:
+    """Handles network errors that may occur during requests."""
     timeout_counter: dict[KoiNetNode, int]
-    processor: ProcessorInterface
-    actor: Actor
+    kobj_queue: KobjQueue
     
     def __init__(
         self, 
-        processor: ProcessorInterface,
-        actor: Actor
+        kobj_queue: KobjQueue,
+        handshaker: Handshaker
     ):
-        self.processor = processor
-        self.actor = actor
+        self.kobj_queue = kobj_queue
+        self.handshaker = handshaker
         self.timeout_counter = {}
         
     def handle_connection_error(self, node: KoiNetNode):
+        """Drops nodes after timing out three times."""
         self.timeout_counter.setdefault(node, 0)
         self.timeout_counter[node] += 1
         
@@ -30,7 +31,7 @@ class ErrorHandler:
         
         if self.timeout_counter[node] > 3:
             logger.debug(f"Exceeded time out limit, forgetting node")
-            self.processor.handle(rid=node, event_type=EventType.FORGET)
+            self.kobj_queue.put_kobj(rid=node, event_type=EventType.FORGET)
             # do something
         
         
@@ -39,11 +40,12 @@ class ErrorHandler:
         error_type: ErrorType, 
         node: KoiNetNode
     ):
+        """Attempts handshake when this node is unknown to target."""
         logger.info(f"Handling protocol error {error_type} for node {node!r}")
         match error_type:
             case ErrorType.UnknownNode:
                 logger.info("Peer doesn't know me, attempting handshake...")
-                self.actor.handshake_with(node)
+                self.handshaker.handshake_with(node)
                 
             case ErrorType.InvalidKey: ...
             case ErrorType.InvalidSignature: ...

koi_net/network/event_queue.py

@@ -1,199 +1,29 @@
 import logging
 from queue import Queue
-import httpx
-from pydantic import BaseModel
-from rid_lib import RID
-from rid_lib.ext import Cache
+
 from rid_lib.types import KoiNetNode
 
-from .graph import NetworkGraph
-from .request_handler import NodeNotFoundError, RequestHandler
-from ..protocol.node import NodeProfile, NodeType
-from ..protocol.edge import EdgeProfile, EdgeType
+from ..models import QueuedEvent
 from ..protocol.event import Event
-from ..identity import NodeIdentity
-from ..config import NodeConfig
-from ..effector import Effector
 
 logger = logging.getLogger(__name__)
 
 
-class EventQueueModel(BaseModel):
-    webhook: dict[KoiNetNode, list[Event]]
-    poll: dict[KoiNetNode, list[Event]]
-
-type EventQueue = dict[RID, Queue[Event]]
-
-class NetworkEventQueue:
-    """A collection of functions and classes to interact with the KOI network."""
-    
-    config: NodeConfig    
-    identity: NodeIdentity
-    effector: Effector
-    cache: Cache
-    graph: NetworkGraph
-    request_handler: RequestHandler
-    poll_event_queue: EventQueue
-    webhook_event_queue: EventQueue
+class EventQueue:
+    """Handles out going network event queues."""
+    q: Queue[QueuedEvent]
     
-    def __init__(
-        self, 
-        config: NodeConfig,
-        cache: Cache, 
-        identity: NodeIdentity,
-        effector: Effector,
-        graph: NetworkGraph,
-        request_handler: RequestHandler,
-    ):
-        self.config = config
-        self.identity = identity
-        self.cache = cache
-        self.graph = graph
-        self.request_handler = request_handler
-        self.effector = effector
-        
-        self.poll_event_queue = dict()
-        self.webhook_event_queue = dict()
-    
-    def _load_event_queues(self):
-        """Loads event queues from storage."""
-        try:
-            with open(self.config.koi_net.event_queues_path, "r") as f:
-                queues = EventQueueModel.model_validate_json(f.read())
-            
-            for node in queues.poll.keys():
-                for event in queues.poll[node]:
-                    queue = self.poll_event_queue.setdefault(node, Queue())
-                    queue.put(event)
-            
-            for node in queues.webhook.keys():
-                for event in queues.webhook[node]:
-                    queue = self.webhook_event_queue.setdefault(node, Queue())
-                    queue.put(event)
-                                
-        except FileNotFoundError:
-            return
-        
-    def _save_event_queues(self):
-        """Writes event queues to storage."""
-        events_model = EventQueueModel(
-            poll={
-                node: list(queue.queue) 
-                for node, queue in self.poll_event_queue.items()
-                if not queue.empty()
-            },
-            webhook={
-                node: list(queue.queue) 
-                for node, queue in self.webhook_event_queue.items()
-                if not queue.empty()
-            }
-        )
-        
-        if len(events_model.poll) == 0 and len(events_model.webhook) == 0:
-            return
-        
-        with open(self.config.koi_net.event_queues_path, "w") as f:
-            f.write(events_model.model_dump_json(indent=2))
+    def __init__(self):
+        self.q = Queue()
     
-    def push_event_to(self, event: Event, node: KoiNetNode, flush=False):
+    def push_event_to(self, event: Event, target: KoiNetNode):
         """Pushes event to queue of specified node.
         
-        Event will be sent to webhook or poll queue depending on the node type and edge type of the specified node. If `flush` is set to `True`, the webhook queued will be flushed after pushing the event.
-        """
-        logger.debug(f"Pushing event {event.event_type} {event.rid!r} to {node}")
-                        
-        node_bundle = self.effector.deref(node)
-        
-        # if there's an edge from me to the target node, override broadcast type
-        edge_rid = self.graph.get_edge(
-            source=self.identity.rid,
-            target=node
-        )
-        
-        edge_bundle = self.effector.deref(edge_rid) if edge_rid else None
-        
-        if edge_bundle:
-            logger.debug(f"Found edge from me to {node!r}")
-            edge_profile = edge_bundle.validate_contents(EdgeProfile)
-            if edge_profile.edge_type == EdgeType.WEBHOOK:
-                event_queue = self.webhook_event_queue
-            elif edge_profile.edge_type == EdgeType.POLL:
-                event_queue = self.poll_event_queue
-                
-        elif node_bundle:
-            logger.debug(f"Found bundle for {node!r}")
-            node_profile = node_bundle.validate_contents(NodeProfile)
-            if node_profile.node_type == NodeType.FULL:
-                event_queue = self.webhook_event_queue
-            elif node_profile.node_type == NodeType.PARTIAL:
-                event_queue = self.poll_event_queue
-        
-        elif node == self.config.koi_net.first_contact.rid:
-            logger.debug(f"Node {node!r} is my first contact")
-            # first contact node is always a webhook node
-            event_queue = self.webhook_event_queue
-        
-        else:
-            logger.warning(f"Node {node!r} unknown to me")
-            return
-        
-        queue = event_queue.setdefault(node, Queue())
-        queue.put(event)
-                
-        if flush and event_queue is self.webhook_event_queue:
-            self.flush_webhook_queue(node)
-            
-    def _flush_queue(self, event_queue: EventQueue, node: KoiNetNode) -> list[Event]:
-        """Flushes a node's queue, returning list of events."""
-        queue = event_queue.get(node)
-        events = list()
-        if queue:
-            while not queue.empty():
-                event = queue.get()
-                logger.debug(f"Dequeued {event.event_type} {event.rid!r}")
-                events.append(event)
-        
-        return events
-    
-    def flush_poll_queue(self, node: KoiNetNode) -> list[Event]:
-        """Flushes a node's poll queue, returning list of events."""
-        logger.debug(f"Flushing poll queue for {node}")
-        return self._flush_queue(self.poll_event_queue, node)
-    
-    def flush_webhook_queue(self, node: KoiNetNode, requeue_on_fail: bool = True):
-        """Flushes a node's webhook queue, and broadcasts events.
-        
-        If node profile is unknown, or node type is not `FULL`, this operation will fail silently. If the remote node cannot be reached, all events will be requeued.
+        Event will be sent to webhook or poll queue depending on the 
+        node type and edge type of the specified node. If `flush` is set 
+        to `True`, the webhook queued will be flushed after pushing the 
+        event.
         """
         
-        logger.debug(f"Flushing webhook queue for {node}")
-        
-        # node_bundle = self.effector.deref(node)
-        
-        # if not node_bundle:
-        #     logger.warning(f"{node!r} not found")
-        #     return
-        
-        # node_profile = node_bundle.validate_contents(NodeProfile)
-        
-        # if node_profile.node_type != NodeType.FULL:
-        #     logger.warning(f"{node!r} is a partial node!")
-        #     return
-        
-        events = self._flush_queue(self.webhook_event_queue, node)
-        if not events: return
-        
-        logger.debug(f"Broadcasting {len(events)} events")
-        
-        try:  
-            self.request_handler.broadcast_events(node, events=events)
-
-        except NodeNotFoundError:
-            logger.warning("Broadcast failed (node not found)")
-            
-        except httpx.ConnectError:
-            logger.warning("Broadcast failed (couldn't connect)")
-            
-            if requeue_on_fail:
-                for event in events:
-                    self.push_event_to(event, node)
+        self.q.put(QueuedEvent(target=target, event=event))
+    

koi_net/network/graph.py

@@ -56,7 +56,9 @@ class NetworkGraph:
     ) -> list[KoiNetEdge]:
         """Returns edges this node belongs to.
         
-        All edges returned by default, specify `direction` to restrict to incoming or outgoing edges only."""
+        All edges returned by default, specify `direction` to restrict 
+        to incoming or outgoing edges only.
+        """
                         
         edges = []
         if direction != "in" and self.dg.out_edges:
@@ -85,7 +87,10 @@ class NetworkGraph:
     ) -> list[KoiNetNode]:
         """Returns neighboring nodes this node shares an edge with.
         
-        All neighboring nodes returned by default, specify `direction` to restrict to neighbors connected by incoming or outgoing edges only."""
+        All neighboring nodes returned by default, specify `direction` 
+        to restrict to neighbors connected by incoming or outgoing edges
+        only.
+        """
         
         neighbors = set()
         for edge_rid in self.get_edges(direction):

koi_net/network/request_handler.py

@@ -1,6 +1,7 @@
 import logging
 import httpx
 from rid_lib import RID
+from rid_lib.ext import Cache
 from rid_lib.types.koi_net_node import KoiNetNode
 
 from ..identity import NodeIdentity
@@ -27,11 +28,7 @@ from ..protocol.consts import (
 )
 from ..protocol.node import NodeProfile, NodeType
 from ..secure import Secure
-from ..effector import Effector
-
-from typing import TYPE_CHECKING
-if TYPE_CHECKING:
-    from .error_handler import ErrorHandler
+from .error_handler import ErrorHandler
 
 
 logger = logging.getLogger(__name__)
@@ -57,26 +54,25 @@ class UnknownPathError(Exception):
 class RequestHandler:
     """Handles making requests to other KOI nodes."""
     
-    effector: Effector
+    cache: Cache
     identity: NodeIdentity
     secure: Secure
-    error_handler: "ErrorHandler"
+    error_handler: ErrorHandler
     
     def __init__(
         self, 
-        effector: Effector,
+        cache: Cache,
         identity: NodeIdentity,
-        secure: Secure
+        secure: Secure,
+        error_handler: ErrorHandler
     ):
-        self.effector = effector
+        self.cache = cache
         self.identity = identity
         self.secure = secure
-        
-    def set_error_handler(self, error_handler: "ErrorHandler"):
         self.error_handler = error_handler
     
     def get_url(self, node_rid: KoiNetNode) -> str:
-        """Retrieves URL of a node."""
+        """Retrieves URL of a node from its RID."""
         
         logger.debug(f"Getting URL for {node_rid!r}")
         node_url = None
@@ -84,7 +80,7 @@ class RequestHandler:
         if node_rid == self.identity.rid:
             raise SelfRequestError("Don't talk to yourself")
         
-        node_bundle = self.effector.deref(node_rid)
+        node_bundle = self.cache.read(node_rid)
                 
         if node_bundle:
             node_profile = node_bundle.validate_contents(NodeProfile)
@@ -110,6 +106,7 @@ class RequestHandler:
         path: str, 
         request: RequestModels,
     ) -> ResponseModels | None:
+        """Makes a request to a node."""
         url = self.get_url(node) + path
         logger.info(f"Making request to {url}")
     
@@ -154,7 +151,10 @@ class RequestHandler:
         req: EventsPayload | None = None,
         **kwargs
     ) -> None:
-        """See protocol.api_models.EventsPayload for available kwargs."""
+        """Broadcasts events to a node.
+        
+        Pass `EventsPayload` object, or see `protocol.api_models.EventsPayload` for available kwargs.
+        """
         request = req or EventsPayload.model_validate(kwargs)
         self.make_request(node, BROADCAST_EVENTS_PATH, request)
         logger.info(f"Broadcasted {len(request.events)} event(s) to {node!r}")
@@ -165,7 +165,10 @@ class RequestHandler:
         req: PollEvents | None = None,
         **kwargs
     ) -> EventsPayload:
-        """See protocol.api_models.PollEvents for available kwargs."""
+        """Polls events from a node.
+        
+        Pass `PollEvents` object as `req` or fields as kwargs.
+        """
         request = req or PollEvents.model_validate(kwargs)
         resp = self.make_request(node, POLL_EVENTS_PATH, request)
         if type(resp) != ErrorResponse:
@@ -178,7 +181,10 @@ class RequestHandler:
         req: FetchRids | None = None,
         **kwargs
     ) -> RidsPayload:
-        """See protocol.api_models.FetchRids for available kwargs."""
+        """Fetches RIDs from a node.
+        
+        Pass `FetchRids` object as `req` or fields as kwargs.
+        """
         request = req or FetchRids.model_validate(kwargs)
         resp = self.make_request(node, FETCH_RIDS_PATH, request)
         if type(resp) != ErrorResponse:
@@ -191,7 +197,10 @@ class RequestHandler:
         req: FetchManifests | None = None,
         **kwargs
     ) -> ManifestsPayload:
-        """See protocol.api_models.FetchManifests for available kwargs."""
+        """Fetches manifests from a node.
+        
+        Pass `FetchManifests` object as `req` or fields as kwargs.
+        """
         request = req or FetchManifests.model_validate(kwargs)
         resp = self.make_request(node, FETCH_MANIFESTS_PATH, request)
         if type(resp) != ErrorResponse:
@@ -204,7 +213,10 @@ class RequestHandler:
         req: FetchBundles | None = None,
         **kwargs
     ) -> BundlesPayload:
-        """See protocol.api_models.FetchBundles for available kwargs."""
+        """Fetches bundles from a node.
+        
+        Pass `FetchBundles` object as `req` or fields as kwargs.
+        """
         request = req or FetchBundles.model_validate(kwargs)
         resp = self.make_request(node, FETCH_BUNDLES_PATH, request)
         if type(resp) != ErrorResponse:

koi_net/network/resolver.py

@@ -12,17 +12,15 @@ from ..protocol.event import Event
 from ..protocol.api_models import ErrorResponse
 from ..identity import NodeIdentity
 from ..config import NodeConfig
-from ..effector import Effector
 
 logger = logging.getLogger(__name__)
 
 
 class NetworkResolver:
-    """A collection of functions and classes to interact with the KOI network."""
+    """Handles resolving nodes or knowledge objects from the network."""
     
     config: NodeConfig    
     identity: NodeIdentity
-    effector: Effector
     cache: Cache
     graph: NetworkGraph
     request_handler: RequestHandler
@@ -32,7 +30,6 @@ class NetworkResolver:
         config: NodeConfig,
         cache: Cache, 
         identity: NodeIdentity,
-        effector: Effector,
         graph: NetworkGraph,
         request_handler: RequestHandler,
     ):
@@ -41,13 +38,12 @@ class NetworkResolver:
         self.cache = cache
         self.graph = graph
         self.request_handler = request_handler
-        self.effector = effector
         
         self.poll_event_queue = dict()
         self.webhook_event_queue = dict()
     
     def get_state_providers(self, rid_type: RIDType) -> list[KoiNetNode]:
-        """Returns list of node RIDs which provide state for the specified RID type."""
+        """Returns list of node RIDs which provide state for specified RID type."""
         
         logger.debug(f"Looking for state providers of {rid_type}")
         provider_nodes = []
@@ -106,9 +102,10 @@ class NetworkResolver:
         return remote_manifest, node_rid
     
     def poll_neighbors(self) -> dict[KoiNetNode, list[Event]]:
-        """Polls all neighboring nodes and returns compiled list of events.
+        """Polls all neighbor nodes and returns compiled list of events.
         
-        If this node has no neighbors, it will instead attempt to poll the provided first contact URL.
+        Neighbor nodes also include the first contact, regardless of
+        whether the first contact profile is known to this node.
         """
         
         graph_neighbors = self.graph.get_neighbors()