koi-net 1.1.0b7__py3-none-any.whl → 1.2.0b1__py3-none-any.whl

koi_net/network/resolver.py

@@ -12,17 +12,15 @@ from ..protocol.event import Event
 from ..protocol.api_models import ErrorResponse
 from ..identity import NodeIdentity
 from ..config import NodeConfig
-from ..effector import Effector
 
 logger = logging.getLogger(__name__)
 
 
 class NetworkResolver:
-    """A collection of functions and classes to interact with the KOI network."""
+    """Handles resolving nodes or knowledge objects from the network."""
     
     config: NodeConfig    
     identity: NodeIdentity
-    effector: Effector
     cache: Cache
     graph: NetworkGraph
     request_handler: RequestHandler
@@ -32,7 +30,6 @@ class NetworkResolver:
         config: NodeConfig,
         cache: Cache, 
         identity: NodeIdentity,
-        effector: Effector,
         graph: NetworkGraph,
         request_handler: RequestHandler,
     ):
@@ -41,15 +38,14 @@ class NetworkResolver:
         self.cache = cache
         self.graph = graph
         self.request_handler = request_handler
-        self.effector = effector
         
         self.poll_event_queue = dict()
         self.webhook_event_queue = dict()
     
     def get_state_providers(self, rid_type: RIDType) -> list[KoiNetNode]:
-        """Returns list of node RIDs which provide state for the specified RID type."""
+        """Returns list of node RIDs which provide state for specified RID type."""
         
-        logger.debug(f"Looking for state providers of '{rid_type}'")
+        logger.debug(f"Looking for state providers of {rid_type}")
         provider_nodes = []
         for node_rid in self.cache.list_rids(rid_types=[KoiNetNode]):
             if node_rid == self.identity.rid:
@@ -106,9 +102,10 @@ class NetworkResolver:
         return remote_manifest, node_rid
     
     def poll_neighbors(self) -> dict[KoiNetNode, list[Event]]:
-        """Polls all neighboring nodes and returns compiled list of events.
+        """Polls all neighbor nodes and returns compiled list of events.
         
-        If this node has no neighbors, it will instead attempt to poll the provided first contact URL.
+        Neighbor nodes also include the first contact, regardless of
+        whether the first contact profile is known to this node.
         """
         
         graph_neighbors = self.graph.get_neighbors()

koi_net/network/response_handler.py

@@ -12,7 +12,6 @@ from ..protocol.api_models import (
     FetchManifests,
     FetchBundles,
 )
-from ..effector import Effector
 
 logger = logging.getLogger(__name__)
 
@@ -21,30 +20,29 @@ class ResponseHandler:
     """Handles generating responses to requests from other KOI nodes."""
     
     cache: Cache
-    effector: Effector
     
     def __init__(
         self, 
         cache: Cache, 
-        effector: Effector,
     ):
         self.cache = cache
-        self.effector = effector
         
     def fetch_rids(self, req: FetchRids, source: KoiNetNode) -> RidsPayload:
+        """Returns response to fetch RIDs request."""
         logger.info(f"Request to fetch rids, allowed types {req.rid_types}")
         rids = self.cache.list_rids(req.rid_types)
         
         return RidsPayload(rids=rids)
         
     def fetch_manifests(self, req: FetchManifests, source: KoiNetNode) -> ManifestsPayload:
+        """Returns response to fetch manifests request."""
         logger.info(f"Request to fetch manifests, allowed types {req.rid_types}, rids {req.rids}")
         
         manifests: list[Manifest] = []
         not_found: list[RID] = []
         
         for rid in (req.rids or self.cache.list_rids(req.rid_types)):
-            bundle = self.effector.deref(rid)
+            bundle = self.cache.read(rid)
             if bundle:
                 manifests.append(bundle.manifest)
             else:
@@ -53,13 +51,14 @@ class ResponseHandler:
         return ManifestsPayload(manifests=manifests, not_found=not_found)
         
     def fetch_bundles(self, req: FetchBundles, source: KoiNetNode) -> BundlesPayload:
+        """Returns response to fetch bundles request."""
         logger.info(f"Request to fetch bundles, requested rids {req.rids}")
         
         bundles: list[Bundle] = []
         not_found: list[RID] = []
 
         for rid in req.rids:
-            bundle = self.effector.deref(rid)
+            bundle = self.cache.read(rid)
             if bundle:
                 bundles.append(bundle)
             else:

koi_net/poll_event_buffer.py

@@ -0,0 +1,17 @@
+from rid_lib.types import KoiNetNode
+
+from koi_net.protocol.event import Event
+
+
+class PollEventBuffer:
+    buffers: dict[KoiNetNode, list[Event]]
+    
+    def __init__(self):
+        self.buffers = dict()
+        
+    def put(self, node: KoiNetNode, event: Event):
+        event_buf = self.buffers.setdefault(node, [])
+        event_buf.append(event)
+        
+    def flush(self, node: KoiNetNode):
+        return self.buffers.pop(node, [])

koi_net/poller.py

@@ -1,7 +1,7 @@
 
 import time
 import logging
-from .processor.interface import ProcessorInterface
+from .processor.kobj_queue import KobjQueue
 from .lifecycle import NodeLifecycle
 from .network.resolver import NetworkResolver
 from .config import NodeConfig
@@ -10,26 +10,33 @@ logger = logging.getLogger(__name__)
 
 
 class NodePoller:
+    """Manages polling based event loop for partial nodes."""
+    kobj_queue: KobjQueue
+    lifecycle: NodeLifecycle
+    resolver: NetworkResolver
+    config: NodeConfig
+    
     def __init__(
         self,
-        processor: ProcessorInterface,
+        kobj_queue: KobjQueue,
         lifecycle: NodeLifecycle,
         resolver: NetworkResolver,
         config: NodeConfig
     ):
-        self.processor = processor
+        self.kobj_queue = kobj_queue
         self.lifecycle = lifecycle
         self.resolver = resolver
         self.config = config
 
     def poll(self):
+        """Polls neighbors and processes returned events."""
         neighbors = self.resolver.poll_neighbors()
         for node_rid in neighbors:
             for event in neighbors[node_rid]:
-                self.processor.handle(event=event, source=node_rid)
-        self.processor.flush_kobj_queue()
+                self.kobj_queue.put_kobj(event=event, source=node_rid)
 
     def run(self):
+        """Runs polling event loop."""
         with self.lifecycle.run():
             while True:
                 start_time = time.time()

koi_net/processor/default_handlers.py

@@ -1,4 +1,4 @@
-"""Provides implementations of default knowledge handlers."""
+"""Implementation of default knowledge handlers."""
 
 import logging
 from rid_lib.ext import Bundle
@@ -20,7 +20,8 @@ logger = logging.getLogger(__name__)
 def basic_rid_handler(ctx: HandlerContext, kobj: KnowledgeObject):
     """Default RID handler.
     
-    Blocks external events about this node. Allows `FORGET` events if RID is known to this node.
+    Blocks external events about this node. Allows `FORGET` events if 
+    RID is known to this node.
     """
     if (kobj.rid == ctx.identity.rid and kobj.source):
         logger.debug("Don't let anyone else tell me who I am!")
@@ -34,9 +35,11 @@ def basic_rid_handler(ctx: HandlerContext, kobj: KnowledgeObject):
 
 @KnowledgeHandler.create(HandlerType.Manifest)
 def basic_manifest_handler(ctx: HandlerContext, kobj: KnowledgeObject):
-    """Default manifest handler.
+    """Decider based on incoming manifest and cache state.
     
-    Blocks manifests with the same hash, or aren't newer than the cached version. Sets the normalized event type to `NEW` or `UPDATE` depending on whether the RID was previously known to this node.
+    Blocks manifests which have the same hash, or aren't newer than the 
+    cached version. Sets the normalized event type to `NEW` or `UPDATE`
+    depending on whether the RID was previously known.
     """
     prev_bundle = ctx.cache.read(kobj.rid)
 
@@ -63,9 +66,14 @@ def basic_manifest_handler(ctx: HandlerContext, kobj: KnowledgeObject):
 @KnowledgeHandler.create(
     handler_type=HandlerType.Bundle,
     rid_types=[KoiNetNode],
-    event_types=[EventType.NEW, EventType.UPDATE]
-)
+    event_types=[EventType.NEW, EventType.UPDATE])
 def secure_profile_handler(ctx: HandlerContext, kobj: KnowledgeObject):
+    """Maintains security of cached node profiles.
+    
+    Blocks bundles with a mismatching public keys in their node profile
+    and RID from continuing through the pipeline.
+    """
+    
     node_profile = kobj.bundle.validate_contents(NodeProfile)
     node_rid: KoiNetNode = kobj.rid
     
@@ -80,7 +88,10 @@ def secure_profile_handler(ctx: HandlerContext, kobj: KnowledgeObject):
 def edge_negotiation_handler(ctx: HandlerContext, kobj: KnowledgeObject):
     """Handles basic edge negotiation process.
     
-    Automatically approves proposed edges if they request RID types this node can provide (or KOI nodes/edges). Validates the edge type is allowed for the node type (partial nodes cannot use webhooks). If edge is invalid, a `FORGET` event is sent to the other node.
+    Automatically approves proposed edges if they request RID types this 
+    node can provide (or KOI nodes/edges). Validates the edge type is 
+    allowed for the node type (partial nodes cannot use webhooks). If 
+    edge is invalid, a `FORGET` event is sent to the other node.
     """
 
     # only respond when source is another node
@@ -96,7 +107,7 @@ def edge_negotiation_handler(ctx: HandlerContext, kobj: KnowledgeObject):
         logger.debug("Handling edge negotiation")
         
         peer_rid = edge_profile.target
-        peer_bundle = ctx.effector.deref(peer_rid)
+        peer_bundle = ctx.cache.read(peer_rid)
         
         if not peer_bundle:
             logger.warning(f"Peer {peer_rid!r} unknown to me")
@@ -132,7 +143,7 @@ def edge_negotiation_handler(ctx: HandlerContext, kobj: KnowledgeObject):
             edge_profile.status = EdgeStatus.APPROVED
             updated_bundle = Bundle.generate(kobj.rid, edge_profile.model_dump())
       
-            ctx.handle(bundle=updated_bundle, event_type=EventType.UPDATE)
+            ctx.kobj_queue.put_kobj(bundle=updated_bundle, event_type=EventType.UPDATE)
             return
               
     elif edge_profile.target == ctx.identity.rid:
@@ -143,45 +154,74 @@ def edge_negotiation_handler(ctx: HandlerContext, kobj: KnowledgeObject):
 # Network handlers
 
 @KnowledgeHandler.create(HandlerType.Network, rid_types=[KoiNetNode])
-def coordinator_contact(ctx: HandlerContext, kobj: KnowledgeObject):
-    node_profile = kobj.bundle.validate_contents(NodeProfile)
-            
-    # looking for event provider of nodes
-    if KoiNetNode not in node_profile.provides.event:
-        return
+def node_contact_handler(ctx: HandlerContext, kobj: KnowledgeObject):
+    """Makes contact with providers of RID types of interest.
     
-    # prevents coordinators from attempting to form a self loop
+    When an incoming node knowledge object is identified as a provider
+    of an RID type of interest, this handler will propose a new edge 
+    subscribing to future node events, and fetch existing nodes to catch 
+    up to the current state.
+    """
+    # prevents nodes from attempting to form a self loop
     if kobj.rid == ctx.identity.rid:
         return
     
-    # already have an edge established
-    if ctx.graph.get_edge(
-        source=kobj.rid,
-        target=ctx.identity.rid,
-    ) is not None:
+    node_profile = kobj.bundle.validate_contents(NodeProfile)
+    
+    available_rid_types = list(
+        set(ctx.config.koi_net.rid_types_of_interest) & 
+        set(node_profile.provides.event)
+    )
+    
+    if not available_rid_types:
         return
     
     logger.info("Identified a coordinator!")
     logger.info("Proposing new edge")
     
-    if ctx.identity.profile.node_type == NodeType.FULL:
-        edge_type = EdgeType.WEBHOOK
+    # already have an edge established
+    edge_rid = ctx.graph.get_edge(
+        source=kobj.rid,
+        target=ctx.identity.rid,
+    )
+    
+    if edge_rid:
+        prev_edge_bundle = ctx.cache.read(edge_rid)
+        edge_profile = prev_edge_bundle.validate_contents(EdgeProfile)
+        
+        if set(edge_profile.rid_types) == set(available_rid_types):
+            # no change in rid types
+            return
+        
+        edge_profile.rid_types = available_rid_types
+        edge_profile.status = EdgeStatus.PROPOSED
+        
     else:
-        edge_type = EdgeType.POLL
+        source = kobj.rid
+        target = ctx.identity.rid
+        if ctx.identity.profile.node_type == NodeType.FULL:
+            edge_type = EdgeType.WEBHOOK
+        else:
+            edge_type = EdgeType.POLL
+        
+        edge_rid = KoiNetEdge(sha256_hash(str(source) + str(target)))
+        edge_profile = EdgeProfile(
+            source=source,
+            target=target,
+            rid_types=available_rid_types,
+            edge_type=edge_type,
+            status=EdgeStatus.PROPOSED
+        )
     
     # queued for processing
-    ctx.handle(bundle=generate_edge_bundle(
-        source=kobj.rid,
-        target=ctx.identity.rid,
-        edge_type=edge_type,
-        rid_types=[KoiNetNode]
-    ))
+    edge_bundle = Bundle.generate(edge_rid, edge_profile.model_dump())
+    ctx.kobj_queue.put_kobj(bundle=edge_bundle)
     
     logger.info("Catching up on network state")
     
     payload = ctx.request_handler.fetch_rids(
         node=kobj.rid, 
-        rid_types=[KoiNetNode]
+        rid_types=available_rid_types
     )
     for rid in payload.rids:
         if rid == ctx.identity.rid:
@@ -193,15 +233,22 @@ def coordinator_contact(ctx: HandlerContext, kobj: KnowledgeObject):
         
         # marked as external since we are handling RIDs from another node
         # will fetch remotely instead of checking local cache
-        ctx.handle(rid=rid, source=kobj.rid)
+        ctx.kobj_queue.put_kobj(rid=rid, source=kobj.rid)
     logger.info("Done")
     
 
 @KnowledgeHandler.create(HandlerType.Network)
 def basic_network_output_filter(ctx: HandlerContext, kobj: KnowledgeObject):
-    """Default network handler.
+    """Adds subscriber nodes to network targetes.
     
-    Allows broadcasting of all RID types this node is an event provider for (set in node profile), and other nodes have subscribed to. All nodes will also broadcast about their own (internally sourced) KOI node, and KOI edges that they are part of, regardless of their node profile configuration. Finally, nodes will also broadcast about edges to the other node involved (regardless of if they are subscribed)."""
+    Allows broadcasting of all RID types this node is an event provider 
+    for (set in node profile), and other nodes have subscribed to. All 
+    nodes will also broadcast about their own (internally sourced) KOI 
+    node, and KOI edges that they are part of, regardless of their node 
+    profile configuration. Finally, nodes will also broadcast about 
+    edges to the other node involved (regardless of if they are 
+    subscribed).
+    """
     
     involves_me = False
     if kobj.source is None:
@@ -236,6 +283,8 @@ def basic_network_output_filter(ctx: HandlerContext, kobj: KnowledgeObject):
 
 @KnowledgeHandler.create(HandlerType.Final, rid_types=[KoiNetNode])
 def forget_edge_on_node_deletion(ctx: HandlerContext, kobj: KnowledgeObject):
+    """Removes edges to forgotten nodes."""
+    
     if kobj.normalized_event_type != EventType.FORGET:
         return
     
@@ -246,4 +295,4 @@ def forget_edge_on_node_deletion(ctx: HandlerContext, kobj: KnowledgeObject):
         
         if kobj.rid in (edge_profile.source, edge_profile.target):
             logger.debug("Identified edge with forgotten node")
-            ctx.handle(rid=edge_rid, event_type=EventType.FORGET)
+            ctx.kobj_queue.put_kobj(rid=edge_rid, event_type=EventType.FORGET)

koi_net/processor/event_worker.py

@@ -0,0 +1,121 @@
+import queue
+import traceback
+import time
+import logging
+
+from rid_lib.ext import Cache
+from rid_lib.types import KoiNetNode
+
+from koi_net.config import NodeConfig
+from koi_net.models import END, QueuedEvent
+from koi_net.network.event_queue import EventQueue
+from koi_net.network.request_handler import RequestHandler
+from koi_net.poll_event_buffer import PollEventBuffer
+from koi_net.protocol.event import Event
+from koi_net.protocol.node import NodeProfile, NodeType
+from koi_net.worker import ThreadWorker
+
+logger = logging.getLogger(__name__)
+
+
+class EventProcessingWorker(ThreadWorker):
+    event_buffer: dict[KoiNetNode, list[Event]]
+    buffer_times: dict[KoiNetNode, float]
+
+    def __init__(
+        self,
+        event_queue: EventQueue,
+        request_handler: RequestHandler,
+        config: NodeConfig,
+        cache: Cache,
+        poll_event_buf: PollEventBuffer,
+        queue_timeout: float = 0.1,
+        max_buf_len: int = 5,
+        max_wait_time: float = 1.0
+    ):
+        self.event_queue = event_queue
+        self.request_handler = request_handler
+        
+        self.config = config
+        self.cache = cache
+        self.poll_event_buf = poll_event_buf
+        
+        self.timeout = queue_timeout 
+        self.max_buf_len = max_buf_len
+        self.max_wait_time = max_wait_time
+        
+        self.event_buffer = dict()
+        self.buffer_times = dict()
+        
+        super().__init__()
+        
+    def flush_buffer(self, target: KoiNetNode, buffer: list[Event]):
+        try:
+            self.request_handler.broadcast_events(target, events=buffer)
+        except Exception as e:
+            traceback.print_exc()
+        
+        self.event_buffer[target] = []
+        self.buffer_times[target] = None
+        
+    def decide_event(self, item: QueuedEvent) -> bool:
+        node_bundle = self.cache.read(item.target)
+        if node_bundle: 
+            node_profile = node_bundle.validate_contents(NodeProfile)
+            
+            if node_profile.node_type == NodeType.FULL:
+                return True
+        
+            elif node_profile.node_type == NodeType.PARTIAL:
+                self.poll_event_buf.put(item.target, item.event)
+                return False
+        
+        elif item.target == self.config.koi_net.first_contact.rid:
+            return True
+        
+        else:
+            logger.warning(f"Couldn't handle event {item.event!r} in queue, node {item.target!r} unknown to me")
+            return False
+        
+
+    def run(self):
+        logger.info("Started event worker")
+        while True:
+            now = time.time()
+            try:
+                item = self.event_queue.q.get(timeout=self.timeout)
+                
+                try:
+                    if item is END:
+                        logger.info("Received 'END' signal, flushing buffer...")
+                        for target in self.event_buffer.keys():
+                            self.flush_buffer(target, self.event_buffer[target])
+                        return
+                    
+                    logger.info(f"Dequeued {item.event!r} -> {item.target!r}")
+                    
+                    if not self.decide_event(item):
+                        continue
+                    
+                    event_buf = self.event_buffer.setdefault(item.target, [])
+                    if not event_buf:
+                        self.buffer_times[item.target] = now
+                    
+                    event_buf.append(item.event)
+
+                    # When new events are dequeued, check buffer for max length
+                    if len(event_buf) >= self.max_buf_len:
+                        self.flush_buffer(item.target, event_buf)
+                finally:
+                    self.event_queue.q.task_done()
+
+            except queue.Empty:
+                # On timeout, check all buffers for max wait time
+                for target, event_buf in self.event_buffer.items():
+                    if (len(event_buf) == 0) or (self.buffer_times.get(target) is None):
+                        continue
+                    if (now - self.buffer_times[target]) >= self.max_wait_time: 
+                        self.flush_buffer(target, event_buf)
+                    
+            except Exception as e:
+                traceback.print_exc()

koi_net/processor/handler.py

@@ -44,9 +44,11 @@ class KnowledgeHandler:
         rid_types: list[RIDType] | None = None,
         event_types: list[EventType | None] | None = None
     ):
-        """Special decorator that returns a KnowledgeHandler instead of a function.
+        """Decorator wraps a function, returns a KnowledgeHandler.
         
-        The function symbol will redefined as a `KnowledgeHandler`, which can be passed into the `ProcessorInterface` constructor. This is used to register default handlers.
+        The function symbol will redefined as a `KnowledgeHandler`, 
+        which can be passed into the `ProcessorInterface` constructor. 
+        This is used to register default handlers.
         """
         def decorator(func: Callable) -> KnowledgeHandler:
             handler = cls(func, handler_type, rid_types, event_types)

koi_net/processor/knowledge_object.py

@@ -9,15 +9,21 @@ from ..protocol.event import Event, EventType
 class KnowledgeObject(BaseModel):
     """A normalized knowledge representation for internal processing.
     
-    Capable of representing an RID, manifest, bundle, or event. Contains three additional fields use for decision making in the knowledge processing pipeline. 
+    Represents an RID, manifest, bundle, or event. Contains three fields 
+    (`normalized_event_type`, `source`, `network_targets`) used for 
+    decision making in the knowledge processing pipeline. The source 
+    indicates which node this object originated from, or `None` if it 
+    was generated by this node. 
     
-    The source indicates whether this object was generated by this node, or sourced from another node in the network. 
+    The normalized event type indicates how the knowledge object is 
+    viewed from the perspective of this node, and what cache actions 
+    will take place. (`NEW`, `UPDATE`) -> cache write, `FORGET` -> 
+    cache delete, `None` -> no cache action.
     
-    The normalized event type indicates how the knowledge object is viewed from the perspective of this node, and what cache actions will take place. `NEW`, `UPDATE` -> cache write, `FORGET` -> cache delete, `None` -> no cache action.
-    
-    The network targets indicate other nodes in the network this knowledge object will be sent to. The event sent to them will be constructed from this knowledge object's RID, manifest, contents, and normalized event type.
-    
-    Constructors are provided to create a knowledge object from an RID, manifest, bundle, or event.
+    The network targets indicate other nodes in the network this 
+    knowledge object will be sent to. The event sent to them will be 
+    constructed from this knowledge object's RID, manifest, contents, 
+    and normalized event type.
     """
     rid: RID
     manifest: Manifest | None = None
@@ -37,6 +43,7 @@ class KnowledgeObject(BaseModel):
         event_type: EventType | None = None, 
         source: KoiNetNode | None = None
     ) -> "KnowledgeObject":
+        """Creates a `KnowledgeObject` from an `RID`."""
         return cls(
             rid=rid,
             event_type=event_type,
@@ -50,6 +57,7 @@ class KnowledgeObject(BaseModel):
         event_type: EventType | None = None, 
         source: KoiNetNode | None = None
     ) -> "KnowledgeObject":
+        """Creates a `KnowledgeObject` from a `Manifest`."""
         return cls(
             rid=manifest.rid,
             manifest=manifest,
@@ -64,6 +72,7 @@ class KnowledgeObject(BaseModel):
         event_type: EventType | None = None, 
         source: KoiNetNode | None = None
     ) -> "KnowledgeObject":
+        """Creates a `KnowledgeObject` from a `Bundle`."""
         return cls(
             rid=bundle.rid,
             manifest=bundle.manifest,
@@ -78,6 +87,7 @@ class KnowledgeObject(BaseModel):
         event: Event,
         source: KoiNetNode | None = None
     ) -> "KnowledgeObject":
+        """Creates a `KnowledgeObject` from an `Event`."""
         return cls(
             rid=event.rid,
             manifest=event.manifest,
@@ -88,6 +98,7 @@ class KnowledgeObject(BaseModel):
     
     @property
     def bundle(self):
+        """Bundle representation of knowledge object."""
         if self.manifest is None or self.contents is None:
             return
         
@@ -98,6 +109,7 @@ class KnowledgeObject(BaseModel):
     
     @property
     def normalized_event(self):
+        """Event representation of knowledge object."""
         if self.normalized_event_type is None:
             raise ValueError("Internal event's normalized event type is None, cannot convert to Event")