koi-net 1.1.0b8__py3-none-any.whl → 1.2.0b2__py3-none-any.whl

koi_net/processor/{default_handlers.py → handlers.py}

@@ -1,6 +1,6 @@
-"""Provides implementations of default knowledge handlers."""
+"""Implementation of default knowledge handlers."""
 
-import logging
+import structlog
 from rid_lib.ext import Bundle
 from rid_lib.ext.utils import sha256_hash
 from rid_lib.types import KoiNetNode, KoiNetEdge
@@ -9,10 +9,11 @@ from .handler import KnowledgeHandler, HandlerType, STOP_CHAIN
 from .knowledge_object import KnowledgeObject
 from ..context import HandlerContext
 from ..protocol.event import Event, EventType
-from ..protocol.edge import EdgeProfile, EdgeStatus, EdgeType, generate_edge_bundle
+from ..protocol.edge import EdgeProfile, EdgeStatus, EdgeType
 from ..protocol.node import NodeProfile
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
+
 
 # RID handlers
 
@@ -20,10 +21,11 @@ logger = logging.getLogger(__name__)
 def basic_rid_handler(ctx: HandlerContext, kobj: KnowledgeObject):
     """Default RID handler.
     
-    Blocks external events about this node. Allows `FORGET` events if RID is known to this node.
+    Blocks external events about this node. Allows `FORGET` events if 
+    RID is known to this node.
     """
     if (kobj.rid == ctx.identity.rid and kobj.source):
-        logger.debug("Don't let anyone else tell me who I am!")
+        log.debug("Don't let anyone else tell me who I am!")
         return STOP_CHAIN
     
     if kobj.event_type == EventType.FORGET:
@@ -34,25 +36,27 @@ def basic_rid_handler(ctx: HandlerContext, kobj: KnowledgeObject):
 
 @KnowledgeHandler.create(HandlerType.Manifest)
 def basic_manifest_handler(ctx: HandlerContext, kobj: KnowledgeObject):
-    """Default manifest handler.
+    """Decider based on incoming manifest and cache state.
     
-    Blocks manifests with the same hash, or aren't newer than the cached version. Sets the normalized event type to `NEW` or `UPDATE` depending on whether the RID was previously known to this node.
+    Blocks manifests which have the same hash, or aren't newer than the 
+    cached version. Sets the normalized event type to `NEW` or `UPDATE`
+    depending on whether the RID was previously known.
     """
     prev_bundle = ctx.cache.read(kobj.rid)
 
     if prev_bundle:
         if kobj.manifest.sha256_hash == prev_bundle.manifest.sha256_hash:
-            logger.debug("Hash of incoming manifest is same as existing knowledge, ignoring")
+            log.debug("Hash of incoming manifest is same as existing knowledge, ignoring")
             return STOP_CHAIN
         if kobj.manifest.timestamp <= prev_bundle.manifest.timestamp:
-            logger.debug("Timestamp of incoming manifest is the same or older than existing knowledge, ignoring")
+            log.debug("Timestamp of incoming manifest is the same or older than existing knowledge, ignoring")
             return STOP_CHAIN
         
-        logger.debug("RID previously known to me, labeling as 'UPDATE'")
+        log.debug("RID previously known to me, labeling as 'UPDATE'")
         kobj.normalized_event_type = EventType.UPDATE
 
     else:
-        logger.debug("RID previously unknown to me, labeling as 'NEW'")
+        log.debug("RID previously unknown to me, labeling as 'NEW'")
         kobj.normalized_event_type = EventType.NEW
         
     return kobj
@@ -63,14 +67,19 @@ def basic_manifest_handler(ctx: HandlerContext, kobj: KnowledgeObject):
 @KnowledgeHandler.create(
     handler_type=HandlerType.Bundle,
     rid_types=[KoiNetNode],
-    event_types=[EventType.NEW, EventType.UPDATE]
-)
+    event_types=[EventType.NEW, EventType.UPDATE])
 def secure_profile_handler(ctx: HandlerContext, kobj: KnowledgeObject):
+    """Maintains security of cached node profiles.
+    
+    Blocks bundles with a mismatching public keys in their node profile
+    and RID from continuing through the pipeline.
+    """
+    
     node_profile = kobj.bundle.validate_contents(NodeProfile)
     node_rid: KoiNetNode = kobj.rid
     
     if sha256_hash(node_profile.public_key) != node_rid.hash:
-        logger.warning(f"Public key hash mismatch for {node_rid!r}!")
+        log.warning(f"Public key hash mismatch for {node_rid!r}!")
         return STOP_CHAIN
 
 @KnowledgeHandler.create(
@@ -80,7 +89,10 @@ def secure_profile_handler(ctx: HandlerContext, kobj: KnowledgeObject):
 def edge_negotiation_handler(ctx: HandlerContext, kobj: KnowledgeObject):
     """Handles basic edge negotiation process.
     
-    Automatically approves proposed edges if they request RID types this node can provide (or KOI nodes/edges). Validates the edge type is allowed for the node type (partial nodes cannot use webhooks). If edge is invalid, a `FORGET` event is sent to the other node.
+    Automatically approves proposed edges if they request RID types this 
+    node can provide (or KOI nodes/edges). Validates the edge type is 
+    allowed for the node type (partial nodes cannot use webhooks). If 
+    edge is invalid, a `FORGET` event is sent to the other node.
     """
 
     # only respond when source is another node
@@ -93,13 +105,13 @@ def edge_negotiation_handler(ctx: HandlerContext, kobj: KnowledgeObject):
         if edge_profile.status != EdgeStatus.PROPOSED:
             return
         
-        logger.debug("Handling edge negotiation")
+        log.debug("Handling edge negotiation")
         
         peer_rid = edge_profile.target
-        peer_bundle = ctx.effector.deref(peer_rid)
+        peer_bundle = ctx.cache.read(peer_rid)
         
         if not peer_bundle:
-            logger.warning(f"Peer {peer_rid!r} unknown to me")
+            log.warning(f"Peer {peer_rid!r} unknown to me")
             return STOP_CHAIN
         
         peer_profile = peer_bundle.validate_contents(NodeProfile)
@@ -114,11 +126,11 @@ def edge_negotiation_handler(ctx: HandlerContext, kobj: KnowledgeObject):
         abort = False
         if (edge_profile.edge_type == EdgeType.WEBHOOK and 
             peer_profile.node_type == NodeType.PARTIAL):
-            logger.debug("Partial nodes cannot use webhooks")
+            log.debug("Partial nodes cannot use webhooks")
             abort = True
         
         if not set(edge_profile.rid_types).issubset(provided_events):
-            logger.debug("Requested RID types not provided by this node")
+            log.debug("Requested RID types not provided by this node")
             abort = True
         
         if abort:
@@ -128,80 +140,116 @@ def edge_negotiation_handler(ctx: HandlerContext, kobj: KnowledgeObject):
 
         else:
             # approve edge profile
-            logger.debug("Approving proposed edge")
+            log.debug("Approving proposed edge")
             edge_profile.status = EdgeStatus.APPROVED
             updated_bundle = Bundle.generate(kobj.rid, edge_profile.model_dump())
       
-            ctx.handle(bundle=updated_bundle, event_type=EventType.UPDATE)
+            ctx.kobj_queue.put_kobj(bundle=updated_bundle, event_type=EventType.UPDATE)
             return
               
     elif edge_profile.target == ctx.identity.rid:
         if edge_profile.status == EdgeStatus.APPROVED:
-            logger.debug("Edge approved by other node!")
+            log.debug("Edge approved by other node!")
 
 
 # Network handlers
 
 @KnowledgeHandler.create(HandlerType.Network, rid_types=[KoiNetNode])
-def coordinator_contact(ctx: HandlerContext, kobj: KnowledgeObject):
-    node_profile = kobj.bundle.validate_contents(NodeProfile)
-            
-    # looking for event provider of nodes
-    if KoiNetNode not in node_profile.provides.event:
-        return
+def node_contact_handler(ctx: HandlerContext, kobj: KnowledgeObject):
+    """Makes contact with providers of RID types of interest.
     
-    # prevents coordinators from attempting to form a self loop
+    When an incoming node knowledge object is identified as a provider
+    of an RID type of interest, this handler will propose a new edge 
+    subscribing to future node events, and fetch existing nodes to catch 
+    up to the current state.
+    """
+    # prevents nodes from attempting to form a self loop
     if kobj.rid == ctx.identity.rid:
         return
     
+    node_profile = kobj.bundle.validate_contents(NodeProfile)
+    
+    available_rid_types = list(
+        set(ctx.config.koi_net.rid_types_of_interest) & 
+        set(node_profile.provides.event)
+    )
+    
+    if not available_rid_types:
+        return
+    
+    log.info("Identified a coordinator!")
+    log.info("Proposing new edge")
+    
     # already have an edge established
-    if ctx.graph.get_edge(
+    edge_rid = ctx.graph.get_edge(
         source=kobj.rid,
         target=ctx.identity.rid,
-    ) is not None:
-        return
-    
-    logger.info("Identified a coordinator!")
-    logger.info("Proposing new edge")
+    )
     
-    if ctx.identity.profile.node_type == NodeType.FULL:
-        edge_type = EdgeType.WEBHOOK
+    if edge_rid:
+        prev_edge_bundle = ctx.cache.read(edge_rid)
+        edge_profile = prev_edge_bundle.validate_contents(EdgeProfile)
+        
+        if set(edge_profile.rid_types) == set(available_rid_types):
+            # no change in rid types
+            return
+        
+        edge_profile.rid_types = available_rid_types
+        edge_profile.status = EdgeStatus.PROPOSED
+        
     else:
-        edge_type = EdgeType.POLL
+        source = kobj.rid
+        target = ctx.identity.rid
+        if ctx.identity.profile.node_type == NodeType.FULL:
+            edge_type = EdgeType.WEBHOOK
+        else:
+            edge_type = EdgeType.POLL
+        
+        edge_rid = KoiNetEdge(sha256_hash(str(source) + str(target)))
+        edge_profile = EdgeProfile(
+            source=source,
+            target=target,
+            rid_types=available_rid_types,
+            edge_type=edge_type,
+            status=EdgeStatus.PROPOSED
+        )
     
     # queued for processing
-    ctx.handle(bundle=generate_edge_bundle(
-        source=kobj.rid,
-        target=ctx.identity.rid,
-        edge_type=edge_type,
-        rid_types=[KoiNetNode]
-    ))
+    edge_bundle = Bundle.generate(edge_rid, edge_profile.model_dump())
+    ctx.kobj_queue.put_kobj(bundle=edge_bundle)
     
-    logger.info("Catching up on network state")
+    log.info("Catching up on network state")
     
     payload = ctx.request_handler.fetch_rids(
         node=kobj.rid, 
-        rid_types=[KoiNetNode]
+        rid_types=available_rid_types
     )
     for rid in payload.rids:
         if rid == ctx.identity.rid:
-            logger.info("Skipping myself")
+            log.info("Skipping myself")
             continue
         if ctx.cache.exists(rid):
-            logger.info(f"Skipping known RID {rid!r}")
+            log.info(f"Skipping known RID {rid!r}")
             continue
         
         # marked as external since we are handling RIDs from another node
         # will fetch remotely instead of checking local cache
-        ctx.handle(rid=rid, source=kobj.rid)
-    logger.info("Done")
+        ctx.kobj_queue.put_kobj(rid=rid, source=kobj.rid)
+    log.info("Done")
     
 
 @KnowledgeHandler.create(HandlerType.Network)
 def basic_network_output_filter(ctx: HandlerContext, kobj: KnowledgeObject):
-    """Default network handler.
+    """Adds subscriber nodes to network targetes.
     
-    Allows broadcasting of all RID types this node is an event provider for (set in node profile), and other nodes have subscribed to. All nodes will also broadcast about their own (internally sourced) KOI node, and KOI edges that they are part of, regardless of their node profile configuration. Finally, nodes will also broadcast about edges to the other node involved (regardless of if they are subscribed)."""
+    Allows broadcasting of all RID types this node is an event provider 
+    for (set in node profile), and other nodes have subscribed to. All 
+    nodes will also broadcast about their own (internally sourced) KOI 
+    node, and KOI edges that they are part of, regardless of their node 
+    profile configuration. Finally, nodes will also broadcast about 
+    edges to the other node involved (regardless of if they are 
+    subscribed).
+    """
     
     involves_me = False
     if kobj.source is None:
@@ -213,12 +261,12 @@ def basic_network_output_filter(ctx: HandlerContext, kobj: KnowledgeObject):
             edge_profile = kobj.bundle.validate_contents(EdgeProfile)
             
             if edge_profile.source == ctx.identity.rid:
-                logger.debug(f"Adding edge target '{edge_profile.target!r}' to network targets")
+                log.debug(f"Adding edge target '{edge_profile.target!r}' to network targets")
                 kobj.network_targets.update([edge_profile.target])
                 involves_me = True
                 
             elif edge_profile.target == ctx.identity.rid:
-                logger.debug(f"Adding edge source '{edge_profile.source!r}' to network targets")
+                log.debug(f"Adding edge source '{edge_profile.source!r}' to network targets")
                 kobj.network_targets.update([edge_profile.source])
                 involves_me = True
     
@@ -229,13 +277,15 @@ def basic_network_output_filter(ctx: HandlerContext, kobj: KnowledgeObject):
             allowed_type=type(kobj.rid)
         )
         
-        logger.debug(f"Updating network targets with '{type(kobj.rid)}' subscribers: {subscribers}")
+        log.debug(f"Updating network targets with '{type(kobj.rid)}' subscribers: {subscribers}")
         kobj.network_targets.update(subscribers)
         
     return kobj
 
 @KnowledgeHandler.create(HandlerType.Final, rid_types=[KoiNetNode])
 def forget_edge_on_node_deletion(ctx: HandlerContext, kobj: KnowledgeObject):
+    """Removes edges to forgotten nodes."""
+    
     if kobj.normalized_event_type != EventType.FORGET:
         return
     
@@ -245,5 +295,5 @@ def forget_edge_on_node_deletion(ctx: HandlerContext, kobj: KnowledgeObject):
         edge_profile = edge_bundle.validate_contents(EdgeProfile)
         
         if kobj.rid in (edge_profile.source, edge_profile.target):
-            logger.debug("Identified edge with forgotten node")
-            ctx.handle(rid=edge_rid, event_type=EventType.FORGET)
+            log.debug("Identified edge with forgotten node")
+            ctx.kobj_queue.put_kobj(rid=edge_rid, event_type=EventType.FORGET)

koi_net/processor/knowledge_object.py

@@ -9,15 +9,21 @@ from ..protocol.event import Event, EventType
 class KnowledgeObject(BaseModel):
     """A normalized knowledge representation for internal processing.
     
-    Capable of representing an RID, manifest, bundle, or event. Contains three additional fields use for decision making in the knowledge processing pipeline. 
+    Represents an RID, manifest, bundle, or event. Contains three fields 
+    (`normalized_event_type`, `source`, `network_targets`) used for 
+    decision making in the knowledge processing pipeline. The source 
+    indicates which node this object originated from, or `None` if it 
+    was generated by this node. 
     
-    The source indicates whether this object was generated by this node, or sourced from another node in the network. 
+    The normalized event type indicates how the knowledge object is 
+    viewed from the perspective of this node, and what cache actions 
+    will take place. (`NEW`, `UPDATE`) -> cache write, `FORGET` -> 
+    cache delete, `None` -> no cache action.
     
-    The normalized event type indicates how the knowledge object is viewed from the perspective of this node, and what cache actions will take place. `NEW`, `UPDATE` -> cache write, `FORGET` -> cache delete, `None` -> no cache action.
-    
-    The network targets indicate other nodes in the network this knowledge object will be sent to. The event sent to them will be constructed from this knowledge object's RID, manifest, contents, and normalized event type.
-    
-    Constructors are provided to create a knowledge object from an RID, manifest, bundle, or event.
+    The network targets indicate other nodes in the network this 
+    knowledge object will be sent to. The event sent to them will be 
+    constructed from this knowledge object's RID, manifest, contents, 
+    and normalized event type.
     """
     rid: RID
     manifest: Manifest | None = None
@@ -37,6 +43,7 @@ class KnowledgeObject(BaseModel):
         event_type: EventType | None = None, 
         source: KoiNetNode | None = None
     ) -> "KnowledgeObject":
+        """Creates a `KnowledgeObject` from an `RID`."""
         return cls(
             rid=rid,
             event_type=event_type,
@@ -50,6 +57,7 @@ class KnowledgeObject(BaseModel):
         event_type: EventType | None = None, 
         source: KoiNetNode | None = None
     ) -> "KnowledgeObject":
+        """Creates a `KnowledgeObject` from a `Manifest`."""
         return cls(
             rid=manifest.rid,
             manifest=manifest,
@@ -64,6 +72,7 @@ class KnowledgeObject(BaseModel):
         event_type: EventType | None = None, 
         source: KoiNetNode | None = None
     ) -> "KnowledgeObject":
+        """Creates a `KnowledgeObject` from a `Bundle`."""
         return cls(
             rid=bundle.rid,
             manifest=bundle.manifest,
@@ -78,6 +87,7 @@ class KnowledgeObject(BaseModel):
         event: Event,
         source: KoiNetNode | None = None
     ) -> "KnowledgeObject":
+        """Creates a `KnowledgeObject` from an `Event`."""
         return cls(
             rid=event.rid,
             manifest=event.manifest,
@@ -88,6 +98,7 @@ class KnowledgeObject(BaseModel):
     
     @property
     def bundle(self):
+        """Bundle representation of knowledge object."""
         if self.manifest is None or self.contents is None:
             return
         
@@ -98,6 +109,7 @@ class KnowledgeObject(BaseModel):
     
     @property
     def normalized_event(self):
+        """Event representation of knowledge object."""
         if self.normalized_event_type is None:
             raise ValueError("Internal event's normalized event type is None, cannot convert to Event")
         

koi_net/processor/kobj_queue.py

@@ -0,0 +1,51 @@
+import structlog
+from queue import Queue
+from rid_lib.core import RID
+from rid_lib.ext import Bundle, Manifest
+from rid_lib.types import KoiNetNode
+from ..protocol.event import Event, EventType
+from .knowledge_object import KnowledgeObject
+
+log = structlog.stdlib.get_logger()
+
+
+class KobjQueue:
+    """Provides access to this node's knowledge processing pipeline."""
+    q: Queue[KnowledgeObject]
+    
+    def __init__(self):
+        self.q = Queue()
+                
+    def put_kobj(
+        self,
+        rid: RID | None = None,
+        manifest: Manifest | None = None,
+        bundle: Bundle | None = None,
+        event: Event | None = None,
+        kobj: KnowledgeObject | None = None,
+        event_type: EventType | None = None,
+        source: KoiNetNode | None = None
+    ):
+        """Queues knowledge object to be handled by processing pipeline.
+        
+        Knowledge may take the form of an RID, manifest, bundle, event, 
+        or knowledge object (with an optional event type for RID, 
+        manifest, or bundle objects). All objects will be normalized 
+        to knowledge objects and queued. If `flush` is `True`, the queue 
+        will be flushed immediately after adding the new knowledge.
+        """
+        if rid:
+            _kobj = KnowledgeObject.from_rid(rid, event_type, source)
+        elif manifest:
+            _kobj = KnowledgeObject.from_manifest(manifest, event_type, source)
+        elif bundle:
+            _kobj = KnowledgeObject.from_bundle(bundle, event_type, source)
+        elif event:
+            _kobj = KnowledgeObject.from_event(event, source)
+        elif kobj:
+            _kobj = kobj
+        else:
+            raise ValueError("One of 'rid', 'manifest', 'bundle', 'event', or 'kobj' must be provided")
+        
+        self.q.put(_kobj)
+        log.debug(f"Queued {_kobj!r}")

koi_net/processor/kobj_worker.py

@@ -0,0 +1,44 @@
+import queue
+import traceback
+import structlog
+
+from .pipeline import KnowledgePipeline
+from .kobj_queue import KobjQueue
+from koi_net.interfaces.worker import ThreadWorker, STOP_WORKER
+
+log = structlog.stdlib.get_logger()
+
+
+class KnowledgeProcessingWorker(ThreadWorker):
+    def __init__(
+        self,
+        kobj_queue: KobjQueue,
+        pipeline: KnowledgePipeline
+    ):
+        self.kobj_queue = kobj_queue
+        self.pipeline = pipeline
+        self.timeout: float = 0.1
+
+        super().__init__()
+        
+    def run(self):
+        log.info("Started kobj worker")
+        while True:
+            try:
+                item = self.kobj_queue.q.get(timeout=self.timeout)
+                try:
+                    if item is STOP_WORKER:
+                        log.info("Received 'STOP_WORKER' signal, shutting down...")
+                        return
+                    
+                    log.info(f"Dequeued {item!r}")
+                    
+                    self.pipeline.process(item)
+                finally:
+                    self.kobj_queue.q.task_done()
+                    
+            except queue.Empty:
+                pass
+            
+            except Exception as e:
+                traceback.print_exc()