koi-net 1.1.0b8__py3-none-any.whl → 1.2.0b2__py3-none-any.whl

koi_net/network/resolver.py

@@ -1,4 +1,4 @@
-import logging
+import structlog
 import httpx
 from rid_lib import RID
 from rid_lib.core import RIDType
@@ -12,17 +12,15 @@ from ..protocol.event import Event
 from ..protocol.api_models import ErrorResponse
 from ..identity import NodeIdentity
 from ..config import NodeConfig
-from ..effector import Effector
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class NetworkResolver:
-    """A collection of functions and classes to interact with the KOI network."""
+    """Handles resolving nodes or knowledge objects from the network."""
     
     config: NodeConfig    
     identity: NodeIdentity
-    effector: Effector
     cache: Cache
     graph: NetworkGraph
     request_handler: RequestHandler
@@ -32,7 +30,6 @@ class NetworkResolver:
         config: NodeConfig,
         cache: Cache, 
         identity: NodeIdentity,
-        effector: Effector,
         graph: NetworkGraph,
         request_handler: RequestHandler,
     ):
@@ -41,15 +38,14 @@ class NetworkResolver:
         self.cache = cache
         self.graph = graph
         self.request_handler = request_handler
-        self.effector = effector
         
         self.poll_event_queue = dict()
         self.webhook_event_queue = dict()
     
     def get_state_providers(self, rid_type: RIDType) -> list[KoiNetNode]:
-        """Returns list of node RIDs which provide state for the specified RID type."""
+        """Returns list of node RIDs which provide state for specified RID type."""
         
-        logger.debug(f"Looking for state providers of {rid_type}")
+        log.debug(f"Looking for state providers of {rid_type}")
         provider_nodes = []
         for node_rid in self.cache.list_rids(rid_types=[KoiNetNode]):
             if node_rid == self.identity.rid:
@@ -60,17 +56,17 @@ class NetworkResolver:
             node_profile = node_bundle.validate_contents(NodeProfile)
             
             if (node_profile.node_type == NodeType.FULL) and (rid_type in node_profile.provides.state):
-                logger.debug(f"Found provider {node_rid!r}")
+                log.debug(f"Found provider {node_rid!r}")
                 provider_nodes.append(node_rid)
         
         if not provider_nodes:
-            logger.debug("Failed to find providers")
+            log.debug("Failed to find providers")
         return provider_nodes
             
     def fetch_remote_bundle(self, rid: RID) -> tuple[Bundle | None, KoiNetNode | None]:
         """Attempts to fetch a bundle by RID from known peer nodes."""
         
-        logger.debug(f"Fetching remote bundle {rid!r}")
+        log.debug(f"Fetching remote bundle {rid!r}")
         remote_bundle, node_rid = None, None
         for node_rid in self.get_state_providers(type(rid)):
             payload = self.request_handler.fetch_bundles(
@@ -78,18 +74,18 @@ class NetworkResolver:
             
             if payload.bundles:
                 remote_bundle = payload.bundles[0]
-                logger.debug(f"Got bundle from {node_rid!r}")
+                log.debug(f"Got bundle from {node_rid!r}")
                 break
         
         if not remote_bundle:
-            logger.warning("Failed to fetch remote bundle")
+            log.warning("Failed to fetch remote bundle")
             
         return remote_bundle, node_rid
     
     def fetch_remote_manifest(self, rid: RID) -> tuple[Bundle | None, KoiNetNode | None]:
         """Attempts to fetch a manifest by RID from known peer nodes."""
         
-        logger.debug(f"Fetching remote manifest {rid!r}")
+        log.debug(f"Fetching remote manifest {rid!r}")
         remote_manifest, node_rid = None, None
         for node_rid in self.get_state_providers(type(rid)):
             payload = self.request_handler.fetch_manifests(
@@ -97,18 +93,19 @@ class NetworkResolver:
             
             if payload.manifests:
                 remote_manifest = payload.manifests[0]
-                logger.debug(f"Got bundle from {node_rid!r}")
+                log.debug(f"Got bundle from {node_rid!r}")
                 break
         
         if not remote_manifest:
-            logger.warning("Failed to fetch remote bundle")
+            log.warning("Failed to fetch remote bundle")
             
         return remote_manifest, node_rid
     
     def poll_neighbors(self) -> dict[KoiNetNode, list[Event]]:
-        """Polls all neighboring nodes and returns compiled list of events.
+        """Polls all neighbor nodes and returns compiled list of events.
         
-        If this node has no neighbors, it will instead attempt to poll the provided first contact URL.
+        Neighbor nodes also include the first contact, regardless of
+        whether the first contact profile is known to this node.
         """
         
         graph_neighbors = self.graph.get_neighbors()
@@ -139,12 +136,12 @@ class NetworkResolver:
                     continue
                     
                 if payload.events:
-                    logger.debug(f"Received {len(payload.events)} events from {node_rid!r}")
+                    log.debug(f"Received {len(payload.events)} events from {node_rid!r}")
                     
                     event_dict[node_rid] = payload.events
                     
             except httpx.ConnectError:
-                logger.debug(f"Failed to reach node {node_rid!r}")
+                log.debug(f"Failed to reach node {node_rid!r}")
                 continue
         
         return event_dict

koi_net/network/response_handler.py

@@ -1,10 +1,20 @@
-import logging
+import structlog
 from rid_lib import RID
 from rid_lib.types import KoiNetNode
 from rid_lib.ext import Manifest, Cache
 from rid_lib.ext.bundle import Bundle
 
+from koi_net.network.poll_event_buffer import PollEventBuffer
+from koi_net.processor.kobj_queue import KobjQueue
+from koi_net.protocol.consts import BROADCAST_EVENTS_PATH, FETCH_BUNDLES_PATH, FETCH_MANIFESTS_PATH, FETCH_RIDS_PATH, POLL_EVENTS_PATH
+from koi_net.protocol.envelope import SignedEnvelope
+from koi_net.protocol.model_map import API_MODEL_MAP
+from koi_net.secure import Secure
+
 from ..protocol.api_models import (
+    ApiModels,
+    EventsPayload,
+    PollEvents,
     RidsPayload,
     ManifestsPayload,
     BundlesPayload,
@@ -12,39 +22,88 @@ from ..protocol.api_models import (
     FetchManifests,
     FetchBundles,
 )
-from ..effector import Effector
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class ResponseHandler:
     """Handles generating responses to requests from other KOI nodes."""
     
     cache: Cache
-    effector: Effector
+    kobj_queue: KobjQueue
+    poll_event_buf: PollEventBuffer
     
     def __init__(
         self, 
-        cache: Cache, 
-        effector: Effector,
+        cache: Cache,
+        kobj_queue: KobjQueue,
+        poll_event_buf: PollEventBuffer,
+        secure: Secure
     ):
         self.cache = cache
-        self.effector = effector
+        self.kobj_queue = kobj_queue
+        self.poll_event_buf = poll_event_buf
+        self.secure = secure
+    
+    def handle_response(self, path: str, req: SignedEnvelope):
+        self.secure.validate_envelope(req)
+        
+        response_map = {
+            BROADCAST_EVENTS_PATH: self.broadcast_events_handler,
+            POLL_EVENTS_PATH: self.poll_events_handler,
+            FETCH_RIDS_PATH: self.fetch_rids_handler,
+            FETCH_MANIFESTS_PATH: self.fetch_manifests_handler,
+            FETCH_BUNDLES_PATH: self.fetch_bundles_handler
+        }
+        
+        response = response_map[path](req.payload, req.source_node)
+        
+        if response is None:
+            return
+        
+        return self.secure.create_envelope(
+            payload=response,
+            target=req.source_node
+        )
         
-    def fetch_rids(self, req: FetchRids, source: KoiNetNode) -> RidsPayload:
-        logger.info(f"Request to fetch rids, allowed types {req.rid_types}")
+    def broadcast_events_handler(self, req: EventsPayload, source: KoiNetNode):
+        log.info(f"Request to broadcast events, received {len(req.events)} event(s)")
+        
+        for event in req.events:
+            self.kobj_queue.put_kobj(event=event, source=source)
+        
+    def poll_events_handler(
+        self, 
+        req: PollEvents, 
+        source: KoiNetNode
+    ) -> EventsPayload:
+        log.info(f"Request to poll events")
+        events = self.poll_event_buf.flush(source, limit=req.limit)
+        return EventsPayload(events=events)
+        
+    def fetch_rids_handler(
+        self, 
+        req: FetchRids, 
+        source: KoiNetNode
+    ) -> RidsPayload:
+        """Returns response to fetch RIDs request."""
+        log.info(f"Request to fetch rids, allowed types {req.rid_types}")
         rids = self.cache.list_rids(req.rid_types)
         
         return RidsPayload(rids=rids)
         
-    def fetch_manifests(self, req: FetchManifests, source: KoiNetNode) -> ManifestsPayload:
-        logger.info(f"Request to fetch manifests, allowed types {req.rid_types}, rids {req.rids}")
+    def fetch_manifests_handler(self, 
+        req: FetchManifests, 
+        source: KoiNetNode
+    ) -> ManifestsPayload:
+        """Returns response to fetch manifests request."""
+        log.info(f"Request to fetch manifests, allowed types {req.rid_types}, rids {req.rids}")
         
         manifests: list[Manifest] = []
         not_found: list[RID] = []
         
         for rid in (req.rids or self.cache.list_rids(req.rid_types)):
-            bundle = self.effector.deref(rid)
+            bundle = self.cache.read(rid)
             if bundle:
                 manifests.append(bundle.manifest)
             else:
@@ -52,14 +111,19 @@ class ResponseHandler:
         
         return ManifestsPayload(manifests=manifests, not_found=not_found)
         
-    def fetch_bundles(self, req: FetchBundles, source: KoiNetNode) -> BundlesPayload:
-        logger.info(f"Request to fetch bundles, requested rids {req.rids}")
+    def fetch_bundles_handler(
+        self, 
+        req: FetchBundles, 
+        source: KoiNetNode
+    ) -> BundlesPayload:
+        """Returns response to fetch bundles request."""
+        log.info(f"Request to fetch bundles, requested rids {req.rids}")
         
         bundles: list[Bundle] = []
         not_found: list[RID] = []
 
         for rid in req.rids:
-            bundle = self.effector.deref(rid)
+            bundle = self.cache.read(rid)
             if bundle:
                 bundles.append(bundle)
             else:

koi_net/poller.py

@@ -1,35 +1,44 @@
 
 import time
-import logging
-from .processor.interface import ProcessorInterface
+import structlog
+
+from koi_net.interfaces.entrypoint import EntryPoint
+from .processor.kobj_queue import KobjQueue
 from .lifecycle import NodeLifecycle
 from .network.resolver import NetworkResolver
 from .config import NodeConfig
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
-class NodePoller:
+class NodePoller(EntryPoint):
+    """Manages polling based event loop for partial nodes."""
+    kobj_queue: KobjQueue
+    lifecycle: NodeLifecycle
+    resolver: NetworkResolver
+    config: NodeConfig
+    
     def __init__(
         self,
-        processor: ProcessorInterface,
+        config: NodeConfig,
         lifecycle: NodeLifecycle,
+        kobj_queue: KobjQueue,
         resolver: NetworkResolver,
-        config: NodeConfig
     ):
-        self.processor = processor
+        self.kobj_queue = kobj_queue
         self.lifecycle = lifecycle
         self.resolver = resolver
         self.config = config
 
     def poll(self):
+        """Polls neighbors and processes returned events."""
         neighbors = self.resolver.poll_neighbors()
         for node_rid in neighbors:
             for event in neighbors[node_rid]:
-                self.processor.handle(event=event, source=node_rid)
-        self.processor.flush_kobj_queue()
+                self.kobj_queue.put_kobj(event=event, source=node_rid)
 
     def run(self):
+        """Runs polling event loop."""
         with self.lifecycle.run():
             while True:
                 start_time = time.time()

koi_net/processor/event_worker.py

@@ -0,0 +1,117 @@
+import queue
+import traceback
+import time
+import structlog
+
+from rid_lib.ext import Cache
+from rid_lib.types import KoiNetNode
+
+from koi_net.config import NodeConfig
+from koi_net.network.event_queue import EventQueue, QueuedEvent
+from koi_net.network.request_handler import RequestHandler
+from koi_net.network.poll_event_buffer import PollEventBuffer
+from koi_net.protocol.event import Event
+from koi_net.protocol.node import NodeProfile, NodeType
+from koi_net.interfaces.worker import ThreadWorker, STOP_WORKER
+
+log = structlog.stdlib.get_logger()
+
+
+class EventProcessingWorker(ThreadWorker):
+    event_buffer: dict[KoiNetNode, list[Event]]
+    buffer_times: dict[KoiNetNode, float]
+
+    def __init__(
+        self,
+        event_queue: EventQueue,
+        request_handler: RequestHandler,
+        config: NodeConfig,
+        cache: Cache,
+        poll_event_buf: PollEventBuffer
+    ):
+        self.event_queue = event_queue
+        self.request_handler = request_handler
+        
+        self.config = config
+        self.cache = cache
+        self.poll_event_buf = poll_event_buf
+        
+        self.timeout: float = 0.1
+        self.max_buf_len: int = 5
+        self.max_wait_time: float = 1.0
+        
+        self.event_buffer = dict()
+        self.buffer_times = dict()
+        
+        super().__init__()
+        
+    def flush_buffer(self, target: KoiNetNode, buffer: list[Event]):
+        try:
+            self.request_handler.broadcast_events(target, events=buffer)
+        except Exception as e:
+            traceback.print_exc()
+        
+        self.event_buffer[target] = []
+        self.buffer_times[target] = None
+        
+    def decide_event(self, item: QueuedEvent) -> bool:
+        node_bundle = self.cache.read(item.target)
+        if node_bundle: 
+            node_profile = node_bundle.validate_contents(NodeProfile)
+            
+            if node_profile.node_type == NodeType.FULL:
+                return True
+        
+            elif node_profile.node_type == NodeType.PARTIAL:
+                self.poll_event_buf.put(item.target, item.event)
+                return False
+        
+        elif item.target == self.config.koi_net.first_contact.rid:
+            return True
+        
+        else:
+            log.warning(f"Couldn't handle event {item.event!r} in queue, node {item.target!r} unknown to me")
+            return False
+        
+
+    def run(self):
+        log.info("Started event worker")
+        while True:
+            now = time.time()
+            try:
+                item = self.event_queue.q.get(timeout=self.timeout)
+                
+                try:
+                    if item is STOP_WORKER:
+                        log.info(f"Received 'STOP_WORKER' signal, flushing buffer...")
+                        for target in self.event_buffer.keys():
+                            self.flush_buffer(target, self.event_buffer[target])
+                        return
+                    
+                    log.info(f"Dequeued {item.event!r} -> {item.target!r}")
+                    
+                    if not self.decide_event(item):
+                        continue
+                    
+                    event_buf = self.event_buffer.setdefault(item.target, [])
+                    if not event_buf:
+                        self.buffer_times[item.target] = now
+                    
+                    event_buf.append(item.event)
+
+                    # When new events are dequeued, check buffer for max length
+                    if len(event_buf) >= self.max_buf_len:
+                        self.flush_buffer(item.target, event_buf)
+                finally:
+                    self.event_queue.q.task_done()
+
+            except queue.Empty:
+                # On timeout, check all buffers for max wait time
+                for target, event_buf in self.event_buffer.items():
+                    if (len(event_buf) == 0) or (self.buffer_times.get(target) is None):
+                        continue
+                    if (now - self.buffer_times[target]) >= self.max_wait_time: 
+                        self.flush_buffer(target, event_buf)
+                    
+            except Exception as e:
+                traceback.print_exc()

koi_net/processor/handler.py

@@ -44,9 +44,11 @@ class KnowledgeHandler:
         rid_types: list[RIDType] | None = None,
         event_types: list[EventType | None] | None = None
     ):
-        """Special decorator that returns a KnowledgeHandler instead of a function.
+        """Decorator wraps a function, returns a KnowledgeHandler.
         
-        The function symbol will redefined as a `KnowledgeHandler`, which can be passed into the `ProcessorInterface` constructor. This is used to register default handlers.
+        The function symbol will redefined as a `KnowledgeHandler`, 
+        which can be passed into the `ProcessorInterface` constructor. 
+        This is used to register default handlers.
         """
         def decorator(func: Callable) -> KnowledgeHandler:
             handler = cls(func, handler_type, rid_types, event_types)