koi-net 1.2.0b1__py3-none-any.whl → 1.2.0b2__py3-none-any.whl

koi_net/network/resolver.py

@@ -1,4 +1,4 @@
-import logging
+import structlog
 import httpx
 from rid_lib import RID
 from rid_lib.core import RIDType
@@ -13,7 +13,7 @@ from ..protocol.api_models import ErrorResponse
 from ..identity import NodeIdentity
 from ..config import NodeConfig
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class NetworkResolver:
@@ -45,7 +45,7 @@ class NetworkResolver:
     def get_state_providers(self, rid_type: RIDType) -> list[KoiNetNode]:
         """Returns list of node RIDs which provide state for specified RID type."""
         
-        logger.debug(f"Looking for state providers of {rid_type}")
+        log.debug(f"Looking for state providers of {rid_type}")
         provider_nodes = []
         for node_rid in self.cache.list_rids(rid_types=[KoiNetNode]):
             if node_rid == self.identity.rid:
@@ -56,17 +56,17 @@ class NetworkResolver:
             node_profile = node_bundle.validate_contents(NodeProfile)
             
             if (node_profile.node_type == NodeType.FULL) and (rid_type in node_profile.provides.state):
-                logger.debug(f"Found provider {node_rid!r}")
+                log.debug(f"Found provider {node_rid!r}")
                 provider_nodes.append(node_rid)
         
         if not provider_nodes:
-            logger.debug("Failed to find providers")
+            log.debug("Failed to find providers")
         return provider_nodes
             
     def fetch_remote_bundle(self, rid: RID) -> tuple[Bundle | None, KoiNetNode | None]:
         """Attempts to fetch a bundle by RID from known peer nodes."""
         
-        logger.debug(f"Fetching remote bundle {rid!r}")
+        log.debug(f"Fetching remote bundle {rid!r}")
         remote_bundle, node_rid = None, None
         for node_rid in self.get_state_providers(type(rid)):
             payload = self.request_handler.fetch_bundles(
@@ -74,18 +74,18 @@ class NetworkResolver:
             
             if payload.bundles:
                 remote_bundle = payload.bundles[0]
-                logger.debug(f"Got bundle from {node_rid!r}")
+                log.debug(f"Got bundle from {node_rid!r}")
                 break
         
         if not remote_bundle:
-            logger.warning("Failed to fetch remote bundle")
+            log.warning("Failed to fetch remote bundle")
             
         return remote_bundle, node_rid
     
     def fetch_remote_manifest(self, rid: RID) -> tuple[Bundle | None, KoiNetNode | None]:
         """Attempts to fetch a manifest by RID from known peer nodes."""
         
-        logger.debug(f"Fetching remote manifest {rid!r}")
+        log.debug(f"Fetching remote manifest {rid!r}")
         remote_manifest, node_rid = None, None
         for node_rid in self.get_state_providers(type(rid)):
             payload = self.request_handler.fetch_manifests(
@@ -93,11 +93,11 @@ class NetworkResolver:
             
             if payload.manifests:
                 remote_manifest = payload.manifests[0]
-                logger.debug(f"Got bundle from {node_rid!r}")
+                log.debug(f"Got bundle from {node_rid!r}")
                 break
         
         if not remote_manifest:
-            logger.warning("Failed to fetch remote bundle")
+            log.warning("Failed to fetch remote bundle")
             
         return remote_manifest, node_rid
     
@@ -136,12 +136,12 @@ class NetworkResolver:
                     continue
                     
                 if payload.events:
-                    logger.debug(f"Received {len(payload.events)} events from {node_rid!r}")
+                    log.debug(f"Received {len(payload.events)} events from {node_rid!r}")
                     
                     event_dict[node_rid] = payload.events
                     
             except httpx.ConnectError:
-                logger.debug(f"Failed to reach node {node_rid!r}")
+                log.debug(f"Failed to reach node {node_rid!r}")
                 continue
         
         return event_dict

koi_net/network/response_handler.py

@@ -1,10 +1,20 @@
-import logging
+import structlog
 from rid_lib import RID
 from rid_lib.types import KoiNetNode
 from rid_lib.ext import Manifest, Cache
 from rid_lib.ext.bundle import Bundle
 
+from koi_net.network.poll_event_buffer import PollEventBuffer
+from koi_net.processor.kobj_queue import KobjQueue
+from koi_net.protocol.consts import BROADCAST_EVENTS_PATH, FETCH_BUNDLES_PATH, FETCH_MANIFESTS_PATH, FETCH_RIDS_PATH, POLL_EVENTS_PATH
+from koi_net.protocol.envelope import SignedEnvelope
+from koi_net.protocol.model_map import API_MODEL_MAP
+from koi_net.secure import Secure
+
 from ..protocol.api_models import (
+    ApiModels,
+    EventsPayload,
+    PollEvents,
     RidsPayload,
     ManifestsPayload,
     BundlesPayload,
@@ -13,30 +23,81 @@ from ..protocol.api_models import (
     FetchBundles,
 )
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class ResponseHandler:
     """Handles generating responses to requests from other KOI nodes."""
     
     cache: Cache
+    kobj_queue: KobjQueue
+    poll_event_buf: PollEventBuffer
     
     def __init__(
         self, 
-        cache: Cache, 
+        cache: Cache,
+        kobj_queue: KobjQueue,
+        poll_event_buf: PollEventBuffer,
+        secure: Secure
     ):
         self.cache = cache
+        self.kobj_queue = kobj_queue
+        self.poll_event_buf = poll_event_buf
+        self.secure = secure
+    
+    def handle_response(self, path: str, req: SignedEnvelope):
+        self.secure.validate_envelope(req)
+        
+        response_map = {
+            BROADCAST_EVENTS_PATH: self.broadcast_events_handler,
+            POLL_EVENTS_PATH: self.poll_events_handler,
+            FETCH_RIDS_PATH: self.fetch_rids_handler,
+            FETCH_MANIFESTS_PATH: self.fetch_manifests_handler,
+            FETCH_BUNDLES_PATH: self.fetch_bundles_handler
+        }
+        
+        response = response_map[path](req.payload, req.source_node)
+        
+        if response is None:
+            return
+        
+        return self.secure.create_envelope(
+            payload=response,
+            target=req.source_node
+        )
         
-    def fetch_rids(self, req: FetchRids, source: KoiNetNode) -> RidsPayload:
+    def broadcast_events_handler(self, req: EventsPayload, source: KoiNetNode):
+        log.info(f"Request to broadcast events, received {len(req.events)} event(s)")
+        
+        for event in req.events:
+            self.kobj_queue.put_kobj(event=event, source=source)
+        
+    def poll_events_handler(
+        self, 
+        req: PollEvents, 
+        source: KoiNetNode
+    ) -> EventsPayload:
+        log.info(f"Request to poll events")
+        events = self.poll_event_buf.flush(source, limit=req.limit)
+        return EventsPayload(events=events)
+        
+    def fetch_rids_handler(
+        self, 
+        req: FetchRids, 
+        source: KoiNetNode
+    ) -> RidsPayload:
         """Returns response to fetch RIDs request."""
-        logger.info(f"Request to fetch rids, allowed types {req.rid_types}")
+        log.info(f"Request to fetch rids, allowed types {req.rid_types}")
         rids = self.cache.list_rids(req.rid_types)
         
         return RidsPayload(rids=rids)
         
-    def fetch_manifests(self, req: FetchManifests, source: KoiNetNode) -> ManifestsPayload:
+    def fetch_manifests_handler(self, 
+        req: FetchManifests, 
+        source: KoiNetNode
+    ) -> ManifestsPayload:
         """Returns response to fetch manifests request."""
-        logger.info(f"Request to fetch manifests, allowed types {req.rid_types}, rids {req.rids}")
+        log.info(f"Request to fetch manifests, allowed types {req.rid_types}, rids {req.rids}")
         
         manifests: list[Manifest] = []
         not_found: list[RID] = []
@@ -50,9 +111,13 @@ class ResponseHandler:
         
         return ManifestsPayload(manifests=manifests, not_found=not_found)
         
-    def fetch_bundles(self, req: FetchBundles, source: KoiNetNode) -> BundlesPayload:
+    def fetch_bundles_handler(
+        self, 
+        req: FetchBundles, 
+        source: KoiNetNode
+    ) -> BundlesPayload:
         """Returns response to fetch bundles request."""
-        logger.info(f"Request to fetch bundles, requested rids {req.rids}")
+        log.info(f"Request to fetch bundles, requested rids {req.rids}")
         
         bundles: list[Bundle] = []
         not_found: list[RID] = []

koi_net/poller.py

@@ -1,15 +1,17 @@
 
 import time
-import logging
+import structlog
+
+from koi_net.interfaces.entrypoint import EntryPoint
 from .processor.kobj_queue import KobjQueue
 from .lifecycle import NodeLifecycle
 from .network.resolver import NetworkResolver
 from .config import NodeConfig
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
-class NodePoller:
+class NodePoller(EntryPoint):
     """Manages polling based event loop for partial nodes."""
     kobj_queue: KobjQueue
     lifecycle: NodeLifecycle
@@ -18,10 +20,10 @@ class NodePoller:
     
     def __init__(
         self,
-        kobj_queue: KobjQueue,
+        config: NodeConfig,
         lifecycle: NodeLifecycle,
+        kobj_queue: KobjQueue,
         resolver: NetworkResolver,
-        config: NodeConfig
     ):
         self.kobj_queue = kobj_queue
         self.lifecycle = lifecycle

koi_net/processor/event_worker.py

@@ -1,21 +1,20 @@
 import queue
 import traceback
 import time
-import logging
+import structlog
 
 from rid_lib.ext import Cache
 from rid_lib.types import KoiNetNode
 
 from koi_net.config import NodeConfig
-from koi_net.models import END, QueuedEvent
-from koi_net.network.event_queue import EventQueue
+from koi_net.network.event_queue import EventQueue, QueuedEvent
 from koi_net.network.request_handler import RequestHandler
-from koi_net.poll_event_buffer import PollEventBuffer
+from koi_net.network.poll_event_buffer import PollEventBuffer
 from koi_net.protocol.event import Event
 from koi_net.protocol.node import NodeProfile, NodeType
-from koi_net.worker import ThreadWorker
+from koi_net.interfaces.worker import ThreadWorker, STOP_WORKER
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class EventProcessingWorker(ThreadWorker):
@@ -28,10 +27,7 @@ class EventProcessingWorker(ThreadWorker):
         request_handler: RequestHandler,
         config: NodeConfig,
         cache: Cache,
-        poll_event_buf: PollEventBuffer,
-        queue_timeout: float = 0.1,
-        max_buf_len: int = 5,
-        max_wait_time: float = 1.0
+        poll_event_buf: PollEventBuffer
     ):
         self.event_queue = event_queue
         self.request_handler = request_handler
@@ -40,9 +36,9 @@ class EventProcessingWorker(ThreadWorker):
         self.cache = cache
         self.poll_event_buf = poll_event_buf
         
-        self.timeout = queue_timeout 
-        self.max_buf_len = max_buf_len
-        self.max_wait_time = max_wait_time
+        self.timeout: float = 0.1
+        self.max_buf_len: int = 5
+        self.max_wait_time: float = 1.0
         
         self.event_buffer = dict()
         self.buffer_times = dict()
@@ -74,25 +70,25 @@ class EventProcessingWorker(ThreadWorker):
             return True
         
         else:
-            logger.warning(f"Couldn't handle event {item.event!r} in queue, node {item.target!r} unknown to me")
+            log.warning(f"Couldn't handle event {item.event!r} in queue, node {item.target!r} unknown to me")
             return False
         
 
     def run(self):
-        logger.info("Started event worker")
+        log.info("Started event worker")
         while True:
             now = time.time()
             try:
                 item = self.event_queue.q.get(timeout=self.timeout)
                 
                 try:
-                    if item is END:
-                        logger.info("Received 'END' signal, flushing buffer...")
+                    if item is STOP_WORKER:
+                        log.info(f"Received 'STOP_WORKER' signal, flushing buffer...")
                         for target in self.event_buffer.keys():
                             self.flush_buffer(target, self.event_buffer[target])
                         return
                     
-                    logger.info(f"Dequeued {item.event!r} -> {item.target!r}")
+                    log.info(f"Dequeued {item.event!r} -> {item.target!r}")
                     
                     if not self.decide_event(item):
                         continue

koi_net/processor/{default_handlers.py → handlers.py}

@@ -1,6 +1,6 @@
 """Implementation of default knowledge handlers."""
 
-import logging
+import structlog
 from rid_lib.ext import Bundle
 from rid_lib.ext.utils import sha256_hash
 from rid_lib.types import KoiNetNode, KoiNetEdge
@@ -9,10 +9,11 @@ from .handler import KnowledgeHandler, HandlerType, STOP_CHAIN
 from .knowledge_object import KnowledgeObject
 from ..context import HandlerContext
 from ..protocol.event import Event, EventType
-from ..protocol.edge import EdgeProfile, EdgeStatus, EdgeType, generate_edge_bundle
+from ..protocol.edge import EdgeProfile, EdgeStatus, EdgeType
 from ..protocol.node import NodeProfile
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
+
 
 # RID handlers
 
@@ -24,7 +25,7 @@ def basic_rid_handler(ctx: HandlerContext, kobj: KnowledgeObject):
     RID is known to this node.
     """
     if (kobj.rid == ctx.identity.rid and kobj.source):
-        logger.debug("Don't let anyone else tell me who I am!")
+        log.debug("Don't let anyone else tell me who I am!")
         return STOP_CHAIN
     
     if kobj.event_type == EventType.FORGET:
@@ -45,17 +46,17 @@ def basic_manifest_handler(ctx: HandlerContext, kobj: KnowledgeObject):
 
     if prev_bundle:
         if kobj.manifest.sha256_hash == prev_bundle.manifest.sha256_hash:
-            logger.debug("Hash of incoming manifest is same as existing knowledge, ignoring")
+            log.debug("Hash of incoming manifest is same as existing knowledge, ignoring")
             return STOP_CHAIN
         if kobj.manifest.timestamp <= prev_bundle.manifest.timestamp:
-            logger.debug("Timestamp of incoming manifest is the same or older than existing knowledge, ignoring")
+            log.debug("Timestamp of incoming manifest is the same or older than existing knowledge, ignoring")
             return STOP_CHAIN
         
-        logger.debug("RID previously known to me, labeling as 'UPDATE'")
+        log.debug("RID previously known to me, labeling as 'UPDATE'")
         kobj.normalized_event_type = EventType.UPDATE
 
     else:
-        logger.debug("RID previously unknown to me, labeling as 'NEW'")
+        log.debug("RID previously unknown to me, labeling as 'NEW'")
         kobj.normalized_event_type = EventType.NEW
         
     return kobj
@@ -78,7 +79,7 @@ def secure_profile_handler(ctx: HandlerContext, kobj: KnowledgeObject):
     node_rid: KoiNetNode = kobj.rid
     
     if sha256_hash(node_profile.public_key) != node_rid.hash:
-        logger.warning(f"Public key hash mismatch for {node_rid!r}!")
+        log.warning(f"Public key hash mismatch for {node_rid!r}!")
         return STOP_CHAIN
 
 @KnowledgeHandler.create(
@@ -104,13 +105,13 @@ def edge_negotiation_handler(ctx: HandlerContext, kobj: KnowledgeObject):
         if edge_profile.status != EdgeStatus.PROPOSED:
             return
         
-        logger.debug("Handling edge negotiation")
+        log.debug("Handling edge negotiation")
         
         peer_rid = edge_profile.target
         peer_bundle = ctx.cache.read(peer_rid)
         
         if not peer_bundle:
-            logger.warning(f"Peer {peer_rid!r} unknown to me")
+            log.warning(f"Peer {peer_rid!r} unknown to me")
             return STOP_CHAIN
         
         peer_profile = peer_bundle.validate_contents(NodeProfile)
@@ -125,11 +126,11 @@ def edge_negotiation_handler(ctx: HandlerContext, kobj: KnowledgeObject):
         abort = False
         if (edge_profile.edge_type == EdgeType.WEBHOOK and 
             peer_profile.node_type == NodeType.PARTIAL):
-            logger.debug("Partial nodes cannot use webhooks")
+            log.debug("Partial nodes cannot use webhooks")
             abort = True
         
         if not set(edge_profile.rid_types).issubset(provided_events):
-            logger.debug("Requested RID types not provided by this node")
+            log.debug("Requested RID types not provided by this node")
             abort = True
         
         if abort:
@@ -139,7 +140,7 @@ def edge_negotiation_handler(ctx: HandlerContext, kobj: KnowledgeObject):
 
         else:
             # approve edge profile
-            logger.debug("Approving proposed edge")
+            log.debug("Approving proposed edge")
             edge_profile.status = EdgeStatus.APPROVED
             updated_bundle = Bundle.generate(kobj.rid, edge_profile.model_dump())
       
@@ -148,7 +149,7 @@ def edge_negotiation_handler(ctx: HandlerContext, kobj: KnowledgeObject):
               
     elif edge_profile.target == ctx.identity.rid:
         if edge_profile.status == EdgeStatus.APPROVED:
-            logger.debug("Edge approved by other node!")
+            log.debug("Edge approved by other node!")
 
 
 # Network handlers
@@ -176,8 +177,8 @@ def node_contact_handler(ctx: HandlerContext, kobj: KnowledgeObject):
     if not available_rid_types:
         return
     
-    logger.info("Identified a coordinator!")
-    logger.info("Proposing new edge")
+    log.info("Identified a coordinator!")
+    log.info("Proposing new edge")
     
     # already have an edge established
     edge_rid = ctx.graph.get_edge(
@@ -217,7 +218,7 @@ def node_contact_handler(ctx: HandlerContext, kobj: KnowledgeObject):
     edge_bundle = Bundle.generate(edge_rid, edge_profile.model_dump())
     ctx.kobj_queue.put_kobj(bundle=edge_bundle)
     
-    logger.info("Catching up on network state")
+    log.info("Catching up on network state")
     
     payload = ctx.request_handler.fetch_rids(
         node=kobj.rid, 
@@ -225,16 +226,16 @@ def node_contact_handler(ctx: HandlerContext, kobj: KnowledgeObject):
     )
     for rid in payload.rids:
         if rid == ctx.identity.rid:
-            logger.info("Skipping myself")
+            log.info("Skipping myself")
             continue
         if ctx.cache.exists(rid):
-            logger.info(f"Skipping known RID {rid!r}")
+            log.info(f"Skipping known RID {rid!r}")
             continue
         
         # marked as external since we are handling RIDs from another node
         # will fetch remotely instead of checking local cache
         ctx.kobj_queue.put_kobj(rid=rid, source=kobj.rid)
-    logger.info("Done")
+    log.info("Done")
     
 
 @KnowledgeHandler.create(HandlerType.Network)
@@ -260,12 +261,12 @@ def basic_network_output_filter(ctx: HandlerContext, kobj: KnowledgeObject):
             edge_profile = kobj.bundle.validate_contents(EdgeProfile)
             
             if edge_profile.source == ctx.identity.rid:
-                logger.debug(f"Adding edge target '{edge_profile.target!r}' to network targets")
+                log.debug(f"Adding edge target '{edge_profile.target!r}' to network targets")
                 kobj.network_targets.update([edge_profile.target])
                 involves_me = True
                 
             elif edge_profile.target == ctx.identity.rid:
-                logger.debug(f"Adding edge source '{edge_profile.source!r}' to network targets")
+                log.debug(f"Adding edge source '{edge_profile.source!r}' to network targets")
                 kobj.network_targets.update([edge_profile.source])
                 involves_me = True
     
@@ -276,7 +277,7 @@ def basic_network_output_filter(ctx: HandlerContext, kobj: KnowledgeObject):
             allowed_type=type(kobj.rid)
         )
         
-        logger.debug(f"Updating network targets with '{type(kobj.rid)}' subscribers: {subscribers}")
+        log.debug(f"Updating network targets with '{type(kobj.rid)}' subscribers: {subscribers}")
         kobj.network_targets.update(subscribers)
         
     return kobj
@@ -294,5 +295,5 @@ def forget_edge_on_node_deletion(ctx: HandlerContext, kobj: KnowledgeObject):
         edge_profile = edge_bundle.validate_contents(EdgeProfile)
         
         if kobj.rid in (edge_profile.source, edge_profile.target):
-            logger.debug("Identified edge with forgotten node")
+            log.debug("Identified edge with forgotten node")
             ctx.kobj_queue.put_kobj(rid=edge_rid, event_type=EventType.FORGET)

koi_net/processor/kobj_queue.py

@@ -1,4 +1,4 @@
-import logging
+import structlog
 from queue import Queue
 from rid_lib.core import RID
 from rid_lib.ext import Bundle, Manifest
@@ -6,7 +6,7 @@ from rid_lib.types import KoiNetNode
 from ..protocol.event import Event, EventType
 from .knowledge_object import KnowledgeObject
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class KobjQueue:
@@ -48,4 +48,4 @@ class KobjQueue:
             raise ValueError("One of 'rid', 'manifest', 'bundle', 'event', or 'kobj' must be provided")
         
         self.q.put(_kobj)
-        logger.debug(f"Queued {_kobj!r}")
+        log.debug(f"Queued {_kobj!r}")

koi_net/{kobj_worker.py → processor/kobj_worker.py}

@@ -1,38 +1,37 @@
 import queue
 import traceback
-import logging
+import structlog
 
-from koi_net.models import END
-from koi_net.processor.knowledge_pipeline import KnowledgePipeline
-from koi_net.processor.kobj_queue import KobjQueue
-from koi_net.worker import ThreadWorker
+from .pipeline import KnowledgePipeline
+from .kobj_queue import KobjQueue
+from koi_net.interfaces.worker import ThreadWorker, STOP_WORKER
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class KnowledgeProcessingWorker(ThreadWorker):
     def __init__(
         self,
         kobj_queue: KobjQueue,
-        pipeline: KnowledgePipeline,
-        timeout: float = 0.1
+        pipeline: KnowledgePipeline
     ):
         self.kobj_queue = kobj_queue
         self.pipeline = pipeline
-        self.timeout = timeout
+        self.timeout: float = 0.1
+
         super().__init__()
         
     def run(self):
-        logger.info("Started kobj worker")
+        log.info("Started kobj worker")
         while True:
             try:
                 item = self.kobj_queue.q.get(timeout=self.timeout)
                 try:
-                    if item is END:
-                        logger.info("Received 'END' signal, shutting down...")
+                    if item is STOP_WORKER:
+                        log.info("Received 'STOP_WORKER' signal, shutting down...")
                         return
                     
-                    logger.info(f"Dequeued {item!r}")
+                    log.info(f"Dequeued {item!r}")
                     
                     self.pipeline.process(item)
                 finally: