koi-net 1.2.4__py3-none-any.whl

koi_net/network/request_handler.py

@@ -0,0 +1,244 @@
+from functools import wraps
+
+import structlog
+import httpx
+from rid_lib import RID
+from rid_lib.ext import Cache
+from rid_lib.types import KoiNetNode
+from pydantic import ValidationError
+
+from ..identity import NodeIdentity
+from ..protocol.api_models import (
+    RidsPayload,
+    ManifestsPayload,
+    BundlesPayload,
+    EventsPayload,
+    FetchRids,
+    FetchManifests,
+    FetchBundles,
+    PollEvents,
+    RequestModels,
+    ResponseModels,
+    ErrorResponse
+)
+from ..protocol.consts import (
+    BROADCAST_EVENTS_PATH,
+    POLL_EVENTS_PATH,
+    FETCH_RIDS_PATH,
+    FETCH_MANIFESTS_PATH,
+    FETCH_BUNDLES_PATH
+)
+from ..protocol.errors import ErrorType
+from ..protocol.node import NodeProfile, NodeType
+from ..protocol.model_map import API_MODEL_MAP
+from ..secure_manager import SecureManager
+from ..exceptions import (
+    RemoteInvalidKeyError,
+    RemoteInvalidSignatureError,
+    RemoteInvalidTargetError,
+    RequestError,
+    SelfRequestError,
+    PartialNodeQueryError,
+    NodeNotFoundError,
+    ServerError,
+    TransportError,
+    RemoteUnknownNodeError
+)
+from .error_handler import ErrorHandler
+
+log = structlog.stdlib.get_logger()
+
+
+def report_exception(func):
+    """Logs request errors as warnings."""
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except RequestError as err:
+            log.warning(err)
+            raise
+    return wrapper
+
+class RequestHandler:
+    """Handles making requests to other KOI nodes."""
+    
+    cache: Cache
+    identity: NodeIdentity
+    secure_manager: SecureManager
+    error_handler: ErrorHandler
+    
+    def __init__(
+        self, 
+        cache: Cache,
+        identity: NodeIdentity,
+        secure_manager: SecureManager,
+        error_handler: ErrorHandler
+    ):
+        self.cache = cache
+        self.identity = identity
+        self.secure_manager = secure_manager
+        self.error_handler = error_handler
+    
+    def get_base_url(self, node_rid: KoiNetNode) -> str:
+        """Retrieves URL of a node from its RID."""
+        
+        node_bundle = self.cache.read(node_rid)
+        if node_bundle:
+            node_profile = node_bundle.validate_contents(NodeProfile)
+            if node_profile.node_type != NodeType.FULL:
+                raise PartialNodeQueryError("Partial nodes don't have URLs")
+            node_url = node_profile.base_url
+        
+        elif node_rid == self.identity.config.koi_net.first_contact.rid:
+            node_url = self.identity.config.koi_net.first_contact.url
+        
+        else:
+            raise NodeNotFoundError(f"URL not found for {node_rid!r}")
+        
+        log.debug(f"Resolved {node_rid!r} to {node_url}")
+        return node_url
+    
+    @report_exception
+    def make_request(
+        self,
+        node: KoiNetNode,
+        path: str, 
+        request: RequestModels,
+    ) -> ResponseModels | None:
+        """Makes a request to a node."""
+        if node == self.identity.rid:
+            raise SelfRequestError("Don't talk to yourself")
+        
+        url = self.get_base_url(node) + path
+        log.info(f"Making request to {url}")
+    
+        signed_envelope = self.secure_manager.create_envelope(
+            payload=request,
+            target=node
+        )
+        
+        data = signed_envelope.model_dump_json(exclude_none=True)
+        
+        try:
+            result = httpx.post(url, data=data)
+            result.raise_for_status()
+            self.error_handler.reset_timeout_counter(node)
+            
+        except httpx.RequestError as e:
+            log.debug("Failed to connect")
+            self.error_handler.handle_connection_error(node)
+            raise TransportError(e)
+        
+        except httpx.HTTPStatusError:
+            """Possible errors:
+            
+            4xx - KOI-net protocol error, validate body
+            404/405 - not implementing endpoints, or misconfigured URL
+            
+            500 - internal server error
+            """
+            try:
+                resp = ErrorResponse.model_validate_json(result.text)
+                self.error_handler.handle_protocol_error(resp.error, node)
+                
+                match resp.error:
+                    case ErrorType.UnknownNode:
+                        raise RemoteUnknownNodeError(f"Peer couldn't resolve this node's RID")
+                    case ErrorType.InvalidKey:
+                        raise RemoteInvalidKeyError(f"Peer marked this node's public key as invalid")
+                    case ErrorType.InvalidSignature:
+                        raise RemoteInvalidSignatureError("Peer marked envelope signature as invalid")
+                    case ErrorType.InvalidTarget:
+                        raise RemoteInvalidTargetError("Envelope target is not the peer node")
+            
+            except ValidationError as e:
+                raise ServerError(e)
+            
+        
+        resp_env_model = API_MODEL_MAP[path].response_envelope
+        if not resp_env_model:
+            return
+        
+        try:
+            resp_envelope = resp_env_model.model_validate_json(result.text)
+        except ValidationError as e:
+            raise ServerError(e)
+        
+        self.secure_manager.validate_envelope(resp_envelope)
+        
+        return resp_envelope.payload
+    
+    def broadcast_events(
+        self, 
+        node: RID, 
+        req: EventsPayload | None = None,
+        **kwargs
+    ) -> None:
+        """Broadcasts events to a node.
+        
+        Pass `EventsPayload` object, or see `protocol.api_models.EventsPayload` for available kwargs.
+        """
+        request = req or EventsPayload.model_validate(kwargs)
+        self.make_request(node, BROADCAST_EVENTS_PATH, request)
+        log.info(f"Broadcasted {len(request.events)} event(s) to {node!r}")
+        
+    def poll_events(
+        self, 
+        node: RID, 
+        req: PollEvents | None = None,
+        **kwargs
+    ) -> EventsPayload:
+        """Polls events from a node.
+        
+        Pass `PollEvents` object as `req` or fields as kwargs.
+        """
+        request = req or PollEvents.model_validate(kwargs)
+        resp = self.make_request(node, POLL_EVENTS_PATH, request)
+        log.info(f"Polled {len(resp.events)} events from {node!r}")
+        return resp
+        
+    def fetch_rids(
+        self, 
+        node: RID, 
+        req: FetchRids | None = None,
+        **kwargs
+    ) -> RidsPayload:
+        """Fetches RIDs from a node.
+        
+        Pass `FetchRids` object as `req` or fields as kwargs.
+        """
+        request = req or FetchRids.model_validate(kwargs)
+        resp = self.make_request(node, FETCH_RIDS_PATH, request)
+        log.info(f"Fetched {len(resp.rids)} RID(s) from {node!r}")
+        return resp
+                
+    def fetch_manifests(
+        self, 
+        node: RID, 
+        req: FetchManifests | None = None,
+        **kwargs
+    ) -> ManifestsPayload:
+        """Fetches manifests from a node.
+        
+        Pass `FetchManifests` object as `req` or fields as kwargs.
+        """
+        request = req or FetchManifests.model_validate(kwargs)
+        resp = self.make_request(node, FETCH_MANIFESTS_PATH, request)
+        log.info(f"Fetched {len(resp.manifests)} manifest(s) from {node!r}")
+        return resp
+                
+    def fetch_bundles(
+        self, 
+        node: RID, 
+        req: FetchBundles | None = None,
+        **kwargs
+    ) -> BundlesPayload:
+        """Fetches bundles from a node.
+        
+        Pass `FetchBundles` object as `req` or fields as kwargs.
+        """
+        request = req or FetchBundles.model_validate(kwargs)
+        resp = self.make_request(node, FETCH_BUNDLES_PATH, request)
+        log.info(f"Fetched {len(resp.bundles)} bundle(s) from {node!r}")
+        return resp

koi_net/network/resolver.py

@@ -0,0 +1,152 @@
+import structlog
+from rid_lib import RID
+from rid_lib.core import RIDType
+from rid_lib.ext import Cache, Bundle
+from rid_lib.types import KoiNetNode
+
+from .graph import NetworkGraph
+from .request_handler import RequestHandler
+from ..protocol.node import NodeProfile, NodeType
+from ..protocol.event import Event
+from ..identity import NodeIdentity
+from ..config.core import NodeConfig
+from ..exceptions import RequestError
+
+log = structlog.stdlib.get_logger()
+
+
+class NetworkResolver:
+    """Handles resolving nodes or knowledge objects from the network."""
+    
+    config: NodeConfig    
+    identity: NodeIdentity
+    cache: Cache
+    graph: NetworkGraph
+    request_handler: RequestHandler
+    
+    def __init__(
+        self, 
+        config: NodeConfig,
+        cache: Cache, 
+        identity: NodeIdentity,
+        graph: NetworkGraph,
+        request_handler: RequestHandler,
+    ):
+        self.config = config
+        self.identity = identity
+        self.cache = cache
+        self.graph = graph
+        self.request_handler = request_handler
+        
+        self.poll_event_queue = dict()
+        self.webhook_event_queue = dict()
+    
+    def get_state_providers(self, rid_type: RIDType) -> list[KoiNetNode]:
+        """Returns list of node RIDs which provide state for specified RID type."""
+        
+        log.debug(f"Looking for state providers of {rid_type}")
+        provider_nodes = []
+        for node_rid in self.cache.list_rids(rid_types=[KoiNetNode]):
+            if node_rid == self.identity.rid:
+                continue
+            
+            node_bundle = self.cache.read(node_rid)
+            node_profile = node_bundle.validate_contents(NodeProfile)
+            
+            if node_profile.node_type != NodeType.FULL:
+                continue
+            
+            if rid_type not in node_profile.provides.state:
+                continue
+            
+            provider_nodes.append(node_rid)
+        
+        if provider_nodes:
+            log.debug(f"Found provider(s) {provider_nodes}")
+        else:
+            log.debug("Failed to find providers")
+            
+        return provider_nodes
+            
+    def fetch_remote_bundle(self, rid: RID) -> tuple[Bundle | None, KoiNetNode | None]:
+        """Attempts to fetch a bundle by RID from known peer nodes."""
+        
+        log.debug(f"Fetching remote bundle {rid!r}")
+        remote_bundle, node_rid = None, None
+        for node_rid in self.get_state_providers(type(rid)):
+            try:
+                payload = self.request_handler.fetch_bundles(
+                    node=node_rid, rids=[rid])
+            except RequestError:
+                continue
+            
+            if payload.bundles:
+                remote_bundle = payload.bundles[0]
+                log.debug(f"Got bundle from {node_rid!r}")
+                break
+        
+        if not remote_bundle:
+            log.warning("Failed to fetch remote bundle")
+            
+        return remote_bundle, node_rid
+    
+    def fetch_remote_manifest(self, rid: RID) -> tuple[Bundle | None, KoiNetNode | None]:
+        """Attempts to fetch a manifest by RID from known peer nodes."""
+        
+        log.debug(f"Fetching remote manifest {rid!r}")
+        remote_manifest, node_rid = None, None
+        for node_rid in self.get_state_providers(type(rid)):
+            try:
+                payload = self.request_handler.fetch_manifests(
+                    node=node_rid, rids=[rid])
+            except RequestError:
+                continue
+            
+            if payload.manifests:
+                remote_manifest = payload.manifests[0]
+                log.debug(f"Got bundle from {node_rid!r}")
+                break
+        
+        if not remote_manifest:
+            log.warning("Failed to fetch remote bundle")
+            
+        return remote_manifest, node_rid
+    
+    def poll_neighbors(self) -> dict[KoiNetNode, list[Event]]:
+        """Polls all neighbor nodes and returns compiled list of events.
+        
+        Neighbor nodes include any node this node shares an edge with,
+        or the first contact, if no neighbors are found.
+        
+        NOTE: This function does not poll nodes that don't share edges
+        with this node. Events sent by non neighboring nodes will not
+        be polled.
+        """
+        
+        neighbors: list[KoiNetNode] = []
+        for node_rid in self.graph.get_neighbors():
+            node_bundle = self.cache.read(node_rid)
+            if not node_bundle: 
+                continue
+            node_profile = node_bundle.validate_contents(NodeProfile)
+            if node_profile.node_type != NodeType.FULL: 
+                continue
+            neighbors.append(node_rid)
+            
+        if not neighbors and self.config.koi_net.first_contact.rid:
+            neighbors.append(self.config.koi_net.first_contact.rid)
+        
+        event_dict: dict[KoiNetNode, list[Event]] = {}
+        for node_rid in neighbors:
+            try:
+                payload = self.request_handler.poll_events(
+                    node=node_rid, 
+                    rid=self.identity.rid
+                )
+            except RequestError:
+                continue
+                
+            log.debug(f"Received {len(payload.events)} events from {node_rid!r}")
+            event_dict[node_rid] = payload.events
+            
+        return event_dict

koi_net/network/response_handler.py

@@ -0,0 +1,130 @@
+import structlog
+from rid_lib import RID
+from rid_lib.types import KoiNetNode
+from rid_lib.ext import Manifest, Cache
+from rid_lib.ext.bundle import Bundle
+
+from ..processor.kobj_queue import KobjQueue
+from ..protocol.consts import BROADCAST_EVENTS_PATH, FETCH_BUNDLES_PATH, FETCH_MANIFESTS_PATH, FETCH_RIDS_PATH, POLL_EVENTS_PATH
+from ..protocol.envelope import SignedEnvelope
+from ..secure_manager import SecureManager
+from .event_buffer import EventBuffer
+
+
+from ..protocol.api_models import (
+    EventsPayload,
+    PollEvents,
+    RidsPayload,
+    ManifestsPayload,
+    BundlesPayload,
+    FetchRids,
+    FetchManifests,
+    FetchBundles,
+)
+
+log = structlog.stdlib.get_logger()
+
+
+class ResponseHandler:
+    """Handles generating responses to requests from other KOI nodes."""
+    
+    cache: Cache
+    kobj_queue: KobjQueue
+    poll_event_buf: EventBuffer
+    
+    def __init__(
+        self, 
+        cache: Cache,
+        kobj_queue: KobjQueue,
+        poll_event_buf: EventBuffer,
+        secure_manager: SecureManager
+    ):
+        self.cache = cache
+        self.kobj_queue = kobj_queue
+        self.poll_event_buf = poll_event_buf
+        self.secure_manager = secure_manager
+    
+    def handle_response(self, path: str, req: SignedEnvelope):
+        self.secure_manager.validate_envelope(req)
+        
+        response_map = {
+            BROADCAST_EVENTS_PATH: self.broadcast_events_handler,
+            POLL_EVENTS_PATH: self.poll_events_handler,
+            FETCH_RIDS_PATH: self.fetch_rids_handler,
+            FETCH_MANIFESTS_PATH: self.fetch_manifests_handler,
+            FETCH_BUNDLES_PATH: self.fetch_bundles_handler
+        }
+        
+        response = response_map[path](req.payload, req.source_node)
+        
+        if response is None:
+            return
+        
+        return self.secure_manager.create_envelope(
+            payload=response,
+            target=req.source_node
+        )
+        
+    def broadcast_events_handler(self, req: EventsPayload, source: KoiNetNode):
+        log.info(f"Request to broadcast events, received {len(req.events)} event(s)")
+        
+        for event in req.events:
+            self.kobj_queue.push(event=event, source=source)
+        
+    def poll_events_handler(
+        self, 
+        req: PollEvents, 
+        source: KoiNetNode
+    ) -> EventsPayload:
+        events = self.poll_event_buf.flush(source, limit=req.limit)
+        log.info(f"Request to poll events, returning {len(events)} event(s)")
+        return EventsPayload(events=events)
+        
+    def fetch_rids_handler(
+        self, 
+        req: FetchRids, 
+        source: KoiNetNode
+    ) -> RidsPayload:
+        """Returns response to fetch RIDs request."""
+        rids = self.cache.list_rids(req.rid_types)
+        log.info(f"Request to fetch rids, allowed types {req.rid_types}, returning {len(rids)} RID(s)")
+        return RidsPayload(rids=rids)
+        
+    def fetch_manifests_handler(
+        self, 
+        req: FetchManifests, 
+        source: KoiNetNode
+    ) -> ManifestsPayload:
+        """Returns response to fetch manifests request."""        
+        manifests: list[Manifest] = []
+        not_found: list[RID] = []
+        
+        for rid in (req.rids or self.cache.list_rids(req.rid_types)):
+            bundle = self.cache.read(rid)
+            if bundle:
+                manifests.append(bundle.manifest)
+            else:
+                not_found.append(rid)
+        
+        log.info(f"Request to fetch manifests, allowed types {req.rid_types}, rids {req.rids}, returning {len(manifests)} manifest(s)")
+        return ManifestsPayload(manifests=manifests, not_found=not_found)
+        
+    def fetch_bundles_handler(
+        self, 
+        req: FetchBundles, 
+        source: KoiNetNode
+    ) -> BundlesPayload:
+        """Returns response to fetch bundles request."""
+        
+        bundles: list[Bundle] = []
+        not_found: list[RID] = []
+
+        for rid in req.rids:
+            bundle = self.cache.read(rid)
+            if bundle:
+                bundles.append(bundle)
+            else:
+                not_found.append(rid)
+                
+        log.info(f"Request to fetch bundles, requested rids {req.rids}, returning {len(bundles)} bundle(s)")
+        return BundlesPayload(bundles=bundles, not_found=not_found)

koi_net/processor/context.py

@@ -0,0 +1,36 @@
+from typing import TYPE_CHECKING
+from dataclasses import dataclass, field
+
+from rid_lib.ext import Cache
+
+from ..network.resolver import NetworkResolver
+from ..config.core import NodeConfig
+from ..config.loader import ConfigLoader
+from ..network.graph import NetworkGraph
+from ..network.event_queue import EventQueue
+from ..network.request_handler import RequestHandler
+from ..identity import NodeIdentity
+from .kobj_queue import KobjQueue
+
+if TYPE_CHECKING:
+    from ..effector import Effector
+
+
+@dataclass
+class HandlerContext:
+    """Context object provides knowledge handlers access to other components."""
+    
+    identity: NodeIdentity
+    config: NodeConfig
+    config_loader: ConfigLoader
+    cache: Cache
+    event_queue: EventQueue
+    kobj_queue: KobjQueue
+    graph: NetworkGraph
+    request_handler: RequestHandler
+    resolver: NetworkResolver
+    effector: "Effector" = field(init=False)
+    
+    def set_effector(self, effector: "Effector"):
+        """Post initialization injection of effector component."""
+        self.effector = effector

koi_net/processor/handler.py

@@ -0,0 +1,61 @@
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Callable
+from rid_lib.core import RIDType
+
+from ..protocol.event import EventType
+from .knowledge_object import KnowledgeObject
+from .context import HandlerContext
+
+
+class StopChain:
+    """Class for STOP_CHAIN sentinel returned by knowledge handlers."""
+    pass
+
+STOP_CHAIN = StopChain()
+
+
+class HandlerType(StrEnum):
+    """Types of handlers used in knowledge processing pipeline.
+    
+    - RID - provided RID; if event type is `FORGET`, this handler decides whether to delete the knowledge from the cache by setting the normalized event type to `FORGET`, otherwise this handler decides whether to validate the manifest (and fetch it if not provided).
+    - Manifest - provided RID, manifest; decides whether to validate the bundle (and fetch it if not provided).
+    - Bundle - provided RID, manifest, contents (bundle); decides whether to write knowledge to the cache by setting the normalized event type to `NEW` or `UPDATE`.
+    - Network - provided RID, manifest, contents (bundle); decides which nodes (if any) to broadcast an event about this knowledge to. (Note, if event type is `FORGET`, the manifest and contents will be retrieved from the local cache, and indicate the last state of the knowledge before it was deleted.)
+    - Final - provided RID, manifests, contents (bundle); final action taken after network broadcast.
+    """
+    
+    RID = "rid", 
+    Manifest = "manifest", 
+    Bundle = "bundle", 
+    Network = "network", 
+    Final = "final"
+
+@dataclass
+class KnowledgeHandler:
+    """Handles knowledge processing events of the provided types."""
+    
+    func: Callable[[HandlerContext, KnowledgeObject], None | KnowledgeObject | StopChain]
+    handler_type: HandlerType
+    rid_types: tuple[RIDType] = ()
+    event_types: tuple[EventType | None] = ()
+    
+    def __call__(
+        self, 
+        ctx: HandlerContext, 
+        kobj: KnowledgeObject
+    ) -> None | KnowledgeObject | StopChain:
+        return self.func(ctx, kobj)
+    
+    @classmethod
+    def create(
+        cls,
+        handler_type: HandlerType,
+        rid_types: tuple[RIDType] = (),
+        event_types: tuple[EventType | None] = ()
+    ):
+        """Decorator wraps a function, returns a KnowledgeHandler."""
+        def decorator(func: Callable) -> KnowledgeHandler:
+            handler = cls(func, handler_type, rid_types, event_types)
+            return handler
+        return decorator