koi-net 1.0.0__py3-none-any.whl

koi_net/network/request_handler.py

@@ -0,0 +1,149 @@
+import logging
+import httpx
+from rid_lib import RID
+from rid_lib.ext import Cache
+from rid_lib.types.koi_net_node import KoiNetNode
+from ..protocol.api_models import (
+    RidsPayload,
+    ManifestsPayload,
+    BundlesPayload,
+    EventsPayload,
+    FetchRids,
+    FetchManifests,
+    FetchBundles,
+    PollEvents,
+    RequestModels,
+    ResponseModels
+)
+from ..protocol.consts import (
+    BROADCAST_EVENTS_PATH,
+    POLL_EVENTS_PATH,
+    FETCH_RIDS_PATH,
+    FETCH_MANIFESTS_PATH,
+    FETCH_BUNDLES_PATH
+)
+from ..protocol.node import NodeType
+from .graph import NetworkGraph
+
+
+logger = logging.getLogger(__name__)
+
+
+class RequestHandler:
+    """Handles making requests to other KOI nodes."""
+    
+    cache: Cache
+    graph: NetworkGraph
+    
+    def __init__(self, cache: Cache, graph: NetworkGraph):
+        self.cache = cache
+        self.graph = graph
+                
+    def make_request(
+        self, 
+        url: str, 
+        request: RequestModels,
+        response_model: type[ResponseModels] | None = None
+    ) -> ResponseModels | None:
+        logger.debug(f"Making request to {url}")
+        resp = httpx.post(
+            url=url,
+            data=request.model_dump_json()
+        )
+        if response_model:
+            return response_model.model_validate_json(resp.text)
+            
+    def get_url(self, node_rid: KoiNetNode, url: str) -> str:
+        """Retrieves URL of a node, or returns provided URL."""
+        
+        if not node_rid and not url:
+            raise ValueError("One of 'node_rid' and 'url' must be provided")
+        
+        if node_rid:
+            node_profile = self.graph.get_node_profile(node_rid)
+            if not node_profile:
+                raise Exception("Node not found")
+            if node_profile.node_type != NodeType.FULL:
+                raise Exception("Can't query partial node")
+            logger.debug(f"Resolved {node_rid!r} to {node_profile.base_url}")
+            return node_profile.base_url
+        else:
+            return url
+    
+    def broadcast_events(
+        self, 
+        node: RID = None, 
+        url: str = None, 
+        req: EventsPayload | None = None,
+        **kwargs
+    ) -> None:
+        """See protocol.api_models.EventsPayload for available kwargs."""
+        request = req or EventsPayload.model_validate(kwargs)
+        self.make_request(
+            self.get_url(node, url) + BROADCAST_EVENTS_PATH, request
+        )
+        logger.info(f"Broadcasted {len(request.events)} event(s) to {node or url!r}")
+        
+    def poll_events(
+        self, 
+        node: RID = None, 
+        url: str = None, 
+        req: PollEvents | None = None,
+        **kwargs
+    ) -> EventsPayload:
+        """See protocol.api_models.PollEvents for available kwargs."""
+        request = req or PollEvents.model_validate(kwargs)
+        resp = self.make_request(
+            self.get_url(node, url) + POLL_EVENTS_PATH, request,
+            response_model=EventsPayload
+        )
+        logger.info(f"Polled {len(resp.events)} events from {node or url!r}")
+        return resp
+        
+    def fetch_rids(
+        self, 
+        node: RID = None, 
+        url: str = None, 
+        req: FetchRids | None = None,
+        **kwargs
+    ) -> RidsPayload:
+        """See protocol.api_models.FetchRids for available kwargs."""
+        request = req or FetchRids.model_validate(kwargs)
+        resp = self.make_request(
+            self.get_url(node, url) + FETCH_RIDS_PATH, request,
+            response_model=RidsPayload
+        )
+        logger.info(f"Fetched {len(resp.rids)} RID(s) from {node or url!r}")
+        return resp
+                
+    def fetch_manifests(
+        self, 
+        node: RID = None, 
+        url: str = None, 
+        req: FetchManifests | None = None,
+        **kwargs
+    ) -> ManifestsPayload:
+        """See protocol.api_models.FetchManifests for available kwargs."""
+        request = req or FetchManifests.model_validate(kwargs)
+        resp = self.make_request(
+            self.get_url(node, url) + FETCH_MANIFESTS_PATH, request,
+            response_model=ManifestsPayload
+        )
+        logger.info(f"Fetched {len(resp.manifests)} manifest(s) from {node or url!r}")
+        return resp
+                
+    def fetch_bundles(
+        self, 
+        node: RID = None, 
+        url: str = None, 
+        req: FetchBundles | None = None,
+        **kwargs
+    ) -> BundlesPayload:
+        """See protocol.api_models.FetchBundles for available kwargs."""
+        request = req or FetchBundles.model_validate(kwargs)
+        resp = self.make_request(
+            self.get_url(node, url) + FETCH_BUNDLES_PATH, request,
+            response_model=BundlesPayload
+        )
+        logger.info(f"Fetched {len(resp.bundles)} bundle(s) from {node or url!r}")
+        return resp

koi_net/network/response_handler.py

@@ -0,0 +1,59 @@
+import logging
+from rid_lib import RID
+from rid_lib.ext import Manifest, Cache
+from rid_lib.ext.bundle import Bundle
+from ..protocol.api_models import (
+    RidsPayload,
+    ManifestsPayload,
+    BundlesPayload,
+    FetchRids,
+    FetchManifests,
+    FetchBundles,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ResponseHandler:
+    """Handles generating responses to requests from other KOI nodes."""
+    
+    cache: Cache
+    
+    def __init__(self, cache: Cache):
+        self.cache = cache
+        
+    def fetch_rids(self, req: FetchRids) -> RidsPayload:
+        logger.info(f"Request to fetch rids, allowed types {req.rid_types}")
+        rids = self.cache.list_rids(req.rid_types)
+        
+        return RidsPayload(rids=rids)
+        
+    def fetch_manifests(self, req: FetchManifests) -> ManifestsPayload:
+        logger.info(f"Request to fetch manifests, allowed types {req.rid_types}, rids {req.rids}")
+        
+        manifests: list[Manifest] = []
+        not_found: list[RID] = []
+        
+        for rid in (req.rids or self.cache.list_rids(req.rid_types)):
+            bundle = self.cache.read(rid)
+            if bundle:
+                manifests.append(bundle.manifest)
+            else:
+                not_found.append(rid)
+        
+        return ManifestsPayload(manifests=manifests, not_found=not_found)
+        
+    def fetch_bundles(self, req: FetchBundles) -> BundlesPayload:
+        logger.info(f"Request to fetch bundles, requested rids {req.rids}")
+        
+        bundles: list[Bundle] = []
+        not_found: list[RID] = []
+
+        for rid in req.rids:
+            bundle = self.cache.read(rid)
+            if bundle:
+                bundles.append(bundle)
+            else:
+                not_found.append(rid)
+            
+        return BundlesPayload(bundles=bundles, not_found=not_found)

koi_net/processor/__init__.py

@@ -0,0 +1 @@
+from .interface import ProcessorInterface

koi_net/processor/default_handlers.py

@@ -0,0 +1,220 @@
+"""Provides implementations of default knowledge handlers."""
+
+import logging
+from rid_lib.ext.bundle import Bundle
+from rid_lib.types import KoiNetNode, KoiNetEdge
+from koi_net.protocol.node import NodeType
+from .interface import ProcessorInterface
+from .handler import KnowledgeHandler, HandlerType, STOP_CHAIN
+from .knowledge_object import KnowledgeObject, KnowledgeSource
+from ..protocol.event import Event, EventType
+from ..protocol.edge import EdgeProfile, EdgeStatus, EdgeType
+from ..protocol.node import NodeProfile
+from ..protocol.helpers import generate_edge_bundle
+
+logger = logging.getLogger(__name__)
+
+# RID handlers
+
+@KnowledgeHandler.create(HandlerType.RID)
+def basic_rid_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
+    """Default RID handler.
+    
+    Blocks external events about this node. Allows `FORGET` events if RID is known to this node.
+    """
+    if (kobj.rid == processor.identity.rid and 
+        kobj.source == KnowledgeSource.External):
+        logger.debug("Don't let anyone else tell me who I am!")
+        return STOP_CHAIN
+    
+    if kobj.event_type == EventType.FORGET:
+        kobj.normalized_event_type = EventType.FORGET
+        return kobj
+
+# Manifest handlers
+
+@KnowledgeHandler.create(HandlerType.Manifest)
+def basic_manifest_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
+    """Default manifest handler.
+    
+    Blocks manifests with the same hash, or aren't newer than the cached version. Sets the normalized event type to `NEW` or `UPDATE` depending on whether the RID was previously known to this node.
+    """
+    prev_bundle = processor.cache.read(kobj.rid)
+
+    if prev_bundle:
+        if kobj.manifest.sha256_hash == prev_bundle.manifest.sha256_hash:
+            logger.debug("Hash of incoming manifest is same as existing knowledge, ignoring")
+            return STOP_CHAIN
+        if kobj.manifest.timestamp <= prev_bundle.manifest.timestamp:
+            logger.debug("Timestamp of incoming manifest is the same or older than existing knowledge, ignoring")
+            return STOP_CHAIN
+        
+        logger.debug("RID previously known to me, labeling as 'UPDATE'")
+        kobj.normalized_event_type = EventType.UPDATE
+
+    else:
+        logger.debug("RID previously unknown to me, labeling as 'NEW'")
+        kobj.normalized_event_type = EventType.NEW
+        
+    return kobj
+
+
+# Bundle handlers
+
+@KnowledgeHandler.create(
+    handler_type=HandlerType.Bundle, 
+    rid_types=[KoiNetEdge], 
+    source=KnowledgeSource.External,
+    event_types=[EventType.NEW, EventType.UPDATE])
+def edge_negotiation_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
+    """Handles basic edge negotiation process.
+    
+    Automatically approves proposed edges if they request RID types this node can provide (or KOI nodes/edges). Validates the edge type is allowed for the node type (partial nodes cannot use webhooks). If edge is invalid, a `FORGET` event is sent to the other node.
+    """
+    
+    edge_profile = EdgeProfile.model_validate(kobj.contents)
+
+    # indicates peer subscribing to me
+    if edge_profile.source == processor.identity.rid:     
+        if edge_profile.status != EdgeStatus.PROPOSED:
+            return
+        
+        logger.debug("Handling edge negotiation")
+        
+        peer_rid = edge_profile.target
+        peer_profile = processor.network.graph.get_node_profile(peer_rid)
+        
+        if not peer_profile:
+            logger.warning(f"Peer {peer_rid} unknown to me")
+            return STOP_CHAIN
+        
+        # explicitly provided event RID types and (self) node + edge objects
+        provided_events = (
+            *processor.identity.profile.provides.event,
+            KoiNetNode, KoiNetEdge
+        )
+        
+        
+        abort = False
+        if (edge_profile.edge_type == EdgeType.WEBHOOK and 
+            peer_profile.node_type == NodeType.PARTIAL):
+            logger.debug("Partial nodes cannot use webhooks")
+            abort = True
+        
+        if not set(edge_profile.rid_types).issubset(provided_events):
+            logger.debug("Requested RID types not provided by this node")
+            abort = True
+        
+        if abort:
+            event = Event.from_rid(EventType.FORGET, kobj.rid)
+            processor.network.push_event_to(event, peer_rid, flush=True)
+            return STOP_CHAIN
+
+        else:
+            # approve edge profile
+            logger.debug("Approving proposed edge")
+            edge_profile.status = EdgeStatus.APPROVED
+            updated_bundle = Bundle.generate(kobj.rid, edge_profile.model_dump())
+      
+            processor.handle(bundle=updated_bundle, event_type=EventType.UPDATE)
+            return
+              
+    elif edge_profile.target == processor.identity.rid:
+        if edge_profile.status == EdgeStatus.APPROVED:
+            logger.debug("Edge approved by other node!")
+
+
+# Network handlers
+
+@KnowledgeHandler.create(HandlerType.Network, rid_types=[KoiNetNode])
+def coordinator_contact(processor: ProcessorInterface, kobj: KnowledgeObject):
+    node_profile = kobj.bundle.validate_contents(NodeProfile)
+    
+    # looking for event provider of nodes
+    if KoiNetNode not in node_profile.provides.event:
+        return
+    
+    # prevents coordinators from attempting to form a self loop
+    if kobj.rid == processor.identity.rid:
+        return
+    
+    # already have an edge established
+    if processor.network.graph.get_edge_profile(
+        source=kobj.rid,
+        target=processor.identity.rid,
+    ) is not None:
+        return
+    
+    logger.info("Identified a coordinator!")
+    logger.info("Proposing new edge")
+    
+    if processor.identity.profile.node_type == NodeType.FULL:
+        edge_type = EdgeType.WEBHOOK
+    else:
+        edge_type = EdgeType.POLL
+    
+    # queued for processing
+    processor.handle(bundle=generate_edge_bundle(
+        source=kobj.rid,
+        target=processor.identity.rid,
+        edge_type=edge_type,
+        rid_types=[KoiNetNode]
+    ))
+    
+    logger.info("Catching up on network state")
+    
+    payload = processor.network.request_handler.fetch_rids(
+        node=kobj.rid, 
+        rid_types=[KoiNetNode]
+    )
+    for rid in payload.rids:
+        if rid == processor.identity.rid:
+            logger.info("Skipping myself")
+            continue
+        if processor.cache.exists(rid):
+            logger.info(f"Skipping known RID '{rid}'")
+            continue
+        
+        # marked as external since we are handling RIDs from another node
+        # will fetch remotely instead of checking local cache
+        processor.handle(rid=rid, source=KnowledgeSource.External)
+    logger.info("Done")
+    
+
+@KnowledgeHandler.create(HandlerType.Network)
+def basic_network_output_filter(processor: ProcessorInterface, kobj: KnowledgeObject):
+    """Default network handler.
+    
+    Allows broadcasting of all RID types this node is an event provider for (set in node profile), and other nodes have subscribed to. All nodes will also broadcast about their own (internally sourced) KOI node, and KOI edges that they are part of, regardless of their node profile configuration. Finally, nodes will also broadcast about edges to the other node involved (regardless of if they are subscribed)."""
+    
+    involves_me = False
+    if kobj.source == KnowledgeSource.Internal:
+        if (type(kobj.rid) == KoiNetNode):
+            if (kobj.rid == processor.identity.rid):
+                involves_me = True
+        
+        elif type(kobj.rid) == KoiNetEdge:
+            edge_profile = kobj.bundle.validate_contents(EdgeProfile)
+            
+            if edge_profile.source == processor.identity.rid:
+                logger.debug(f"Adding edge target '{edge_profile.target!r}' to network targets")
+                kobj.network_targets.update([edge_profile.target])
+                involves_me = True
+                
+            elif edge_profile.target == processor.identity.rid:
+                logger.debug(f"Adding edge source '{edge_profile.source!r}' to network targets")
+                kobj.network_targets.update([edge_profile.source])
+                involves_me = True
+    
+    if (type(kobj.rid) in processor.identity.profile.provides.event or involves_me):
+        # broadcasts to subscribers if I'm an event provider of this RID type OR it involves me
+        subscribers = processor.network.graph.get_neighbors(
+            direction="out",
+            allowed_type=type(kobj.rid)
+        )
+        
+        logger.debug(f"Updating network targets with '{type(kobj.rid)}' subscribers: {subscribers}")
+        kobj.network_targets.update(subscribers)
+        
+    return kobj
+            

koi_net/processor/handler.py

@@ -0,0 +1,59 @@
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Callable
+from rid_lib import RIDType
+
+from ..protocol.event import EventType
+from .knowledge_object import KnowledgeSource, KnowledgeEventType
+
+
+class StopChain:
+    """Class for a sentinel value by knowledge handlers."""
+    pass
+
+STOP_CHAIN = StopChain()
+
+
+class HandlerType(StrEnum):
+    """Types of handlers used in knowledge processing pipeline.
+    
+    - RID - provided RID; if event type is `FORGET`, this handler decides whether to delete the knowledge from the cache by setting the normalized event type to `FORGET`, otherwise this handler decides whether to validate the manifest (and fetch it if not provided).
+    - Manifest - provided RID, manifest; decides whether to validate the bundle (and fetch it if not provided).
+    - Bundle - provided RID, manifest, contents (bundle); decides whether to write knowledge to the cache by setting the normalized event type to `NEW` or `UPDATE`.
+    - Network - provided RID, manifest, contents (bundle); decides which nodes (if any) to broadcast an event about this knowledge to. (Note, if event type is `FORGET`, the manifest and contents will be retrieved from the local cache, and indicate the last state of the knowledge before it was deleted.)
+    - Final - provided RID, manifests, contents (bundle); final action taken after network broadcast.
+    """
+    
+    RID = "rid", 
+    Manifest = "manifest", 
+    Bundle = "bundle", 
+    Network = "network", 
+    Final = "final"
+
+@dataclass
+class KnowledgeHandler:
+    """Handles knowledge processing events of the provided types."""
+    
+    func: Callable
+    handler_type: HandlerType
+    rid_types: list[RIDType] | None
+    source: KnowledgeSource | None = None
+    event_types: list[KnowledgeEventType] | None = None
+    
+    @classmethod
+    def create(
+        cls,
+        handler_type: HandlerType,
+        rid_types: list[RIDType] | None = None,
+        source: KnowledgeSource | None = None,
+        event_types: list[KnowledgeEventType] | None = None
+    ):
+        """Special decorator that returns a KnowledgeHandler instead of a function.
+        
+        The function symbol will redefined as a `KnowledgeHandler`, which can be passed into the `ProcessorInterface` constructor. This is used to register default handlers.
+        """
+        def decorator(func: Callable) -> KnowledgeHandler:
+            handler = cls(func, handler_type, rid_types, source, event_types)
+            return handler
+        return decorator
+