koi-net 1.0.0b1__py3-none-any.whl → 1.0.0b3__py3-none-any.whl

koi_net/core.py

@@ -3,7 +3,7 @@ import httpx
 from rid_lib.ext import Cache, Bundle
 from .network import NetworkInterface
 from .processor import ProcessorInterface
-from .processor import default_handlers as _default_handlers
+from .processor import default_handlers
 from .processor.handler import KnowledgeHandler
 from .identity import NodeIdentity
 from .protocol.node import NodeProfile
@@ -12,13 +12,19 @@ from .protocol.event import Event, EventType
 logger = logging.getLogger(__name__)
 
 class NodeInterface:
+    cache: Cache
+    identity: NodeIdentity
+    network: NetworkInterface
+    processor: ProcessorInterface
+    first_contact: str
+    
     def __init__(
         self, 
         name: str,
         profile: NodeProfile,
         identity_file_path: str = "identity.json",
         first_contact: str | None = None,
-        default_handlers: list[KnowledgeHandler] | None = None,
+        handlers: list[KnowledgeHandler] | None = None,
         cache: Cache | None = None,
         network: NetworkInterface | None = None,
         processor: ProcessorInterface | None = None
@@ -39,9 +45,9 @@ class NodeInterface:
         )
         
         # pull all handlers defined in default_handlers module
-        if not default_handlers:
-            default_handlers = [
-                obj for obj in vars(_default_handlers).values() 
+        if not handlers:
+            handlers = [
+                obj for obj in vars(default_handlers).values() 
                 if isinstance(obj, KnowledgeHandler)
             ]
         
@@ -49,10 +55,16 @@ class NodeInterface:
             cache=self.cache, 
             network=self.network, 
             identity=self.identity, 
-            default_handlers=default_handlers
+            default_handlers=handlers
         )
     
-    def initialize(self):
+    def initialize(self) -> None:
+        """Initializes node, call on startup.
+        
+        Loads event queues into memory. Generates network graph from nodes and edges in cache. Processes any state changes of node bundle. Initiates handshake with first contact (if provided) if node doesn't have any neighbors.
+        """
+        self.network._load_event_queues()
+        
         self.network.graph.generate()
         
         self.processor.handle(
@@ -83,4 +95,8 @@ class NodeInterface:
             
                         
     def finalize(self):
-        self.network.save_event_queues()
+        """Finalizes node, call on shutdown.
+        
+        Saves event queues to storage.
+        """
+        self.network._save_event_queues()

koi_net/identity.py

@@ -13,6 +13,8 @@ class NodeIdentityModel(BaseModel):
     profile: NodeProfile
     
 class NodeIdentity:
+    """Represents a node's identity (RID, profile, bundle)."""
+    
     _identity: NodeIdentityModel
     file_path: str
     cache: Cache
@@ -24,6 +26,12 @@ class NodeIdentity:
         cache: Cache,
         file_path: str = "identity.json"
     ):
+        """Initializes node identity from a name and profile.
+        
+        Attempts to read identity from storage. If it doesn't already exist, a new RID is generated from the provided name, and that RID and profile are written to storage. Changes to the name or profile will update the stored identity.
+        
+        WARNING: If the name is changed, the RID will be overwritten which will have consequences for the rest of the network.
+        """
         self.cache = cache
         self.file_path = file_path
         

koi_net/network/graph.py

@@ -12,12 +12,19 @@ logger = logging.getLogger(__name__)
 
 
 class NetworkGraph:
+    """Graph functions for this node's view of its network."""
+    
+    cache: Cache
+    identity: NodeIdentity
+    dg: nx.DiGraph
+    
     def __init__(self, cache: Cache, identity: NodeIdentity):
         self.cache = cache
         self.dg = nx.DiGraph()
         self.identity = identity
         
     def generate(self):
+        """Generates directed graph from cached KOI nodes and edges."""
         logger.info("Generating network graph")
         self.dg.clear()
         for rid in self.cache.list_rids():
@@ -35,6 +42,7 @@ class NetworkGraph:
         logger.info("Done")
         
     def get_node_profile(self, rid: KoiNetNode) -> NodeProfile | None:
+        """Returns node profile given its RID."""
         bundle = self.cache.read(rid)
         if bundle:
             return bundle.validate_contents(NodeProfile)
@@ -45,6 +53,7 @@ class NetworkGraph:
         source: KoiNetNode | None = None, 
         target: KoiNetNode | None = None,
     ) -> EdgeProfile | None:
+        """Returns edge profile given its RID, or source and target node RIDs."""
         if source and target:
             if (source, target) not in self.dg.edges: return
             edge_data = self.dg.get_edge_data(source, target)
@@ -62,6 +71,9 @@ class NetworkGraph:
         self,
         direction: Literal["in", "out"] | None = None,
     ) -> list[KoiNetEdge]:
+        """Returns edges this node belongs to.
+        
+        All edges returned by default, specify `direction` to restrict to incoming or outgoing edges only."""
         
         edges = []
         if direction != "in":
@@ -88,6 +100,9 @@ class NetworkGraph:
         status: EdgeStatus | None = None,
         allowed_type: RIDType | None = None
     ) -> list[KoiNetNode]:
+        """Returns neighboring nodes this node shares an edge with.
+        
+        All neighboring nodes returned by default, specify `direction` to restrict to neighbors connected by incoming or outgoing edges only."""
         
         neighbors = []
         for edge_rid in self.get_edges(direction):

koi_net/network/interface.py

@@ -24,6 +24,8 @@ class EventQueueModel(BaseModel):
 type EventQueue = dict[RID, Queue[Event]]
 
 class NetworkInterface:
+    """A collection of functions and classes to interact with the KOI network."""
+    
     identity: NodeIdentity
     cache: Cache
     first_contact: str | None
@@ -45,15 +47,16 @@ class NetworkInterface:
         self.cache = cache
         self.first_contact = first_contact
         self.graph = NetworkGraph(cache, identity)
-        self.request_handler = RequestHandler(cache)
+        self.request_handler = RequestHandler(cache, self.graph)
         self.response_handler = ResponseHandler(cache)
         self.event_queues_file_path = file_path
         
         self.poll_event_queue = dict()
         self.webhook_event_queue = dict()
-        self.load_event_queues()
+        self._load_event_queues()
     
-    def load_event_queues(self):
+    def _load_event_queues(self):
+        """Loads event queues from storage."""
         try:
             with open(self.event_queues_file_path, "r") as f:
                 queues = EventQueueModel.model_validate_json(f.read())
@@ -71,7 +74,8 @@ class NetworkInterface:
         except FileNotFoundError:
             return
         
-    def save_event_queues(self):
+    def _save_event_queues(self):
+        """Writes event queues to storage."""
         events_model = EventQueueModel(
             poll={
                 node: list(queue.queue) 
@@ -92,6 +96,10 @@ class NetworkInterface:
             f.write(events_model.model_dump_json(indent=2))
     
     def push_event_to(self, event: Event, node: KoiNetNode, flush=False):
+        """Pushes event to queue of specified node.
+        
+        Event will be sent to webhook or poll queue depending on the node type and edge type of the specified node. If `flush` is set to `True`, the webhook queued will be flushed after pushing the event.
+        """
         logger.info(f"Pushing event {event.event_type} {event.rid} to {node}")
             
         node_profile = self.graph.get_node_profile(node)
@@ -121,7 +129,8 @@ class NetworkInterface:
         if flush and event_queue is self.webhook_event_queue:
             self.flush_webhook_queue(node)
             
-    def flush_queue(self, event_queue: EventQueue, node: KoiNetNode) -> list[Event]:
+    def _flush_queue(self, event_queue: EventQueue, node: KoiNetNode) -> list[Event]:
+        """Flushes a node's queue, returning list of events."""
         queue = event_queue.get(node)
         events = list()
         if queue:
@@ -133,22 +142,29 @@ class NetworkInterface:
         return events
     
     def flush_poll_queue(self, node: KoiNetNode) -> list[Event]:
+        """Flushes a node's poll queue, returning list of events."""
         logger.info(f"Flushing poll queue for {node}")
-        return self.flush_queue(self.poll_event_queue, node)
+        return self._flush_queue(self.poll_event_queue, node)
     
-    def flush_webhook_queue(self, node: RID):
+    def flush_webhook_queue(self, node: KoiNetNode):
+        """Flushes a node's webhook queue, and broadcasts events.
+        
+        If node profile is unknown, or node type is not `FULL`, this operation will fail silently. If the remote node cannot be reached, all events will be requeued.
+        """
+        
         logger.info(f"Flushing webhook queue for {node}")
         
         node_profile = self.graph.get_node_profile(node)
         
         if not node_profile:
             logger.warning(f"{node!r} not found")
+            return
         
         if node_profile.node_type != NodeType.FULL:
             logger.warning(f"{node!r} is a partial node!")
             return
         
-        events = self.flush_queue(self.webhook_event_queue, node)
+        events = self._flush_queue(self.webhook_event_queue, node)
         logger.info(f"Broadcasting {len(events)} events")
         
         try:  
@@ -159,10 +175,13 @@ class NetworkInterface:
                 self.push_event_to(event, node)
     
     def flush_all_webhook_queues(self):
+        """Flushes all nodes' webhook queues and broadcasts events."""
         for node in self.webhook_event_queue.keys():
             self.flush_webhook_queue(node)
             
-    def get_state_providers(self, rid_type: RIDType):
+    def get_state_providers(self, rid_type: RIDType) -> list[KoiNetNode]:
+        """Returns list of node RIDs which provide state for the specified RID type."""
+        
         logger.info(f"Looking for state providers of '{rid_type}'")
         provider_nodes = []
         for node_rid in self.cache.list_rids(rid_types=[KoiNetNode]):
@@ -177,6 +196,8 @@ class NetworkInterface:
         return provider_nodes
             
     def fetch_remote_bundle(self, rid: RID):
+        """Attempts to fetch a bundle by RID from known peer nodes."""
+        
         logger.info(f"Fetching remote bundle '{rid}'")
         remote_bundle = None
         for node_rid in self.get_state_providers(type(rid)):
@@ -194,6 +215,8 @@ class NetworkInterface:
         return remote_bundle
     
     def fetch_remote_manifest(self, rid: RID):
+        """Attempts to fetch a manifest by RID from known peer nodes."""
+        
         logger.info(f"Fetching remote manifest '{rid}'")
         remote_manifest = None
         for node_rid in self.get_state_providers(type(rid)):
@@ -211,9 +234,14 @@ class NetworkInterface:
         return remote_manifest
     
     def poll_neighbors(self) -> list[Event]:
+        """Polls all neighboring nodes and returns compiled list of events.
+        
+        If this node has no neighbors, it will instead attempt to poll the provided first contact URL.
+        """
+        
         neighbors = self.graph.get_neighbors()
         
-        if not neighbors:
+        if not neighbors and self.first_contact:
             logger.info("No neighbors found, polling first contact")
             try:
                 payload = self.request_handler.poll_events(

koi_net/network/request_handler.py

@@ -12,7 +12,9 @@ from ..protocol.api_models import (
     FetchRids,
     FetchManifests,
     FetchBundles,
-    PollEvents
+    PollEvents,
+    RequestModels,
+    ResponseModels
 )
 from ..protocol.consts import (
     BROADCAST_EVENTS_PATH,
@@ -21,85 +23,119 @@ from ..protocol.consts import (
     FETCH_MANIFESTS_PATH,
     FETCH_BUNDLES_PATH
 )
-from ..protocol.node import NodeProfile, NodeType
+from ..protocol.node import NodeType
+from .graph import NetworkGraph
 
 
 logger = logging.getLogger(__name__)
 
 
 class RequestHandler:
+    """Handles making requests to other KOI nodes."""
+    
     cache: Cache
+    graph: NetworkGraph
     
-    def __init__(self, cache: Cache):
+    def __init__(self, cache: Cache, graph: NetworkGraph):
         self.cache = cache
+        self.graph = graph
                 
-    def make_request(self, url, request: BaseModel) -> httpx.Response:
+    def make_request(
+        self, 
+        url: str, 
+        request: RequestModels,
+        response_model: type[ResponseModels] | None = None
+    ) -> ResponseModels | None:
         logger.info(f"Making request to {url}")
         resp = httpx.post(
             url=url,
             data=request.model_dump_json()
         )
-        return resp
+        if response_model:
+            return response_model.model_validate_json(resp.text)
             
     def get_url(self, node_rid: KoiNetNode, url: str) -> str:
+        """Retrieves URL of a node, or returns provided URL."""
+        
         if not node_rid and not url:
             raise ValueError("One of 'node_rid' and 'url' must be provided")
         
         if node_rid:
-            # can't access get_node rn
-            bundle = self.cache.read(node_rid)
-            node = NodeProfile.model_validate(bundle.contents)
-            if node.node_type != NodeType.FULL:
+            node_profile = self.graph.get_node_profile(node_rid)
+            if not node_profile:
+                raise Exception("Node not found")
+            if node_profile.node_type != NodeType.FULL:
                 raise Exception("Can't query partial node")
-            logger.info(f"Resolved {node_rid!r} to {node.base_url}")
-            return node.base_url
+            logger.info(f"Resolved {node_rid!r} to {node_profile.base_url}")
+            return node_profile.base_url
         else:
             return url
     
     def broadcast_events(
-        self, node: RID = None, url: str = None, **kwargs
+        self, 
+        node: RID = None, 
+        url: str = None, 
+        req: EventsPayload | None = None,
+        **kwargs
     ) -> None:
+        """See protocol.api_models.EventsPayload for available kwargs."""
         self.make_request(
             self.get_url(node, url) + BROADCAST_EVENTS_PATH,
-            EventsPayload.model_validate(kwargs)
+            req or EventsPayload.model_validate(kwargs)
         )
         
     def poll_events(
-        self, node: RID = None, url: str = None, **kwargs
-    ) -> EventsPayload:      
-        resp = self.make_request(
+        self, 
+        node: RID = None, 
+        url: str = None, 
+        req: PollEvents | None = None,
+        **kwargs
+    ) -> EventsPayload:
+        """See protocol.api_models.PollEvents for available kwargs."""
+        return self.make_request(
             self.get_url(node, url) + POLL_EVENTS_PATH,
-            PollEvents.model_validate(kwargs)
+            req or PollEvents.model_validate(kwargs),
+            response_model=EventsPayload
         )
-        
-        return EventsPayload.model_validate_json(resp.text)
-    
+            
     def fetch_rids(
-        self, node: RID = None, url: str = None, **kwargs
-    ) -> RidsPayload:      
-        resp = self.make_request(
+        self, 
+        node: RID = None, 
+        url: str = None, 
+        req: FetchRids | None = None,
+        **kwargs
+    ) -> RidsPayload:
+        """See protocol.api_models.FetchRids for available kwargs."""
+        return self.make_request(
             self.get_url(node, url) + FETCH_RIDS_PATH,
-            FetchRids.model_validate(kwargs)
+            req or FetchRids.model_validate(kwargs),
+            response_model=RidsPayload
         )
-        
-        return RidsPayload.model_validate_json(resp.text)
-        
+                
     def fetch_manifests(
-        self, node: RID = None, url: str = None, **kwargs
-    ) -> ManifestsPayload:    
-        resp = self.make_request(
+        self, 
+        node: RID = None, 
+        url: str = None, 
+        req: FetchManifests | None = None,
+        **kwargs
+    ) -> ManifestsPayload:
+        """See protocol.api_models.FetchManifests for available kwargs."""
+        return self.make_request(
             self.get_url(node, url) + FETCH_MANIFESTS_PATH,
-            FetchManifests.model_validate(kwargs)
+            req or FetchManifests.model_validate(kwargs),
+            response_model=ManifestsPayload
         )
-        
-        return ManifestsPayload.model_validate_json(resp.text)
-        
+                
     def fetch_bundles(
-        self, node: RID = None, url: str = None, **kwargs
-    ) -> BundlesPayload:        
-        resp = self.make_request(
+        self, 
+        node: RID = None, 
+        url: str = None, 
+        req: FetchBundles | None = None,
+        **kwargs
+    ) -> BundlesPayload:
+        """See protocol.api_models.FetchBundles for available kwargs."""
+        return self.make_request(
             self.get_url(node, url) + FETCH_BUNDLES_PATH,
-            FetchBundles.model_validate(kwargs)
-        )
-        
-        return BundlesPayload.model_validate_json(resp.text)
+            req or FetchBundles.model_validate(kwargs),
+            response_model=BundlesPayload
+        )

koi_net/network/response_handler.py

@@ -15,6 +15,8 @@ logger = logging.getLogger(__name__)
 
 
 class ResponseHandler:
+    """Handles generating responses to requests from other KOI nodes."""
+    
     cache: Cache
     
     def __init__(self, cache: Cache):

koi_net/processor/default_handlers.py

@@ -1,9 +1,11 @@
+"""Provides implementations of default knowledge handlers."""
+
 import logging
 from rid_lib.ext.bundle import Bundle
 from rid_lib.types import KoiNetNode, KoiNetEdge
 from koi_net.protocol.node import NodeType
 from .interface import ProcessorInterface
-from .handler import HandlerType, STOP_CHAIN
+from .handler import KnowledgeHandler, HandlerType, STOP_CHAIN
 from .knowledge_object import KnowledgeObject, KnowledgeSource
 from ..protocol.event import Event, EventType
 from ..protocol.edge import EdgeProfile, EdgeStatus, EdgeType
@@ -12,14 +14,18 @@ logger = logging.getLogger(__name__)
 
 # RID handlers
 
-@ProcessorInterface.as_handler(handler_type=HandlerType.RID)
+@KnowledgeHandler.create(HandlerType.RID)
 def basic_rid_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
+    """Default RID handler.
+    
+    Blocks external events about this node. Allows `FORGET` events if RID is known to this node.
+    """
     if (kobj.rid == processor.identity.rid and 
         kobj.source == KnowledgeSource.External):
         logger.info("Don't let anyone else tell me who I am!")
         return STOP_CHAIN
     
-    if kobj.event_type == EventType.FORGET:        
+    if kobj.event_type == EventType.FORGET:
         if processor.cache.exists(kobj.rid):
             logger.info("Allowing cache forget")
             kobj.normalized_event_type = EventType.FORGET
@@ -32,8 +38,12 @@ def basic_rid_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
 
 # Manifest handlers
 
-@ProcessorInterface.as_handler(handler_type=HandlerType.Manifest)
+@KnowledgeHandler.create(HandlerType.Manifest)
 def basic_state_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
+    """Default manifest handler.
+    
+    Blocks manifests with the same hash, or aren't newer than the cached version. Sets the normalized event type to `NEW` or `UPDATE` depending on whether the RID was previously known to this node.
+    """
     prev_bundle = processor.cache.read(kobj.rid)
 
     if prev_bundle:
@@ -56,8 +66,13 @@ def basic_state_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
 
 # Bundle handlers
 
-@ProcessorInterface.as_handler(HandlerType.Bundle, rid_types=[KoiNetEdge])
+@KnowledgeHandler.create(HandlerType.Bundle, rid_types=[KoiNetEdge])
 def edge_negotiation_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
+    """Handles basic edge negotiation process.
+    
+    Automatically approves proposed edges if they request RID types this node can provide (or KOI nodes/edges). Validates the edge type is allowed for the node type (partial nodes cannot use webhooks). If edge is invalid, a `FORGET` event is sent to the other node.
+    """
+    
     edge_profile = EdgeProfile.model_validate(kobj.contents)
     
     # only want to handle external knowledge events (not edges this node created)
@@ -116,8 +131,12 @@ def edge_negotiation_handler(processor: ProcessorInterface, kobj: KnowledgeObjec
 
 # Network handlers
 
-@ProcessorInterface.as_handler(HandlerType.Network)
+@KnowledgeHandler.create(HandlerType.Network)
 def basic_network_output_filter(processor: ProcessorInterface, kobj: KnowledgeObject):
+    """Default network handler.
+    
+    Allows broadcasting of all RID types this node is an event provider for (set in node profile), and other nodes have subscribed to. All nodes will also broadcast about their own (internally sourced) KOI node, and KOI edges that they are part of, regardless of their node profile configuration. Finally, nodes will also broadcast about edges to the other node involved (regardless of if they are subscribed)."""
+    
     involves_me = False
     if kobj.source == KnowledgeSource.Internal:
         if (type(kobj.rid) == KoiNetNode):

koi_net/processor/handler.py

@@ -4,19 +4,49 @@ from typing import Callable
 from rid_lib import RIDType
 
 
-# sentinel
-STOP_CHAIN = object()
+class StopChain:
+    """Class for a sentinel value by knowledge handlers."""
+    pass
+
+STOP_CHAIN = StopChain()
+
 
 class HandlerType(StrEnum):
-    RID = "rid", # guaranteed RID - decides whether validate manifest OR cache delete
-    Manifest = "manifest", # guaranteed Manifest - decides whether to validate bundle
-    Bundle = "bundle", # guaranteed Bundle - decides whether to write to cache
-    Network = "network", # guaranteed Bundle, after cache write/delete  - decides network targets
-    Final = "final" # guaranteed Bundle, after network push - final action
+    """Types of handlers used in knowledge processing pipeline.
+    
+    - RID - provided RID; if event type is `FORGET`, this handler decides whether to delete the knowledge from the cache by setting the normalized event type to `FORGET`, otherwise this handler decides whether to validate the manifest (and fetch it if not provided).
+    - Manifest - provided RID, manifest; decides whether to validate the bundle (and fetch it if not provided).
+    - Bundle - provided RID, manifest, contents (bundle); decides whether to write knowledge to the cache by setting the normalized event type to `NEW` or `UPDATE`.
+    - Network - provided RID, manifest, contents (bundle); decides which nodes (if any) to broadcast an event about this knowledge to. (Note, if event type is `FORGET`, the manifest and contents will be retrieved from the local cache, and indicate the last state of the knowledge before it was deleted.)
+    - Final - provided RID, manifests, contents (bundle); final action taken after network broadcast.
+    """
+    
+    RID = "rid", 
+    Manifest = "manifest", 
+    Bundle = "bundle", 
+    Network = "network", 
+    Final = "final"
 
 @dataclass
 class KnowledgeHandler:
+    """Handles knowledge processing events of the provided types."""
+    
     func: Callable
     handler_type: HandlerType
     rid_types: list[RIDType] | None
+    
+    @classmethod
+    def create(
+        cls,
+        handler_type: HandlerType,
+        rid_types: list[RIDType] | None = None
+    ):
+        """Special decorator that returns a KnowledgeHandler instead of a function.
+        
+        The function symbol will redefined as a `KnowledgeHandler`, which can be passed into the `ProcessorInterface` constructor. This is used to register default handlers.
+        """
+        def decorator(func: Callable) -> KnowledgeHandler:
+            handler = cls(func, handler_type, rid_types)
+            return handler
+        return decorator