koi-net 1.0.0b1__py3-none-any.whl → 1.0.0b2__py3-none-any.whl

koi_net/core.py

@@ -3,7 +3,7 @@ import httpx
 from rid_lib.ext import Cache, Bundle
 from .network import NetworkInterface
 from .processor import ProcessorInterface
-from .processor import default_handlers as _default_handlers
+from .processor import default_handlers
 from .processor.handler import KnowledgeHandler
 from .identity import NodeIdentity
 from .protocol.node import NodeProfile
@@ -12,13 +12,19 @@ from .protocol.event import Event, EventType
 logger = logging.getLogger(__name__)
 
 class NodeInterface:
+    cache: Cache
+    identity: NodeIdentity
+    network: NetworkInterface
+    processor: ProcessorInterface
+    first_contact: str
+    
     def __init__(
         self, 
         name: str,
         profile: NodeProfile,
         identity_file_path: str = "identity.json",
         first_contact: str | None = None,
-        default_handlers: list[KnowledgeHandler] | None = None,
+        handlers: list[KnowledgeHandler] | None = None,
         cache: Cache | None = None,
         network: NetworkInterface | None = None,
         processor: ProcessorInterface | None = None
@@ -39,9 +45,9 @@ class NodeInterface:
         )
         
         # pull all handlers defined in default_handlers module
-        if not default_handlers:
-            default_handlers = [
-                obj for obj in vars(_default_handlers).values() 
+        if not handlers:
+            handlers = [
+                obj for obj in vars(default_handlers).values() 
                 if isinstance(obj, KnowledgeHandler)
             ]
         
@@ -49,10 +55,16 @@ class NodeInterface:
             cache=self.cache, 
             network=self.network, 
             identity=self.identity, 
-            default_handlers=default_handlers
+            default_handlers=handlers
         )
     
-    def initialize(self):
+    def initialize(self) -> None:
+        """Initializes node, call on startup.
+        
+        Loads event queues into memory. Generates network graph from nodes and edges in cache. Processes any state changes of node bundle. Initiates handshake with first contact (if provided) if node doesn't have any neighbors.
+        """
+        self.network._load_event_queues()
+        
         self.network.graph.generate()
         
         self.processor.handle(
@@ -83,4 +95,8 @@ class NodeInterface:
             
                         
     def finalize(self):
-        self.network.save_event_queues()
+        """Finalizes node, call on shutdown.
+        
+        Saves event queues to storage.
+        """
+        self.network._save_event_queues()

koi_net/identity.py

@@ -13,6 +13,8 @@ class NodeIdentityModel(BaseModel):
     profile: NodeProfile
     
 class NodeIdentity:
+    """Represents a node's identity (RID, profile, bundle)."""
+    
     _identity: NodeIdentityModel
     file_path: str
     cache: Cache
@@ -24,6 +26,12 @@ class NodeIdentity:
         cache: Cache,
         file_path: str = "identity.json"
     ):
+        """Initializes node identity from a name and profile.
+        
+        Attempts to read identity from storage. If it doesn't already exist, a new RID is generated from the provided name, and that RID and profile are written to storage. Changes to the name or profile will update the stored identity.
+        
+        WARNING: If the name is changed, the RID will be overwritten which will have consequences for the rest of the network.
+        """
         self.cache = cache
         self.file_path = file_path
         

koi_net/network/graph.py

@@ -12,12 +12,15 @@ logger = logging.getLogger(__name__)
 
 
 class NetworkGraph:
+    """Graph functions for this node's view of its network."""
+    
     def __init__(self, cache: Cache, identity: NodeIdentity):
         self.cache = cache
         self.dg = nx.DiGraph()
         self.identity = identity
         
     def generate(self):
+        """Generates directed graph from cached KOI nodes and edges."""
         logger.info("Generating network graph")
         self.dg.clear()
         for rid in self.cache.list_rids():
@@ -35,6 +38,7 @@ class NetworkGraph:
         logger.info("Done")
         
     def get_node_profile(self, rid: KoiNetNode) -> NodeProfile | None:
+        """Returns node profile given its RID."""
         bundle = self.cache.read(rid)
         if bundle:
             return bundle.validate_contents(NodeProfile)
@@ -45,6 +49,7 @@ class NetworkGraph:
         source: KoiNetNode | None = None, 
         target: KoiNetNode | None = None,
     ) -> EdgeProfile | None:
+        """Returns edge profile given its RID, or source and target node RIDs."""
         if source and target:
             if (source, target) not in self.dg.edges: return
             edge_data = self.dg.get_edge_data(source, target)
@@ -62,6 +67,9 @@ class NetworkGraph:
         self,
         direction: Literal["in", "out"] | None = None,
     ) -> list[KoiNetEdge]:
+        """Returns edges this node belongs to.
+        
+        All edges returned by default, specify `direction` to restrict to incoming or outgoing edges only."""
         
         edges = []
         if direction != "in":
@@ -88,6 +96,9 @@ class NetworkGraph:
         status: EdgeStatus | None = None,
         allowed_type: RIDType | None = None
     ) -> list[KoiNetNode]:
+        """Returns neighboring nodes this node shares an edge with.
+        
+        All neighboring nodes returned by default, specify `direction` to restrict to neighbors connected by incoming or outgoing edges only."""
         
         neighbors = []
         for edge_rid in self.get_edges(direction):

koi_net/network/interface.py

@@ -24,6 +24,8 @@ class EventQueueModel(BaseModel):
 type EventQueue = dict[RID, Queue[Event]]
 
 class NetworkInterface:
+    """A collection of functions and classes to interact with the KOI network."""
+    
     identity: NodeIdentity
     cache: Cache
     first_contact: str | None
@@ -45,15 +47,16 @@ class NetworkInterface:
         self.cache = cache
         self.first_contact = first_contact
         self.graph = NetworkGraph(cache, identity)
-        self.request_handler = RequestHandler(cache)
+        self.request_handler = RequestHandler(cache, self.graph)
         self.response_handler = ResponseHandler(cache)
         self.event_queues_file_path = file_path
         
         self.poll_event_queue = dict()
         self.webhook_event_queue = dict()
-        self.load_event_queues()
+        self._load_event_queues()
     
-    def load_event_queues(self):
+    def _load_event_queues(self):
+        """Loads event queues from storage."""
         try:
             with open(self.event_queues_file_path, "r") as f:
                 queues = EventQueueModel.model_validate_json(f.read())
@@ -71,7 +74,8 @@ class NetworkInterface:
         except FileNotFoundError:
             return
         
-    def save_event_queues(self):
+    def _save_event_queues(self):
+        """Writes event queues to storage."""
         events_model = EventQueueModel(
             poll={
                 node: list(queue.queue) 
@@ -92,6 +96,10 @@ class NetworkInterface:
             f.write(events_model.model_dump_json(indent=2))
     
     def push_event_to(self, event: Event, node: KoiNetNode, flush=False):
+        """Pushes event to queue of specified node.
+        
+        Event will be sent to webhook or poll queue depending on the node type and edge type of the specified node. If `flush` is set to `True`, the webhook queued will be flushed after pushing the event.
+        """
         logger.info(f"Pushing event {event.event_type} {event.rid} to {node}")
             
         node_profile = self.graph.get_node_profile(node)
@@ -121,7 +129,8 @@ class NetworkInterface:
         if flush and event_queue is self.webhook_event_queue:
             self.flush_webhook_queue(node)
             
-    def flush_queue(self, event_queue: EventQueue, node: KoiNetNode) -> list[Event]:
+    def _flush_queue(self, event_queue: EventQueue, node: KoiNetNode) -> list[Event]:
+        """Flushes a node's queue, returning list of events."""
         queue = event_queue.get(node)
         events = list()
         if queue:
@@ -133,22 +142,29 @@ class NetworkInterface:
         return events
     
     def flush_poll_queue(self, node: KoiNetNode) -> list[Event]:
+        """Flushes a node's poll queue, returning list of events."""
         logger.info(f"Flushing poll queue for {node}")
-        return self.flush_queue(self.poll_event_queue, node)
+        return self._flush_queue(self.poll_event_queue, node)
     
-    def flush_webhook_queue(self, node: RID):
+    def flush_webhook_queue(self, node: KoiNetNode):
+        """Flushes a node's webhook queue, and broadcasts events.
+        
+        If node profile is unknown, or node type is not `FULL`, this operation will fail silently. If the remote node cannot be reached, all events will be requeued.
+        """
+        
         logger.info(f"Flushing webhook queue for {node}")
         
         node_profile = self.graph.get_node_profile(node)
         
         if not node_profile:
             logger.warning(f"{node!r} not found")
+            return
         
         if node_profile.node_type != NodeType.FULL:
             logger.warning(f"{node!r} is a partial node!")
             return
         
-        events = self.flush_queue(self.webhook_event_queue, node)
+        events = self._flush_queue(self.webhook_event_queue, node)
         logger.info(f"Broadcasting {len(events)} events")
         
         try:  
@@ -159,10 +175,13 @@ class NetworkInterface:
                 self.push_event_to(event, node)
     
     def flush_all_webhook_queues(self):
+        """Flushes all nodes' webhook queues and broadcasts events."""
         for node in self.webhook_event_queue.keys():
             self.flush_webhook_queue(node)
             
-    def get_state_providers(self, rid_type: RIDType):
+    def get_state_providers(self, rid_type: RIDType) -> list[KoiNetNode]:
+        """Returns list of node RIDs which provide state for the specified RID type."""
+        
         logger.info(f"Looking for state providers of '{rid_type}'")
         provider_nodes = []
         for node_rid in self.cache.list_rids(rid_types=[KoiNetNode]):
@@ -177,6 +196,8 @@ class NetworkInterface:
         return provider_nodes
             
     def fetch_remote_bundle(self, rid: RID):
+        """Attempts to fetch a bundle by RID from known peer nodes."""
+        
         logger.info(f"Fetching remote bundle '{rid}'")
         remote_bundle = None
         for node_rid in self.get_state_providers(type(rid)):
@@ -194,6 +215,8 @@ class NetworkInterface:
         return remote_bundle
     
     def fetch_remote_manifest(self, rid: RID):
+        """Attempts to fetch a manifest by RID from known peer nodes."""
+        
         logger.info(f"Fetching remote manifest '{rid}'")
         remote_manifest = None
         for node_rid in self.get_state_providers(type(rid)):
@@ -211,9 +234,14 @@ class NetworkInterface:
         return remote_manifest
     
     def poll_neighbors(self) -> list[Event]:
+        """Polls all neighboring nodes and returns compiled list of events.
+        
+        If this node has no neighbors, it will instead attempt to poll the provided first contact URL.
+        """
+        
         neighbors = self.graph.get_neighbors()
         
-        if not neighbors:
+        if not neighbors and self.first_contact:
             logger.info("No neighbors found, polling first contact")
             try:
                 payload = self.request_handler.poll_events(

koi_net/network/request_handler.py

@@ -21,17 +21,22 @@ from ..protocol.consts import (
     FETCH_MANIFESTS_PATH,
     FETCH_BUNDLES_PATH
 )
-from ..protocol.node import NodeProfile, NodeType
+from ..protocol.node import NodeType
+from .graph import NetworkGraph
 
 
 logger = logging.getLogger(__name__)
 
 
 class RequestHandler:
+    """Handles making requests to other KOI nodes."""
+    
     cache: Cache
+    graph: NetworkGraph
     
-    def __init__(self, cache: Cache):
+    def __init__(self, cache: Cache, graph: NetworkGraph):
         self.cache = cache
+        self.graph = graph
                 
     def make_request(self, url, request: BaseModel) -> httpx.Response:
         logger.info(f"Making request to {url}")
@@ -42,23 +47,26 @@ class RequestHandler:
         return resp
             
     def get_url(self, node_rid: KoiNetNode, url: str) -> str:
+        """Retrieves URL of a node, or returns provided URL."""
+        
         if not node_rid and not url:
             raise ValueError("One of 'node_rid' and 'url' must be provided")
         
         if node_rid:
-            # can't access get_node rn
-            bundle = self.cache.read(node_rid)
-            node = NodeProfile.model_validate(bundle.contents)
-            if node.node_type != NodeType.FULL:
+            node_profile = self.graph.get_node_profile(node_rid)
+            if not node_profile:
+                raise Exception("Node not found")
+            if node_profile.node_type != NodeType.FULL:
                 raise Exception("Can't query partial node")
-            logger.info(f"Resolved {node_rid!r} to {node.base_url}")
-            return node.base_url
+            logger.info(f"Resolved {node_rid!r} to {node_profile.base_url}")
+            return node_profile.base_url
         else:
             return url
     
     def broadcast_events(
         self, node: RID = None, url: str = None, **kwargs
     ) -> None:
+        """See protocol.api_models.EventsPayload for available kwargs."""
         self.make_request(
             self.get_url(node, url) + BROADCAST_EVENTS_PATH,
             EventsPayload.model_validate(kwargs)
@@ -66,7 +74,8 @@ class RequestHandler:
         
     def poll_events(
         self, node: RID = None, url: str = None, **kwargs
-    ) -> EventsPayload:      
+    ) -> EventsPayload:
+        """See protocol.api_models.PollEvents for available kwargs."""
         resp = self.make_request(
             self.get_url(node, url) + POLL_EVENTS_PATH,
             PollEvents.model_validate(kwargs)
@@ -76,7 +85,8 @@ class RequestHandler:
     
     def fetch_rids(
         self, node: RID = None, url: str = None, **kwargs
-    ) -> RidsPayload:      
+    ) -> RidsPayload:
+        """See protocol.api_models.FetchRids for available kwargs."""
         resp = self.make_request(
             self.get_url(node, url) + FETCH_RIDS_PATH,
             FetchRids.model_validate(kwargs)
@@ -86,7 +96,8 @@ class RequestHandler:
         
     def fetch_manifests(
         self, node: RID = None, url: str = None, **kwargs
-    ) -> ManifestsPayload:    
+    ) -> ManifestsPayload:
+        """See protocol.api_models.FetchManifests for available kwargs."""
         resp = self.make_request(
             self.get_url(node, url) + FETCH_MANIFESTS_PATH,
             FetchManifests.model_validate(kwargs)
@@ -96,7 +107,8 @@ class RequestHandler:
         
     def fetch_bundles(
         self, node: RID = None, url: str = None, **kwargs
-    ) -> BundlesPayload:        
+    ) -> BundlesPayload:
+        """See protocol.api_models.FetchBundles for available kwargs."""
         resp = self.make_request(
             self.get_url(node, url) + FETCH_BUNDLES_PATH,
             FetchBundles.model_validate(kwargs)

koi_net/network/response_handler.py

@@ -15,6 +15,8 @@ logger = logging.getLogger(__name__)
 
 
 class ResponseHandler:
+    """Handles generating responses to requests from other KOI nodes."""
+    
     cache: Cache
     
     def __init__(self, cache: Cache):

koi_net/processor/default_handlers.py

@@ -1,3 +1,5 @@
+"""Provides implementations of default knowledge handlers."""
+
 import logging
 from rid_lib.ext.bundle import Bundle
 from rid_lib.types import KoiNetNode, KoiNetEdge
@@ -14,6 +16,10 @@ logger = logging.getLogger(__name__)
 
 @ProcessorInterface.as_handler(handler_type=HandlerType.RID)
 def basic_rid_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
+    """Default RID handler.
+    
+    Blocks external events about this node. Allows `FORGET` events if RID is known to this node.
+    """
     if (kobj.rid == processor.identity.rid and 
         kobj.source == KnowledgeSource.External):
         logger.info("Don't let anyone else tell me who I am!")
@@ -34,6 +40,10 @@ def basic_rid_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
 
 @ProcessorInterface.as_handler(handler_type=HandlerType.Manifest)
 def basic_state_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
+    """Default manifest handler.
+    
+    Blocks manifests with the same hash, or aren't newer than the cached version. Sets the normalized event type to `NEW` or `UPDATE` depending on whether the RID was previously known to this node.
+    """
     prev_bundle = processor.cache.read(kobj.rid)
 
     if prev_bundle:
@@ -58,6 +68,11 @@ def basic_state_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
 
 @ProcessorInterface.as_handler(HandlerType.Bundle, rid_types=[KoiNetEdge])
 def edge_negotiation_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
+    """Handles basic edge negotiation process.
+    
+    Automatically approves proposed edges if they request RID types this node can provide (or KOI nodes/edges). Validates the edge type is allowed for the node type (partial nodes cannot use webhooks). If edge is invalid, a `FORGET` event is sent to the other node.
+    """
+    
     edge_profile = EdgeProfile.model_validate(kobj.contents)
     
     # only want to handle external knowledge events (not edges this node created)
@@ -118,6 +133,10 @@ def edge_negotiation_handler(processor: ProcessorInterface, kobj: KnowledgeObjec
 
 @ProcessorInterface.as_handler(HandlerType.Network)
 def basic_network_output_filter(processor: ProcessorInterface, kobj: KnowledgeObject):
+    """Default network handler.
+    
+    Allows broadcasting of all RID types this node is an event provider for (set in node profile), and other nodes have subscribed to. All nodes will also broadcast about their own (internally sourced) KOI node, and KOI edges that they are part of, regardless of their node profile configuration. Finally, nodes will also broadcast about edges to the other node involved (regardless of if they are subscribed)."""
+    
     involves_me = False
     if kobj.source == KnowledgeSource.Internal:
         if (type(kobj.rid) == KoiNetNode):

koi_net/processor/handler.py

@@ -4,18 +4,33 @@ from typing import Callable
 from rid_lib import RIDType
 
 
-# sentinel
-STOP_CHAIN = object()
+class StopChain:
+    """Class for a sentinel value by knowledge handlers."""
+    pass
+
+STOP_CHAIN = StopChain()
+
 
 class HandlerType(StrEnum):
-    RID = "rid", # guaranteed RID - decides whether validate manifest OR cache delete
-    Manifest = "manifest", # guaranteed Manifest - decides whether to validate bundle
-    Bundle = "bundle", # guaranteed Bundle - decides whether to write to cache
-    Network = "network", # guaranteed Bundle, after cache write/delete  - decides network targets
-    Final = "final" # guaranteed Bundle, after network push - final action
+    """Types of handlers used in knowledge processing pipeline.
+    
+    - RID - provided RID; if event type is `FORGET`, this handler decides whether to delete the knowledge from the cache by setting the normalized event type to `FORGET`, otherwise this handler decides whether to validate the manifest (and fetch it if not provided).
+    - Manifest - provided RID, manifest; decides whether to validate the bundle (and fetch it if not provided).
+    - Bundle - provided RID, manifest, contents (bundle); decides whether to write knowledge to the cache by setting the normalized event type to `NEW` or `UPDATE`.
+    - Network - provided RID, manifest, contents (bundle); decides which nodes (if any) to broadcast an event about this knowledge to. (Note, if event type is `FORGET`, the manifest and contents will be retrieved from the local cache, and indicate the last state of the knowledge before it was deleted.)
+    - Final - provided RID, manifests, contents (bundle); final action taken after network broadcast.
+    """
+    
+    RID = "rid", 
+    Manifest = "manifest", 
+    Bundle = "bundle", 
+    Network = "network", 
+    Final = "final"
 
 @dataclass
 class KnowledgeHandler:
+    """Handles knowledge processing events of the provided types."""
+    
     func: Callable
     handler_type: HandlerType
     rid_types: list[RIDType] | None

koi_net/processor/interface.py

@@ -11,7 +11,8 @@ from ..protocol.event import Event, EventType
 from .handler import (
     KnowledgeHandler, 
     HandlerType, 
-    STOP_CHAIN
+    STOP_CHAIN,
+    StopChain
 )
 from .knowledge_object import (
     KnowledgeObject,
@@ -23,6 +24,8 @@ logger = logging.getLogger(__name__)
 
 
 class ProcessorInterface:
+    """Provides access to this node's knowledge processing pipeline."""
+    
     cache: Cache
     network: NetworkInterface
     identity: NodeIdentity
@@ -48,7 +51,10 @@ class ProcessorInterface:
         handler_type: HandlerType,
         rid_types: list[RIDType] | None = None
     ):
-        """Special decorator that returns a Handler instead of a function."""
+        """Special decorator that returns a handler instead of a function.
+        
+        The function symbol will redefined as a `KnowledgeHandler`, which can be passed into the `ProcessorInterface` constructor. This is used to register the default handlers.
+        """
         def decorator(func: Callable) -> KnowledgeHandler:
             handler = KnowledgeHandler(func, handler_type, rid_types, )
             return handler
@@ -59,7 +65,7 @@ class ProcessorInterface:
         handler_type: HandlerType,
         rid_types: list[RIDType] | None = None
     ):
-        """Assigns decorated function as handler for this Processor."""
+        """Assigns decorated function as handler for this processor."""
         def decorator(func: Callable) -> Callable:
             handler = KnowledgeHandler(func, handler_type, rid_types)
             self.handlers.append(handler)
@@ -70,7 +76,17 @@ class ProcessorInterface:
         self, 
         handler_type: HandlerType,
         kobj: KnowledgeObject
-    ):
+    ) -> KnowledgeObject | StopChain:
+        """Calls handlers of provided type, chaining their inputs and outputs together.
+        
+        The knowledge object provided when this function is called will be passed to the first handler. A handler may return one of three types: 
+        - `KnowledgeObject` - to modify the knowledge object for the next handler in the chain
+        - `None` - to keep the same knowledge object for the next handler in the chain
+        - `STOP_CHAIN` - to stop the handler chain and immediately exit the processing pipeline
+        
+        Handlers will only be called in the chain if their handler and RID type match that of the inputted knowledge object. 
+        """
+        
         for handler in self.handlers:
             if handler_type != handler.handler_type: 
                 continue
@@ -98,7 +114,19 @@ class ProcessorInterface:
         return kobj
 
         
-    def handle_kobj(self, kobj: KnowledgeObject):
+    def handle_kobj(self, kobj: KnowledgeObject) -> None:
+        """Sends provided knowledge obejct through knowledge processing pipeline.
+        
+        Handler chains are called in between major events in the pipeline, indicated by their handler type. Each handler type is guaranteed to have access to certain knowledge, and may affect a subsequent action in the pipeline. The five handler types are as follows:
+        - RID - provided RID; if event type is `FORGET`, this handler decides whether to delete the knowledge from the cache by setting the normalized event type to `FORGET`, otherwise this handler decides whether to validate the manifest (and fetch it if not provided).
+        - Manifest - provided RID, manifest; decides whether to validate the bundle (and fetch it if not provided).
+        - Bundle - provided RID, manifest, contents (bundle); decides whether to write knowledge to the cache by setting the normalized event type to `NEW` or `UPDATE`.
+        - Network - provided RID, manifest, contents (bundle); decides which nodes (if any) to broadcast an event about this knowledge to. (Note, if event type is `FORGET`, the manifest and contents will be retrieved from the local cache, and indicate the last state of the knowledge before it was deleted.)
+        - Final - provided RID, manifests, contents (bundle); final action taken after network broadcast.
+        
+        The pipeline may be stopped by any point by a single handler returning the `STOP_CHAIN` sentinel. In that case, the process will exit immediately. Further handlers of that type and later handler chains will not be called.
+        """
+        
         logger.info(f"Handling {kobj!r}")
         kobj = self.call_handler_chain(HandlerType.RID, kobj)
         if kobj is STOP_CHAIN: return
@@ -185,6 +213,7 @@ class ProcessorInterface:
         kobj = self.call_handler_chain(HandlerType.Final, kobj)
             
     def queue_kobj(self, kobj: KnowledgeObject, flush: bool = False):
+        """Queues a knowledge object to be put processed in the pipeline."""
         self.kobj_queue.put(kobj)
         logger.info(f"Queued {kobj!r}")
         
@@ -192,6 +221,7 @@ class ProcessorInterface:
             self.flush_kobj_queue()
                 
     def flush_kobj_queue(self):
+        """Flushes all knowledge objects from queue and processes them."""
         while not self.kobj_queue.empty():
             kobj = self.kobj_queue.get()
             logger.info(f"Dequeued {kobj!r}")
@@ -208,6 +238,10 @@ class ProcessorInterface:
         source: KnowledgeSource = KnowledgeSource.Internal,
         flush: bool = False
     ):
+        """Queues provided knowledge to be handled by processing pipeline.
+        
+        Knowledge may take the form of an RID, manifest, bundle, or event (with an optional event type for non event objects). All objects will be normalized into knowledge objects and queued. If `flush` is `True`, the queue will be flushed immediately after adding the new knowledge.
+        """
         if rid:
             kobj = KnowledgeObject.from_rid(rid, event_type, source)
         elif manifest:

koi_net/processor/knowledge_object.py

@@ -14,6 +14,18 @@ class KnowledgeSource(StrEnum):
     External = "EXTERNAL"
 
 class KnowledgeObject(BaseModel):
+    """A normalized knowledge representation for internal processing.
+    
+    Capable of representing an RID, manifest, bundle, or event. Contains three additional fields use for decision making in the knowledge processing pipeline. 
+    
+    The source indicates whether this object was generated by this node, or sourced from another node in the network. 
+    
+    The normalized event type indicates how the knowledge object is viewed from the perspective of this node, and what cache actions will take place. `NEW`, `UPDATE` -> cache write, `FORGET` -> cache delete, `None` -> no cache action.
+    
+    The network targets indicate other nodes in the network this knowledge object will be sent to. The event sent to them will be constructed from this knowledge object's RID, manifest, contents, and normalized event type.
+    
+    Constructors are provided to create a knowledge object from an RID, manifest, bundle, or event.
+    """
     rid: RID
     manifest: Manifest | None = None
     contents: dict | None = None
@@ -93,12 +105,19 @@ class KnowledgeObject(BaseModel):
     
     @property
     def normalized_event(self):
-        if not self.normalized_event_type:
+        if self.normalized_event_type is None:
             raise ValueError("Internal event's normalized event type is None, cannot convert to Event")
         
-        return Event(
-            rid=self.rid,
-            event_type=self.normalized_event_type,
-            manifest=self.manifest,
-            contents=self.contents
-        )
+        elif self.normalized_event_type == EventType.FORGET:
+            return Event(
+                rid=self.rid,
+                event_type=EventType.FORGET
+            )
+        
+        else:    
+            return Event(
+                rid=self.rid,
+                event_type=self.normalized_event_type,
+                manifest=self.manifest,
+                contents=self.contents
+            )

koi_net/protocol/api_models.py

@@ -1,3 +1,5 @@
+"""Pydantic models for request and response/payload objects in the KOI-net API."""
+
 from pydantic import BaseModel
 from rid_lib import RID, RIDType
 from rid_lib.ext import Bundle, Manifest

koi_net/protocol/consts.py

@@ -1,3 +1,5 @@
+"""API paths for KOI-net protocol."""
+
 BROADCAST_EVENTS_PATH = "/events/broadcast"
 POLL_EVENTS_PATH      = "/events/poll"
 FETCH_RIDS_PATH       = "/rids/fetch"

koi_net-1.0.0b2.dist-info/METADATA

@@ -0,0 +1,209 @@
+Metadata-Version: 2.4
+Name: koi-net
+Version: 1.0.0b2
+Summary: Implementation of KOI-net protocol in Python
+Project-URL: Homepage, https://github.com/BlockScience/koi-net/
+Author-email: Luke Miller <luke@block.science>
+License: MIT License
+        
+        Copyright (c) 2025 BlockScience
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: networkx>=3.4.2
+Requires-Dist: pydantic>=2.10.6
+Requires-Dist: rid-lib>=3.2.1
+Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: twine>=6.0; extra == 'dev'
+Provides-Extra: examples
+Requires-Dist: fastapi; extra == 'examples'
+Requires-Dist: rich; extra == 'examples'
+Requires-Dist: uvicorn; extra == 'examples'
+Description-Content-Type: text/markdown
+
+# KOI-net
+
+*This specification is the result of several iterations of KOI research, [read more here](https://github.com/BlockScience/koi).*
+
+# Protocol
+## Introduction
+
+*This project builds upon and uses the [RID protocol](https://github.com/BlockScience/rid-lib) to identify and coordinate around knowledge objects.*
+
+This protocol defines the standard communication patterns and coordination norms needed to establish and maintain Knowledge Organization Infrastructure (KOI) networks. KOI-nets are heterogenous compositions of KOI nodes, each of which is capable of autonomously inputting, processing, and outputting knowledge. The behavior of each node and configuration of each network can vary greatly, thus the protocol is designed to be a simple and flexible but interoperable foundation for future projects to build on. The protocol only governs communication between nodes, not how they operate internally. As a result we consider KOI-nets to be fractal-like, in that a network of nodes may act like a single node from an outside perspective.
+
+Generated OpenAPI documentation is provided in this repository, and can be [viewed interactively with Swagger](https://generator.swagger.io/?url=https://raw.githubusercontent.com/BlockScience/koi/refs/heads/main/koi_v3_node_api.yaml).
+
+## Communication Methods
+
+There are two classes of communication methods, event and state communication. 
+- Event communication is one way, a node send an event to another node. 
+- State communication is two way, a node asks another node for RIDs, manifests, or bundles and receives a response containing the requested resource (if available).
+
+There are also two types of nodes, full and partial nodes. 
+- Full nodes are web servers, implementing the endpoints defined in the KOi-net protocol. They are capable of receiving events via webhooks (another node calls their endpoint), and serving state queries. They can also call the endpoints of other full nodes to broadcast events or retrieve state. 
+- Partial nodes are web clients and don't implement any API endpoints. They are capable of receiving events via polling (asking another node for events). They can also call the endpoints of full nodes to broadcast events or retrieve state.
+
+There are five endpoints defined by the API spec. The first two are for event communication with full and partial nodes respectively. The remaining three are for state communication with full nodes. As a result, partial nodes are unable to directly transfer state and may only output events to other nodes.
+- Broadcast events - `/events/broadcast`
+- Poll events - `/events/poll`
+- Fetch bundles - `/bundles/fetch`
+- Fetch manifests - `/manifests/fetch`
+- Fetch RIDs - `/rids/fetch`
+
+All endpoints are called with via POST request with a JSON body, and will receive a response containing a JSON payload (with the exception of broadcast events, which won't return anything). The JSON schemas can be found in the attached OpenAPI specification or the Pydantic models in the "protocol" module.
+
+The request and payload JSON objects are composed of the fundamental "knowledge types" from the RID / KOI-net system: RIDs, manifests, bundles, and events. RIDs, manifests, and bundles are defined by the RID protocol and imported from rid-lib, which you can [read about here](https://github.com/BlockScience/rid-lib). Events are now part of the KOI-net protocol, and are defined as an RID and an event type with an optional manifest and contents. 
+
+```json
+{
+    "rid": "...",
+    "event_type": "NEW | UPDATE | FORGET",
+    "manifest": {
+        "rid": "...",
+        "timestamp": "...",
+        "sha256_hash": "...",
+    },
+    "contents": {}
+}
+```
+
+This means that events are essentially just an RID, manifest, or bundle with an event type attached. Event types can be one of `FORGET`, `UPDATE`, or `NEW` forming the "FUN" acronym. While these types roughly correspond to delete, update, and create from CRUD operations, but they are not commands, they are signals. A node emits an event to indicate that its internal state has changed:
+- `NEW` - indicates an previously unknown RID was cached
+- `UPDATE` - indicates a previously known RID was cached
+- `FORGET` - indicates a previously known RID was deleted
+
+Nodes may broadcast events to other nodes to indicate its internal state changed. Conversely, nodes may also listen to events from other nodes and decide to change their internal state, take some other action, or do nothing.
+
+
+# Implementation
+## Setup
+
+The bulk of the code in this repo is taken up by the Python reference implementation, which can be used in other projects to easily set up and configure your own KOI node.
+
+This package can be installed with pip:
+```
+pip install koi-net
+```
+
+## Node Interface
+All of the KOI-net functionality comes from the `NodeInterface` class which provides methods to interact with the protocol API, a local RID cache, a view of the network, and an internal processing pipeline. To create a new node, you will need to give it a name and a profile. The name will be used to generate its unique node RID, and the profile stores basic configuration data which will be shared with other nodes you communciate with (it will be stored in the bundle associated with your node's RID). 
+- Partial nodes only need to indicate their type, and optionally the RID types of events they provide.
+- Full nodes need to indicate their type, the base URL for their KOI-net API, and optionally the RID types of events and state they provide.
+
+```python
+from koi_net import NodeInterface
+from koi_net.protocol.node import NodeProfile, NodeProvides, NodeType
+
+# partial node configuration
+
+partial_node = NodeInterface(
+    name="mypartialnode",
+    profile=NodeProfile(
+        node_type=NodeType.PARTIAL,
+        provides=NodeProvides(
+            event=[]
+        )
+    )
+)
+
+# full node configuration
+
+full_node = NodeInterface(
+    name="myfullnode",
+    profile=NodeProfile(
+        base_url="http://127.0.0.1:8000",
+        node_type=NodeType.FULL,
+        provides=NodeProvides(
+            event=[],
+            state=[]
+        )
+    )
+)
+```
+
+The node class mostly acts as a container for other classes with more specialized behavior, with special functions that should be called to start up and shut down a node. We'll take a look at each of these components in turn, but here is the class stub:
+```python
+class NodeInterface:
+    cache: Cache
+    identity: NodeIdentity
+    network: NetworkInterface
+    processor: ProcessorInterface
+    first_contact: str
+
+    def __init__(
+        self, 
+        name: str,
+        profile: NodeProfile,
+        identity_file_path: str = "identity.json",
+        first_contact: str | None = None,
+        handlers: list[KnowledgeHandler] | None = None,
+        cache: Cache | None = None,
+        network: NetworkInterface | None = None,
+        processor: ProcessorInterface | None = None
+    ): ...
+
+    def initialize(self): ...
+    def finalize(self): ...
+```
+As you can see, only a name and profile are required. The other fields allow for additional customization if needed.
+
+## Node Identity
+The `NodeIdentity` class provides easy access to a node's own RID, profile, and bundle. It provides access to the following properties after initialization, accessed with `node.identity`.
+```
+class NodeIdentity:
+    rid: KoiNetNode # an RID type
+    profile: NodeProfile
+    bundle: Bundle
+```
+This it what is initialized from the required `name` and `profile` fields in the `NodeInterface` constructor.
+
+## Network Interface
+The `NetworkInterface` class provides access to high level network actions, and contains several other network related classes. It is accessed with `node.network`.
+```python
+class NetworkInterface:
+    graph: NetworkGraph
+    request_handler: RequestHandler
+    response_handler: ResponseHandler
+
+    def __init__(
+        self, 
+        file_path: str,
+        first_contact: str | None,
+        cache: Cache, 
+        identity: NodeIdentity
+    ): ...
+
+    def push_event_to(self, event: Event, node: KoiNetNode, flush=False): ...
+    
+    def flush_poll_queue(self, node: KoiNetNode) -> list[Event]: ...
+    def flush_webhook_queue(self, node: RID): ...
+    def flush_all_webhook_queues(self): ...
+
+    def get_state_providers(self, rid_type: RIDType): ...
+
+    def fetch_remote_bundle(self, rid: RID): ...
+
+    def fetch_remote_manifest(self, rid: RID): ...
+
+    def poll_neighbors(self) -> list[Event]: ...
+

koi_net-1.0.0b2.dist-info/RECORD

@@ -0,0 +1,24 @@
+koi_net/__init__.py,sha256=b0Ze0pZmJAuygpWUFHM6Kvqo3DkU_uzmkptv1EpAArw,31
+koi_net/core.py,sha256=ZsBoTay7Z1_7JKzKt-vB3x_zl9GEit-fFkCSifLPoOk,3582
+koi_net/identity.py,sha256=PBgmAx5f3zzQmHASB1TJW2g19n9TLfmSJMXg2eQFg0A,2386
+koi_net/network/__init__.py,sha256=r_RN-q_mDYC-2RAkN-lJoMUX76TXyfEUc_MVKW87z0g,39
+koi_net/network/graph.py,sha256=VSvjF2p_EsuJsNPFhK4MEjgXW9oZL0Vg9Tn6X4-c3WY,4710
+koi_net/network/interface.py,sha256=paBJjQFJC8UkUz-BWeRwvuWD2cv8WFt7PyhoV7VOhWI,10823
+koi_net/network/request_handler.py,sha256=FDS6b3MLjeM_w6josbDtHZZUg0D1kuGDwE1H0si8mus,3870
+koi_net/network/response_handler.py,sha256=mA3FtrN3aTZATcLaHQhJUWrJdIKNv6d24fhvOl-nDKY,1890
+koi_net/processor/__init__.py,sha256=x4fAY0hvQEDcpfdTB3POIzxBQjYAtn0qQazPo1Xm0m4,41
+koi_net/processor/default_handlers.py,sha256=kmXBKBCAs6ygsPWl9IFbjZe1KDC_6M0YgkRF7hfI7kU,7224
+koi_net/processor/handler.py,sha256=9ZHsoPyAwFhtJPmvrLY-7C9dQ7KhLwI3DS5_Ms4HWAY,1640
+koi_net/processor/interface.py,sha256=4oiSWz91WWQywbfRdT2H2-aVh_6BTYE2AHbZSovzd0U,11671
+koi_net/processor/knowledge_object.py,sha256=cGv33fwNZQMylkhlTaQTbk96FVIVbdOUaBsG06u0m4k,4187
+koi_net/protocol/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+koi_net/protocol/api_models.py,sha256=Vm2Szp2OZ3EsZrYRwkAiBi_smk4MF1dAt-xPGOLi6ME,919
+koi_net/protocol/consts.py,sha256=zeWJvRpqcERrqJq39heyNHb6f_9QrvoBZJHd70yE914,249
+koi_net/protocol/edge.py,sha256=G3D9Ie0vbTSMJdoTw9g_oBmFCqzJ1gO7U1PVrw7p3j8,447
+koi_net/protocol/event.py,sha256=dzJmcHbimo7p5NwH2drccF0vMcAj9oQRj3iZ9Bjf7kg,1275
+koi_net/protocol/helpers.py,sha256=9E9PaoIuSNrTBATGCLJ_kSBMZ2z-KIMnLJzGOTqQDC0,719
+koi_net/protocol/node.py,sha256=Ntrx01dbm39ViKGtr4gLmztcMwKpTIweS6rRL-zoU_Y,391
+koi_net-1.0.0b2.dist-info/METADATA,sha256=Gry4tH3P7Ed1_d0EiC3oBnLIwHa6BmQaM4gTw6V1cbc,10358
+koi_net-1.0.0b2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+koi_net-1.0.0b2.dist-info/licenses/LICENSE,sha256=XBcvl8yjCAezfuqN1jadQykrX7H2g4nr2WRDmHLW6ik,1090
+koi_net-1.0.0b2.dist-info/RECORD,,

koi_net-1.0.0b1.dist-info/METADATA

@@ -1,43 +0,0 @@
-Metadata-Version: 2.4
-Name: koi-net
-Version: 1.0.0b1
-Summary: Implementation of KOI-net protocol in Python
-Project-URL: Homepage, https://github.com/BlockScience/koi-net/
-Author-email: Luke Miller <luke@block.science>
-License: MIT License
-        
-        Copyright (c) 2025 BlockScience
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy
-        of this software and associated documentation files (the "Software"), to deal
-        in the Software without restriction, including without limitation the rights
-        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-        copies of the Software, and to permit persons to whom the Software is
-        furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all
-        copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-        SOFTWARE.
-License-File: LICENSE
-Requires-Python: >=3.10
-Requires-Dist: httpx>=0.28.1
-Requires-Dist: networkx>=3.4.2
-Requires-Dist: pydantic>=2.10.6
-Requires-Dist: rid-lib>=3.2.1
-Provides-Extra: dev
-Requires-Dist: build; extra == 'dev'
-Requires-Dist: twine>=6.0; extra == 'dev'
-Provides-Extra: examples
-Requires-Dist: fastapi; extra == 'examples'
-Requires-Dist: rich; extra == 'examples'
-Requires-Dist: uvicorn; extra == 'examples'
-Description-Content-Type: text/markdown
-
-# koi-net

koi_net-1.0.0b1.dist-info/RECORD

@@ -1,24 +0,0 @@
-koi_net/__init__.py,sha256=b0Ze0pZmJAuygpWUFHM6Kvqo3DkU_uzmkptv1EpAArw,31
-koi_net/core.py,sha256=71ntaewhGBZ2zFQ7BxoSu2hhzPHQRsYYcRZGgTcdLhU,3024
-koi_net/identity.py,sha256=0ISIzQHUbwtPhXl6ZzrphzvwhBmb3K9t5wHmupr2_eA,1854
-koi_net/network/__init__.py,sha256=r_RN-q_mDYC-2RAkN-lJoMUX76TXyfEUc_MVKW87z0g,39
-koi_net/network/graph.py,sha256=qSHlilXZe2ikbDkE1tDRhazQfE2S_bxkNcExYmbxXqs,4039
-koi_net/network/interface.py,sha256=8WADUb5qADgw65fyPbUagYwK0rjYRa1XId7zegGJSa4,9316
-koi_net/network/request_handler.py,sha256=y7E80jgGfrXYxrOrHMMMzQCKrm5yV_GDessRE0GwOq4,3273
-koi_net/network/response_handler.py,sha256=EWOJzSBdbMYfsUx51avhZXAJEU0QqeR6USfKafHK-f8,1810
-koi_net/processor/__init__.py,sha256=x4fAY0hvQEDcpfdTB3POIzxBQjYAtn0qQazPo1Xm0m4,41
-koi_net/processor/default_handlers.py,sha256=djgPWj_bcNsT8Wxp41JcCk-kjlDIHTK6r4mcjdQWasQ,5973
-koi_net/processor/handler.py,sha256=6WkBJavbAs61cUwShcMjMb6lcWXh1Xu4g1PlMa3lu_A,747
-koi_net/processor/interface.py,sha256=uEAktg8cfFYJGO9rbCSja0wsdcnbDhe44iBXKCxyT9k,8511
-koi_net/processor/knowledge_object.py,sha256=fr0TBqSbjd8tdgbAxG_B5onKHRez52-dtsrz71K8Vew,2998
-koi_net/protocol/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-koi_net/protocol/api_models.py,sha256=dUvNtOor8cI_xhhMvuQuIgcEY7JhVrBg-6KoC_Jih6I,833
-koi_net/protocol/consts.py,sha256=GmaODi2V-BGZ5rMyFgRsrPz4iDfJZwkNpul2aSTQY5Q,208
-koi_net/protocol/edge.py,sha256=G3D9Ie0vbTSMJdoTw9g_oBmFCqzJ1gO7U1PVrw7p3j8,447
-koi_net/protocol/event.py,sha256=dzJmcHbimo7p5NwH2drccF0vMcAj9oQRj3iZ9Bjf7kg,1275
-koi_net/protocol/helpers.py,sha256=9E9PaoIuSNrTBATGCLJ_kSBMZ2z-KIMnLJzGOTqQDC0,719
-koi_net/protocol/node.py,sha256=Ntrx01dbm39ViKGtr4gLmztcMwKpTIweS6rRL-zoU_Y,391
-koi_net-1.0.0b1.dist-info/METADATA,sha256=7nR4xHkuAJI3yctTJEaEZGtLjkYXOlpN43DEZsukehc,1927
-koi_net-1.0.0b1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-koi_net-1.0.0b1.dist-info/licenses/LICENSE,sha256=XBcvl8yjCAezfuqN1jadQykrX7H2g4nr2WRDmHLW6ik,1090
-koi_net-1.0.0b1.dist-info/RECORD,,