koi-net 1.0.0__py3-none-any.whl

koi_net/processor/interface.py

@@ -0,0 +1,300 @@
+import logging
+import queue
+import threading
+from typing import Callable, Generic
+from rid_lib.core import RID, RIDType
+from rid_lib.ext import Bundle, Cache, Manifest
+from rid_lib.types.koi_net_edge import KoiNetEdge
+from rid_lib.types.koi_net_node import KoiNetNode
+from ..identity import NodeIdentity
+from ..network import NetworkInterface
+from ..protocol.event import Event, EventType
+from ..config import NodeConfig
+from .handler import (
+    KnowledgeHandler, 
+    HandlerType, 
+    STOP_CHAIN,
+    StopChain
+)
+from .knowledge_object import (
+    KnowledgeObject,
+    KnowledgeSource, 
+    KnowledgeEventType
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ProcessorInterface():
+    """Provides access to this node's knowledge processing pipeline."""
+    
+    config: NodeConfig
+    cache: Cache
+    network: NetworkInterface
+    identity: NodeIdentity
+    handlers: list[KnowledgeHandler]
+    kobj_queue: queue.Queue[KnowledgeObject]
+    use_kobj_processor_thread: bool
+    worker_thread: threading.Thread | None = None
+    
+    def __init__(
+        self,
+        config: NodeConfig,
+        cache: Cache, 
+        network: NetworkInterface,
+        identity: NodeIdentity,
+        use_kobj_processor_thread: bool,
+        default_handlers: list[KnowledgeHandler] = []
+    ):
+        self.config = config
+        self.cache = cache
+        self.network = network
+        self.identity = identity
+        self.use_kobj_processor_thread = use_kobj_processor_thread
+        self.handlers: list[KnowledgeHandler] = default_handlers
+        self.kobj_queue = queue.Queue()
+        
+        if self.use_kobj_processor_thread:
+            self.worker_thread = threading.Thread(
+                target=self.kobj_processor_worker,
+                daemon=True
+            )
+        
+    def add_handler(self, handler: KnowledgeHandler):
+        self.handlers.append(handler)
+            
+    def register_handler(
+        self,
+        handler_type: HandlerType,
+        rid_types: list[RIDType] | None = None,
+        source: KnowledgeSource | None = None,
+        event_types: list[KnowledgeEventType] | None = None
+    ):
+        """Assigns decorated function as handler for this processor."""
+        def decorator(func: Callable) -> Callable:
+            handler = KnowledgeHandler(func, handler_type, rid_types, source, event_types)
+            self.add_handler(handler)
+            return func
+        return decorator
+            
+    def call_handler_chain(
+        self, 
+        handler_type: HandlerType,
+        kobj: KnowledgeObject
+    ) -> KnowledgeObject | StopChain:
+        """Calls handlers of provided type, chaining their inputs and outputs together.
+        
+        The knowledge object provided when this function is called will be passed to the first handler. A handler may return one of three types: 
+        - `KnowledgeObject` - to modify the knowledge object for the next handler in the chain
+        - `None` - to keep the same knowledge object for the next handler in the chain
+        - `STOP_CHAIN` - to stop the handler chain and immediately exit the processing pipeline
+        
+        Handlers will only be called in the chain if their handler and RID type match that of the inputted knowledge object. 
+        """
+        
+        for handler in self.handlers:
+            if handler_type != handler.handler_type: 
+                continue
+            
+            if handler.rid_types and type(kobj.rid) not in handler.rid_types:
+                continue
+            
+            if handler.source and handler.source != kobj.source:
+                continue
+            
+            if handler.event_types and kobj.event_type not in handler.event_types:
+                continue
+            
+            logger.debug(f"Calling {handler_type} handler '{handler.func.__name__}'")
+            resp = handler.func(self, kobj.model_copy())
+            
+            # stops handler chain execution
+            if resp is STOP_CHAIN:
+                logger.debug(f"Handler chain stopped by {handler.func.__name__}")
+                return STOP_CHAIN
+            # kobj unmodified
+            elif resp is None:
+                continue
+            # kobj modified by handler
+            elif isinstance(resp, KnowledgeObject):
+                kobj = resp
+                logger.debug(f"Knowledge object modified by {handler.func.__name__}")
+            else:
+                raise ValueError(f"Handler {handler.func.__name__} returned invalid response '{resp}'")
+                    
+        return kobj
+
+        
+    def process_kobj(self, kobj: KnowledgeObject) -> None:
+        """Sends provided knowledge obejct through knowledge processing pipeline.
+        
+        Handler chains are called in between major events in the pipeline, indicated by their handler type. Each handler type is guaranteed to have access to certain knowledge, and may affect a subsequent action in the pipeline. The five handler types are as follows:
+        - RID - provided RID; if event type is `FORGET`, this handler decides whether to delete the knowledge from the cache by setting the normalized event type to `FORGET`, otherwise this handler decides whether to validate the manifest (and fetch it if not provided).
+        - Manifest - provided RID, manifest; decides whether to validate the bundle (and fetch it if not provided).
+        - Bundle - provided RID, manifest, contents (bundle); decides whether to write knowledge to the cache by setting the normalized event type to `NEW` or `UPDATE`.
+        - Network - provided RID, manifest, contents (bundle); decides which nodes (if any) to broadcast an event about this knowledge to. (Note, if event type is `FORGET`, the manifest and contents will be retrieved from the local cache, and indicate the last state of the knowledge before it was deleted.)
+        - Final - provided RID, manifests, contents (bundle); final action taken after network broadcast.
+        
+        The pipeline may be stopped by any point by a single handler returning the `STOP_CHAIN` sentinel. In that case, the process will exit immediately. Further handlers of that type and later handler chains will not be called.
+        """
+        
+        logger.debug(f"Handling {kobj!r}")
+        kobj = self.call_handler_chain(HandlerType.RID, kobj)
+        if kobj is STOP_CHAIN: return
+        
+        if kobj.event_type == EventType.FORGET:
+            bundle = self.cache.read(kobj.rid)
+            if not bundle: 
+                logger.debug("Local bundle not found")
+                return
+            
+            # the bundle (to be deleted) attached to kobj for downstream analysis
+            logger.debug("Adding local bundle (to be deleted) to knowledge object")
+            kobj.manifest = bundle.manifest
+            kobj.contents = bundle.contents
+            
+        else:
+            # attempt to retrieve manifest
+            if not kobj.manifest:
+                logger.debug("Manifest not found")
+                if kobj.source == KnowledgeSource.External:
+                    logger.debug("Attempting to fetch remote manifest")
+                    manifest = self.network.fetch_remote_manifest(kobj.rid)
+                    
+                elif kobj.source == KnowledgeSource.Internal:
+                    logger.debug("Attempting to read manifest from cache")
+                    bundle = self.cache.read(kobj.rid)
+                    if bundle: 
+                        manifest = bundle.manifest
+                    else:
+                        manifest = None
+                        return
+                    
+                if not manifest:
+                    logger.debug("Failed to find manifest")
+                    return
+                
+                kobj.manifest = manifest
+                
+            kobj = self.call_handler_chain(HandlerType.Manifest, kobj)
+            if kobj is STOP_CHAIN: return
+            
+            # attempt to retrieve bundle
+            if not kobj.bundle:
+                logger.debug("Bundle not found")
+                if kobj.source == KnowledgeSource.External:
+                    logger.debug("Attempting to fetch remote bundle")
+                    bundle = self.network.fetch_remote_bundle(kobj.rid)
+                    
+                elif kobj.source == KnowledgeSource.Internal:
+                    logger.debug("Attempting to read bundle from cache")
+                    bundle = self.cache.read(kobj.rid)
+                
+                if not bundle: 
+                    logger.debug("Failed to find bundle")
+                    return
+                
+                if kobj.manifest != bundle.manifest:
+                    logger.warning("Retrieved bundle contains a different manifest")
+                
+                kobj.manifest = bundle.manifest
+                kobj.contents = bundle.contents                
+                
+        kobj = self.call_handler_chain(HandlerType.Bundle, kobj)
+        if kobj is STOP_CHAIN: return
+            
+        if kobj.normalized_event_type in (EventType.UPDATE, EventType.NEW):
+            logger.info(f"Writing to cache: {kobj!r}")
+            self.cache.write(kobj.bundle)
+            
+        elif kobj.normalized_event_type == EventType.FORGET:
+            logger.info(f"Deleting from cache: {kobj!r}")
+            self.cache.delete(kobj.rid)
+            
+        else:
+            logger.debug("Normalized event type was never set, no cache or network operations will occur")
+            return
+        
+        if type(kobj.rid) in (KoiNetNode, KoiNetEdge):
+            logger.debug("Change to node or edge, regenerating network graph")
+            self.network.graph.generate()
+        
+        kobj = self.call_handler_chain(HandlerType.Network, kobj)
+        if kobj is STOP_CHAIN: return
+        
+        if kobj.network_targets:
+            logger.debug(f"Broadcasting event to {len(kobj.network_targets)} network target(s)")
+        else:
+            logger.debug("No network targets set")
+        
+        for node in kobj.network_targets:
+            self.network.push_event_to(kobj.normalized_event, node)
+            self.network.flush_webhook_queue(node)
+        
+        kobj = self.call_handler_chain(HandlerType.Final, kobj)
+
+    def flush_kobj_queue(self):
+        """Flushes all knowledge objects from queue and processes them.
+        
+        NOTE: ONLY CALL THIS METHOD IN SINGLE THREADED NODES, OTHERWISE THIS WILL CAUSE RACE CONDITIONS.
+        """
+        if self.use_kobj_processor_thread:
+            logger.warning("You are using a worker thread, calling this method can cause race conditions!")
+        
+        while not self.kobj_queue.empty():
+            kobj = self.kobj_queue.get()
+            logger.debug(f"Dequeued {kobj!r}")
+            
+            try:
+                self.process_kobj(kobj)
+            finally:
+                self.kobj_queue.task_done()
+            logger.debug("Done")
+    
+    def kobj_processor_worker(self, timeout=0.1):
+        while True:
+            try:
+                kobj = self.kobj_queue.get(timeout=timeout)
+                logger.debug(f"Dequeued {kobj!r}")
+                
+                try:
+                    self.process_kobj(kobj)
+                finally:
+                    self.kobj_queue.task_done()
+                logger.debug("Done")
+            
+            except queue.Empty:
+                pass
+            
+            except Exception as e:
+                logger.warning(f"Error processing kobj: {e}")
+        
+    def handle(
+        self,
+        rid: RID | None = None,
+        manifest: Manifest | None = None,
+        bundle: Bundle | None = None,
+        event: Event | None = None,
+        kobj: KnowledgeObject | None = None,
+        event_type: KnowledgeEventType = None,
+        source: KnowledgeSource = KnowledgeSource.Internal
+    ):
+        """Queues provided knowledge to be handled by processing pipeline.
+        
+        Knowledge may take the form of an RID, manifest, bundle, event, or knowledge object (with an optional event type for RID, manifest, or bundle objects). All objects will be normalized into knowledge objects and queued. If `flush` is `True`, the queue will be flushed immediately after adding the new knowledge.
+        """
+        if rid:
+            _kobj = KnowledgeObject.from_rid(rid, event_type, source)
+        elif manifest:
+            _kobj = KnowledgeObject.from_manifest(manifest, event_type, source)
+        elif bundle:
+            _kobj = KnowledgeObject.from_bundle(bundle, event_type, source)
+        elif event:
+            _kobj = KnowledgeObject.from_event(event, source)
+        elif kobj:
+            _kobj = kobj
+        else:
+            raise ValueError("One of 'rid', 'manifest', 'bundle', 'event', or 'kobj' must be provided")
+        
+        self.kobj_queue.put(_kobj)
+        logger.debug(f"Queued {_kobj!r}")

koi_net/processor/knowledge_object.py

@@ -0,0 +1,123 @@
+from enum import StrEnum
+from pydantic import BaseModel
+from rid_lib import RID
+from rid_lib.ext import Manifest
+from rid_lib.ext.bundle import Bundle
+from rid_lib.types.koi_net_node import KoiNetNode
+from ..protocol.event import Event, EventType
+
+
+type KnowledgeEventType = EventType | None
+
+class KnowledgeSource(StrEnum):
+    Internal = "INTERNAL"
+    External = "EXTERNAL"
+
+class KnowledgeObject(BaseModel):
+    """A normalized knowledge representation for internal processing.
+    
+    Capable of representing an RID, manifest, bundle, or event. Contains three additional fields use for decision making in the knowledge processing pipeline. 
+    
+    The source indicates whether this object was generated by this node, or sourced from another node in the network. 
+    
+    The normalized event type indicates how the knowledge object is viewed from the perspective of this node, and what cache actions will take place. `NEW`, `UPDATE` -> cache write, `FORGET` -> cache delete, `None` -> no cache action.
+    
+    The network targets indicate other nodes in the network this knowledge object will be sent to. The event sent to them will be constructed from this knowledge object's RID, manifest, contents, and normalized event type.
+    
+    Constructors are provided to create a knowledge object from an RID, manifest, bundle, or event.
+    """
+    rid: RID
+    manifest: Manifest | None = None
+    contents: dict | None = None
+    event_type: KnowledgeEventType = None
+    normalized_event_type: KnowledgeEventType = None
+    source: KnowledgeSource
+    network_targets: set[KoiNetNode] = set()
+    
+    def __repr__(self):
+        return f"<KObj '{self.rid}' event type: '{self.event_type}' -> '{self.normalized_event_type}', source: '{self.source}'>"
+    
+    @classmethod
+    def from_rid(
+        cls, 
+        rid: RID, 
+        event_type: KnowledgeEventType = None, 
+        source: KnowledgeSource = KnowledgeSource.Internal
+    ) -> "KnowledgeObject":
+        return cls(
+            rid=rid,
+            event_type=event_type,
+            source=source
+        )
+        
+    @classmethod
+    def from_manifest(
+        cls, 
+        manifest: Manifest, 
+        event_type: KnowledgeEventType = None, 
+        source: KnowledgeSource = KnowledgeSource.Internal
+    ) -> "KnowledgeObject":
+        return cls(
+            rid=manifest.rid,
+            manifest=manifest,
+            event_type=event_type,
+            source=source
+        )
+        
+    @classmethod
+    def from_bundle(
+        cls, 
+        bundle: Bundle, 
+        event_type: KnowledgeEventType = None, 
+        source: KnowledgeSource = KnowledgeSource.Internal
+    ) -> "KnowledgeObject":
+        return cls(
+            rid=bundle.rid,
+            manifest=bundle.manifest,
+            contents=bundle.contents,
+            event_type=event_type,
+            source=source
+        )
+        
+    @classmethod
+    def from_event(
+        cls,
+        event: Event,
+        source: KnowledgeSource = KnowledgeSource.Internal
+    ) -> "KnowledgeObject":
+        return cls(
+            rid=event.rid,
+            manifest=event.manifest,
+            contents=event.contents,
+            event_type=event.event_type,
+            source=source
+        )
+    
+    @property
+    def bundle(self):
+        if self.manifest is None or self.contents is None:
+            return
+        
+        return Bundle(
+            manifest=self.manifest,
+            contents=self.contents
+        )
+    
+    @property
+    def normalized_event(self):
+        if self.normalized_event_type is None:
+            raise ValueError("Internal event's normalized event type is None, cannot convert to Event")
+        
+        elif self.normalized_event_type == EventType.FORGET:
+            return Event(
+                rid=self.rid,
+                event_type=EventType.FORGET
+            )
+        
+        else:    
+            return Event(
+                rid=self.rid,
+                event_type=self.normalized_event_type,
+                manifest=self.manifest,
+                contents=self.contents
+            )

koi_net/protocol/api_models.py

@@ -0,0 +1,47 @@
+"""Pydantic models for request and response/payload objects in the KOI-net API."""
+
+from pydantic import BaseModel
+from rid_lib import RID, RIDType
+from rid_lib.ext import Bundle, Manifest
+from .event import Event
+
+
+# REQUEST MODELS
+
+class PollEvents(BaseModel):
+    rid: RID
+    limit: int = 0
+    
+class FetchRids(BaseModel):
+    rid_types: list[RIDType] = []
+    
+class FetchManifests(BaseModel):
+    rid_types: list[RIDType] = []
+    rids: list[RID] = []
+    
+class FetchBundles(BaseModel):
+    rids: list[RID]
+    
+
+# RESPONSE/PAYLOAD MODELS
+
+class RidsPayload(BaseModel):
+    rids: list[RID]
+
+class ManifestsPayload(BaseModel):
+    manifests: list[Manifest]
+    not_found: list[RID] = []
+    
+class BundlesPayload(BaseModel):
+    bundles: list[Bundle]
+    not_found: list[RID] = []
+    deferred: list[RID] = []
+    
+class EventsPayload(BaseModel):
+    events: list[Event]
+    
+
+# TYPES
+
+type RequestModels = EventsPayload | PollEvents | FetchRids | FetchManifests | FetchBundles
+type ResponseModels = RidsPayload | ManifestsPayload | BundlesPayload | EventsPayload

koi_net/protocol/consts.py

@@ -0,0 +1,7 @@
+"""API paths for KOI-net protocol."""
+
+BROADCAST_EVENTS_PATH = "/events/broadcast"
+POLL_EVENTS_PATH      = "/events/poll"
+FETCH_RIDS_PATH       = "/rids/fetch"
+FETCH_MANIFESTS_PATH  = "/manifests/fetch"
+FETCH_BUNDLES_PATH    = "/bundles/fetch"

koi_net/protocol/edge.py

@@ -0,0 +1,20 @@
+from enum import StrEnum
+from pydantic import BaseModel
+from rid_lib import RIDType
+from rid_lib.types import KoiNetNode
+
+
+class EdgeStatus(StrEnum):
+    PROPOSED = "PROPOSED"
+    APPROVED = "APPROVED"
+    
+class EdgeType(StrEnum):
+    WEBHOOK = "WEBHOOK"
+    POLL = "POLL"
+
+class EdgeProfile(BaseModel):
+    source: KoiNetNode
+    target: KoiNetNode
+    edge_type: EdgeType
+    status: EdgeStatus
+    rid_types: list[RIDType]

koi_net/protocol/event.py

@@ -0,0 +1,54 @@
+from enum import StrEnum
+from pydantic import BaseModel
+from rid_lib import RID
+from rid_lib.ext import Manifest, Bundle
+
+
+class EventType(StrEnum):
+    NEW = "NEW"
+    UPDATE = "UPDATE"
+    FORGET = "FORGET"
+
+class Event(BaseModel):
+    rid: RID
+    event_type: EventType
+    manifest: Manifest | None = None
+    contents: dict | None = None
+    
+    class Config:
+        exclude_none = True
+    
+    def __repr__(self):
+        return f"<Event '{self.rid}' event type: '{self.event_type}'>"
+    
+    @classmethod
+    def from_bundle(cls, event_type: EventType, bundle: Bundle):
+        return cls(
+            rid=bundle.manifest.rid,
+            event_type=event_type,
+            manifest=bundle.manifest,
+            contents=bundle.contents
+        )
+        
+    @classmethod
+    def from_manifest(cls, event_type: EventType, manifest: Manifest):
+        return cls(
+            rid=manifest.rid,
+            event_type=event_type,
+            manifest=manifest
+        )
+        
+    @classmethod
+    def from_rid(cls, event_type: EventType, rid: RID):
+        return cls(
+            rid=rid,
+            event_type=event_type
+        )
+        
+    @property
+    def bundle(self):
+        if self.manifest is not None and self.contents is not None:
+            return Bundle(
+                manifest=self.manifest,
+                contents=self.contents
+            )

koi_net/protocol/helpers.py

@@ -0,0 +1,25 @@
+from rid_lib.core import RIDType
+from rid_lib.ext.bundle import Bundle
+from rid_lib.types import KoiNetEdge
+from rid_lib.types.koi_net_node import KoiNetNode
+from .edge import EdgeProfile, EdgeStatus, EdgeType
+
+def generate_edge_bundle(
+    source: KoiNetNode,
+    target: KoiNetNode,
+    rid_types: list[RIDType],
+    edge_type: EdgeType
+) -> Bundle:
+    edge_rid = KoiNetEdge.generate(source, target)
+    edge_profile = EdgeProfile(
+        source=source,
+        target=target,
+        rid_types=rid_types,
+        edge_type=edge_type,
+        status=EdgeStatus.PROPOSED
+    )
+    edge_bundle = Bundle.generate(
+        edge_rid,
+        edge_profile.model_dump()
+    )
+    return edge_bundle

koi_net/protocol/node.py

@@ -0,0 +1,17 @@
+from enum import StrEnum
+from pydantic import BaseModel
+from rid_lib import RIDType
+
+
+class NodeType(StrEnum):
+    FULL = "FULL"
+    PARTIAL = "PARTIAL"
+
+class NodeProvides(BaseModel):
+    event: list[RIDType] = []
+    state: list[RIDType] = []
+
+class NodeProfile(BaseModel):
+    base_url: str | None = None
+    node_type: NodeType
+    provides: NodeProvides = NodeProvides()