koi-net 1.0.0b1__py3-none-any.whl

koi_net/processor/default_handlers.py

@@ -0,0 +1,151 @@
+import logging
+from rid_lib.ext.bundle import Bundle
+from rid_lib.types import KoiNetNode, KoiNetEdge
+from koi_net.protocol.node import NodeType
+from .interface import ProcessorInterface
+from .handler import HandlerType, STOP_CHAIN
+from .knowledge_object import KnowledgeObject, KnowledgeSource
+from ..protocol.event import Event, EventType
+from ..protocol.edge import EdgeProfile, EdgeStatus, EdgeType
+
+logger = logging.getLogger(__name__)
+
+# RID handlers
+
+@ProcessorInterface.as_handler(handler_type=HandlerType.RID)
+def basic_rid_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
+    if (kobj.rid == processor.identity.rid and 
+        kobj.source == KnowledgeSource.External):
+        logger.info("Don't let anyone else tell me who I am!")
+        return STOP_CHAIN
+    
+    if kobj.event_type == EventType.FORGET:        
+        if processor.cache.exists(kobj.rid):
+            logger.info("Allowing cache forget")
+            kobj.normalized_event_type = EventType.FORGET
+            return kobj
+        
+        else:
+            # can't forget something I don't know about
+            return STOP_CHAIN
+
+
+# Manifest handlers
+
+@ProcessorInterface.as_handler(handler_type=HandlerType.Manifest)
+def basic_state_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
+    prev_bundle = processor.cache.read(kobj.rid)
+
+    if prev_bundle:
+        if kobj.manifest.sha256_hash == prev_bundle.manifest.sha256_hash:
+            logger.info("Hash of incoming manifest is same as existing knowledge, ignoring")
+            return STOP_CHAIN
+        if kobj.manifest.timestamp <= prev_bundle.manifest.timestamp:
+            logger.info("Timestamp of incoming manifest is the same or older than existing knowledge, ignoring")
+            return STOP_CHAIN
+        
+        logger.info("RID previously known to me, labeling as 'UPDATE'")
+        kobj.normalized_event_type = EventType.UPDATE
+
+    else:
+        logger.info("RID previously unknown to me, labeling as 'NEW'")
+        kobj.normalized_event_type = EventType.NEW
+        
+    return kobj
+
+
+# Bundle handlers
+
+@ProcessorInterface.as_handler(HandlerType.Bundle, rid_types=[KoiNetEdge])
+def edge_negotiation_handler(processor: ProcessorInterface, kobj: KnowledgeObject):
+    edge_profile = EdgeProfile.model_validate(kobj.contents)
+    
+    # only want to handle external knowledge events (not edges this node created)
+    if kobj.source != KnowledgeSource.External:
+        return
+
+    # indicates peer subscribing to me
+    if edge_profile.source == processor.identity.rid:     
+        if edge_profile.status != EdgeStatus.PROPOSED:
+            return
+        
+        logger.info("Handling edge negotiation")
+        
+        peer_rid = edge_profile.target
+        peer_profile = processor.network.graph.get_node_profile(peer_rid)
+        
+        if not peer_profile:
+            logger.warning(f"Peer {peer_rid} unknown to me")
+            return STOP_CHAIN
+        
+        # explicitly provided event RID types and (self) node + edge objects
+        provided_events = (
+            *processor.identity.profile.provides.event,
+            KoiNetNode, KoiNetEdge
+        )
+        
+        
+        abort = False
+        if (edge_profile.edge_type == EdgeType.WEBHOOK and 
+            peer_profile.node_type == NodeType.PARTIAL):
+            logger.info("Partial nodes cannot use webhooks")
+            abort = True
+        
+        if not set(edge_profile.rid_types).issubset(provided_events):
+            logger.info("Requested RID types not provided by this node")
+            abort = True
+        
+        if abort:
+            event = Event.from_rid(EventType.FORGET, kobj.rid)
+            processor.network.push_event_to(event, peer_rid, flush=True)
+            return STOP_CHAIN
+
+        else:
+            # approve edge profile
+            logger.info("Approving proposed edge")
+            edge_profile.status = EdgeStatus.APPROVED
+            updated_bundle = Bundle.generate(kobj.rid, edge_profile.model_dump())
+      
+            processor.handle(bundle=updated_bundle)
+            return
+              
+    elif edge_profile.target == processor.identity.rid:
+        if edge_profile.status == EdgeStatus.APPROVED:
+            logger.info("Edge approved by other node!")
+
+
+# Network handlers
+
+@ProcessorInterface.as_handler(HandlerType.Network)
+def basic_network_output_filter(processor: ProcessorInterface, kobj: KnowledgeObject):
+    involves_me = False
+    if kobj.source == KnowledgeSource.Internal:
+        if (type(kobj.rid) == KoiNetNode):
+            if (kobj.rid == processor.identity.rid):
+                involves_me = True
+        
+        elif type(kobj.rid) == KoiNetEdge:
+            edge_profile = kobj.bundle.validate_contents(EdgeProfile)
+            
+            if edge_profile.source == processor.identity.rid:
+                logger.info(f"Adding edge target '{edge_profile.target!r}' to network targets")
+                kobj.network_targets.update([edge_profile.target])
+                involves_me = True
+                
+            elif edge_profile.target == processor.identity.rid:
+                logger.info(f"Adding edge source '{edge_profile.source!r}' to network targets")
+                kobj.network_targets.update([edge_profile.source])
+                involves_me = True
+    
+    if (type(kobj.rid) in processor.identity.profile.provides.event or involves_me):
+        # broadcasts to subscribers if I'm an event provider of this RID type OR it involves me
+        subscribers = processor.network.graph.get_neighbors(
+            direction="out",
+            allowed_type=type(kobj.rid)
+        )
+        
+        logger.info(f"Updating network targets with '{type(kobj.rid)}' subscribers: {subscribers}")
+        kobj.network_targets.update(subscribers)
+        
+    return kobj
+            

koi_net/processor/handler.py

@@ -0,0 +1,22 @@
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Callable
+from rid_lib import RIDType
+
+
+# sentinel
+STOP_CHAIN = object()
+
+class HandlerType(StrEnum):
+    RID = "rid", # guaranteed RID - decides whether validate manifest OR cache delete
+    Manifest = "manifest", # guaranteed Manifest - decides whether to validate bundle
+    Bundle = "bundle", # guaranteed Bundle - decides whether to write to cache
+    Network = "network", # guaranteed Bundle, after cache write/delete  - decides network targets
+    Final = "final" # guaranteed Bundle, after network push - final action
+
+@dataclass
+class KnowledgeHandler:
+    func: Callable
+    handler_type: HandlerType
+    rid_types: list[RIDType] | None
+

koi_net/processor/interface.py

@@ -0,0 +1,222 @@
+import logging
+from queue import Queue
+from typing import Callable
+from rid_lib.core import RID, RIDType
+from rid_lib.ext import Bundle, Cache, Manifest
+from rid_lib.types.koi_net_edge import KoiNetEdge
+from rid_lib.types.koi_net_node import KoiNetNode
+from ..identity import NodeIdentity
+from ..network import NetworkInterface
+from ..protocol.event import Event, EventType
+from .handler import (
+    KnowledgeHandler, 
+    HandlerType, 
+    STOP_CHAIN
+)
+from .knowledge_object import (
+    KnowledgeObject,
+    KnowledgeSource, 
+    KnowledgeEventType
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ProcessorInterface:
+    cache: Cache
+    network: NetworkInterface
+    identity: NodeIdentity
+    handlers: list[KnowledgeHandler]
+    kobj_queue: Queue[KnowledgeObject]
+    
+    def __init__(
+        self, 
+        cache: Cache, 
+        network: NetworkInterface,
+        identity: NodeIdentity,
+        default_handlers: list[KnowledgeHandler] = []
+    ):
+        self.cache = cache
+        self.network = network
+        self.identity = identity
+        self.handlers: list[KnowledgeHandler] = default_handlers
+        self.kobj_queue = Queue()
+    
+    @classmethod
+    def as_handler(
+        cls,
+        handler_type: HandlerType,
+        rid_types: list[RIDType] | None = None
+    ):
+        """Special decorator that returns a Handler instead of a function."""
+        def decorator(func: Callable) -> KnowledgeHandler:
+            handler = KnowledgeHandler(func, handler_type, rid_types, )
+            return handler
+        return decorator
+            
+    def register_handler(
+        self,
+        handler_type: HandlerType,
+        rid_types: list[RIDType] | None = None
+    ):
+        """Assigns decorated function as handler for this Processor."""
+        def decorator(func: Callable) -> Callable:
+            handler = KnowledgeHandler(func, handler_type, rid_types)
+            self.handlers.append(handler)
+            return func
+        return decorator
+            
+    def call_handler_chain(
+        self, 
+        handler_type: HandlerType,
+        kobj: KnowledgeObject
+    ):
+        for handler in self.handlers:
+            if handler_type != handler.handler_type: 
+                continue
+            
+            if handler.rid_types and type(kobj.rid) not in handler.rid_types:
+                continue
+            
+            logger.info(f"Calling {handler_type} handler '{handler.func.__name__}'")
+            resp = handler.func(self, kobj.model_copy())
+            
+            # stops handler chain execution
+            if resp is STOP_CHAIN:
+                logger.info(f"Handler chain stopped by {handler.func.__name__}")
+                return STOP_CHAIN
+            # kobj unmodified
+            elif resp is None:
+                continue
+            # kobj modified by handler
+            elif isinstance(resp, KnowledgeObject):
+                kobj = resp
+                logger.info(f"Knowledge object modified by {handler.func.__name__}")
+            else:
+                raise ValueError(f"Handler {handler.func.__name__} returned invalid response '{resp}'")
+                    
+        return kobj
+
+        
+    def handle_kobj(self, kobj: KnowledgeObject):
+        logger.info(f"Handling {kobj!r}")
+        kobj = self.call_handler_chain(HandlerType.RID, kobj)
+        if kobj is STOP_CHAIN: return
+        
+        if kobj.event_type == EventType.FORGET:
+            bundle = self.cache.read(kobj.rid)
+            if not bundle: 
+                logger.info("Local bundle not found")
+                return
+            
+            # the bundle (to be deleted) attached to kobj for downstream analysis
+            logger.info("Adding local bundle (to be deleted) to knowledge object")
+            kobj.manifest = bundle.manifest
+            kobj.contents = bundle.contents
+            
+        else:
+            # attempt to retrieve manifest
+            if not kobj.manifest:
+                if kobj.source == KnowledgeSource.External:
+                    logger.info("Manifest not found, attempting to fetch remotely")
+                    manifest = self.network.fetch_remote_manifest(kobj.rid)
+                    if not manifest: return
+                    
+                elif kobj.source == KnowledgeSource.Internal:
+                    logger.info("Manifest not found, attempting to read cache")
+                    bundle = self.cache.read(kobj.rid)
+                    if not bundle: return
+                    manifest = bundle.manifest
+                
+                kobj.manifest = manifest
+                
+            kobj = self.call_handler_chain(HandlerType.Manifest, kobj)
+            if kobj is STOP_CHAIN: return
+            
+            # attempt to retrieve bundle
+            if not kobj.bundle:
+                if kobj.source == KnowledgeSource.External:
+                    logger.info("Bundle not found, attempting to fetch")
+                    bundle = self.network.fetch_remote_bundle(kobj.rid)
+                    # TODO: WARNING MANIFEST MAY BE DIFFERENT
+                    
+                elif kobj.source == KnowledgeSource.Internal:
+                    logger.info("Bundle not found, attempting to read cache")
+                    bundle = self.cache.read(kobj.rid)
+                
+                if kobj.manifest != bundle.manifest:
+                    logger.warning("Retrieved bundle contains a different manifest")
+                
+                if not bundle: return
+                kobj.manifest = bundle.manifest
+                kobj.contents = bundle.contents                
+                
+            kobj = self.call_handler_chain(HandlerType.Bundle, kobj)
+            if kobj is STOP_CHAIN: return
+            
+        if kobj.normalized_event_type in (EventType.UPDATE, EventType.NEW):
+            logger.info(f"Writing {kobj!r} to cache")
+            self.cache.write(kobj.bundle)
+            
+        elif kobj.normalized_event_type == EventType.FORGET:
+            logger.info(f"Deleting {kobj!r} from cache")
+            self.cache.delete(kobj.rid)
+            
+        else:
+            logger.info("Normalized event type was never set, no cache or network operations will occur")
+            return
+        
+        if type(kobj.rid) in (KoiNetNode, KoiNetEdge):
+            logger.info("Change to node or edge, regenerating network graph")
+            self.network.graph.generate()
+        
+        kobj = self.call_handler_chain(HandlerType.Network, kobj)
+        if kobj is STOP_CHAIN: return
+        
+        if kobj.network_targets:
+            logger.info(f"Broadcasting event to {len(kobj.network_targets)} network target(s)")
+        else:
+            logger.info("No network targets set")
+        
+        for node in kobj.network_targets:
+            self.network.push_event_to(kobj.normalized_event, node)
+        self.network.flush_all_webhook_queues()
+        
+        kobj = self.call_handler_chain(HandlerType.Final, kobj)
+            
+    def queue_kobj(self, kobj: KnowledgeObject, flush: bool = False):
+        self.kobj_queue.put(kobj)
+        logger.info(f"Queued {kobj!r}")
+        
+        if flush:
+            self.flush_kobj_queue()
+                
+    def flush_kobj_queue(self):
+        while not self.kobj_queue.empty():
+            kobj = self.kobj_queue.get()
+            logger.info(f"Dequeued {kobj!r}")
+            self.handle_kobj(kobj)
+            logger.info("Done handling")
+        
+    def handle(
+        self,
+        rid: RID | None = None,
+        manifest: Manifest | None = None,
+        bundle: Bundle | None = None,
+        event: Event | None = None,
+        event_type: KnowledgeEventType = None,
+        source: KnowledgeSource = KnowledgeSource.Internal,
+        flush: bool = False
+    ):
+        if rid:
+            kobj = KnowledgeObject.from_rid(rid, event_type, source)
+        elif manifest:
+            kobj = KnowledgeObject.from_manifest(manifest, event_type, source)
+        elif bundle:
+            kobj = KnowledgeObject.from_bundle(bundle, event_type, source)
+        elif event:
+            kobj = KnowledgeObject.from_event(event, source)
+        else:
+            raise ValueError("One of 'rid', 'manifest', 'bundle', or 'event' must be provided")
+          
+        self.queue_kobj(kobj, flush)

koi_net/processor/knowledge_object.py

@@ -0,0 +1,104 @@
+from enum import StrEnum
+from pydantic import BaseModel
+from rid_lib import RID
+from rid_lib.ext import Manifest
+from rid_lib.ext.bundle import Bundle
+from rid_lib.types.koi_net_node import KoiNetNode
+from ..protocol.event import Event, EventType
+
+
+type KnowledgeEventType = EventType | None
+
+class KnowledgeSource(StrEnum):
+    Internal = "INTERNAL"
+    External = "EXTERNAL"
+
+class KnowledgeObject(BaseModel):
+    rid: RID
+    manifest: Manifest | None = None
+    contents: dict | None = None
+    event_type: KnowledgeEventType = None
+    normalized_event_type: KnowledgeEventType = None
+    source: KnowledgeSource
+    network_targets: set[KoiNetNode] = set()
+    
+    def __repr__(self):
+        return f"<Knowledge Object '{self.rid}' ({self.event_type}) -> ({self.normalized_event_type})>"
+    
+    @classmethod
+    def from_rid(
+        cls, 
+        rid: RID, 
+        event_type: KnowledgeEventType = None, 
+        source: KnowledgeSource = KnowledgeSource.Internal
+    ) -> "KnowledgeObject":
+        return cls(
+            rid=rid,
+            event_type=event_type,
+            source=source
+        )
+        
+    @classmethod
+    def from_manifest(
+        cls, 
+        manifest: Manifest, 
+        event_type: KnowledgeEventType = None, 
+        source: KnowledgeSource = KnowledgeSource.Internal
+    ) -> "KnowledgeObject":
+        return cls(
+            rid=manifest.rid,
+            manifest=manifest,
+            event_type=event_type,
+            source=source
+        )
+        
+    @classmethod
+    def from_bundle(
+        cls, 
+        bundle: Bundle, 
+        event_type: KnowledgeEventType = None, 
+        source: KnowledgeSource = KnowledgeSource.Internal
+    ) -> "KnowledgeObject":
+        return cls(
+            rid=bundle.rid,
+            manifest=bundle.manifest,
+            contents=bundle.contents,
+            event_type=event_type,
+            source=source
+        )
+        
+    @classmethod
+    def from_event(
+        cls,
+        event: Event,
+        source: KnowledgeSource = KnowledgeSource.Internal
+    ) -> "KnowledgeObject":
+        return cls(
+            rid=event.rid,
+            manifest=event.manifest,
+            contents=event.contents,
+            event_type=event.event_type,
+            source=source
+        )
+    
+    @property
+    def bundle(self):
+        if self.manifest is None or self.contents is None:
+            return
+        
+        return Bundle(
+            manifest=self.manifest,
+            contents=self.contents
+        )
+    
+    @property
+    def normalized_event(self):
+        if not self.normalized_event_type:
+            raise ValueError("Internal event's normalized event type is None, cannot convert to Event")
+        
+        return Event(
+            rid=self.rid,
+            event_type=self.normalized_event_type,
+            manifest=self.manifest,
+            contents=self.contents
+        )

koi_net/protocol/api_models.py

@@ -0,0 +1,39 @@
+from pydantic import BaseModel
+from rid_lib import RID, RIDType
+from rid_lib.ext import Bundle, Manifest
+from .event import Event
+
+
+# REQUEST MODELS
+
+class PollEvents(BaseModel):
+    rid: RID
+    limit: int = 0
+    
+class FetchRids(BaseModel):
+    rid_types: list[RIDType] = []
+    
+class FetchManifests(BaseModel):
+    rid_types: list[RIDType] = []
+    rids: list[RID] = []
+    
+class FetchBundles(BaseModel):
+    rids: list[RID]
+    
+
+# RESPONSE/PAYLOAD MODELS
+
+class RidsPayload(BaseModel):
+    rids: list[RID]
+
+class ManifestsPayload(BaseModel):
+    manifests: list[Manifest]
+    not_found: list[RID] = []
+    
+class BundlesPayload(BaseModel):
+    manifests: list[Bundle]
+    not_found: list[RID] = []
+    deferred: list[RID] = []
+    
+class EventsPayload(BaseModel):
+    events: list[Event]

koi_net/protocol/consts.py

@@ -0,0 +1,5 @@
+BROADCAST_EVENTS_PATH = "/events/broadcast"
+POLL_EVENTS_PATH      = "/events/poll"
+FETCH_RIDS_PATH       = "/rids/fetch"
+FETCH_MANIFESTS_PATH  = "/manifests/fetch"
+FETCH_BUNDLES_PATH    = "/bundles/fetch"

koi_net/protocol/edge.py

@@ -0,0 +1,20 @@
+from enum import StrEnum
+from pydantic import BaseModel
+from rid_lib import RIDType
+from rid_lib.types import KoiNetNode
+
+
+class EdgeStatus(StrEnum):
+    PROPOSED = "PROPOSED"
+    APPROVED = "APPROVED"
+    
+class EdgeType(StrEnum):
+    WEBHOOK = "WEBHOOK"
+    POLL = "POLL"
+
+class EdgeProfile(BaseModel):
+    source: KoiNetNode
+    target: KoiNetNode
+    edge_type: EdgeType
+    status: EdgeStatus
+    rid_types: list[RIDType]

koi_net/protocol/event.py

@@ -0,0 +1,48 @@
+from enum import StrEnum
+from pydantic import BaseModel
+from rid_lib import RID
+from rid_lib.ext import Manifest, Bundle
+
+
+class EventType(StrEnum):
+    NEW = "NEW"
+    UPDATE = "UPDATE"
+    FORGET = "FORGET"
+
+class Event(BaseModel):
+    rid: RID
+    event_type: EventType
+    manifest: Manifest | None = None
+    contents: dict | None = None
+    
+    @classmethod
+    def from_bundle(cls, event_type: EventType, bundle: Bundle):
+        return cls(
+            rid=bundle.manifest.rid,
+            event_type=event_type,
+            manifest=bundle.manifest,
+            contents=bundle.contents
+        )
+        
+    @classmethod
+    def from_manifest(cls, event_type: EventType, manifest: Manifest):
+        return cls(
+            rid=manifest.rid,
+            event_type=event_type,
+            manifest=manifest
+        )
+        
+    @classmethod
+    def from_rid(cls, event_type: EventType, rid: RID):
+        return cls(
+            rid=rid,
+            event_type=event_type
+        )
+        
+    @property
+    def bundle(self):
+        if self.manifest is not None and self.contents is not None:
+            return Bundle(
+                manifest=self.manifest,
+                contents=self.contents
+            )

koi_net/protocol/helpers.py

@@ -0,0 +1,25 @@
+from rid_lib.core import RIDType
+from rid_lib.ext.bundle import Bundle
+from rid_lib.types import KoiNetEdge
+from rid_lib.types.koi_net_node import KoiNetNode
+from .edge import EdgeProfile, EdgeStatus, EdgeType
+
+def generate_edge_bundle(
+    source: KoiNetNode,
+    target: KoiNetNode,
+    rid_types: list[RIDType],
+    edge_type: EdgeType
+) -> Bundle:
+    edge_rid = KoiNetEdge.generate(source, target)
+    edge_profile = EdgeProfile(
+        source=source,
+        target=target,
+        rid_types=rid_types,
+        edge_type=edge_type,
+        status=EdgeStatus.PROPOSED
+    )
+    edge_bundle = Bundle.generate(
+        edge_rid,
+        edge_profile.model_dump()
+    )
+    return edge_bundle

koi_net/protocol/node.py

@@ -0,0 +1,17 @@
+from enum import StrEnum
+from pydantic import BaseModel
+from rid_lib import RIDType
+
+
+class NodeType(StrEnum):
+    FULL = "FULL"
+    PARTIAL = "PARTIAL"
+
+class NodeProvides(BaseModel):
+    event: list[RIDType] = []
+    state: list[RIDType] = []
+
+class NodeProfile(BaseModel):
+    base_url: str | None = None
+    node_type: NodeType
+    provides: NodeProvides = NodeProvides()