koi-net 1.2.4__py3-none-any.whl

koi_net/processor/pipeline.py

@@ -0,0 +1,222 @@
+import structlog
+from rid_lib.types import KoiNetEdge, KoiNetNode
+from rid_lib.ext import Cache
+
+from ..exceptions import RequestError
+from ..protocol.event import EventType
+from ..network.request_handler import RequestHandler
+from ..network.event_queue import EventQueue
+from ..network.graph import NetworkGraph
+from ..identity import NodeIdentity
+from .handler import (
+    KnowledgeHandler,
+    HandlerType, 
+    STOP_CHAIN,
+    StopChain
+)
+from .knowledge_object import KnowledgeObject
+from .context import HandlerContext
+
+log = structlog.stdlib.get_logger()
+
+
+class KnowledgePipeline:
+    handler_context: HandlerContext
+    cache: Cache
+    identity: NodeIdentity
+    request_handler: RequestHandler
+    event_queue: EventQueue
+    graph: NetworkGraph
+    knowledge_handlers: list[KnowledgeHandler]
+    
+    def __init__(
+        self, 
+        handler_context: HandlerContext,
+        cache: Cache, 
+        request_handler: RequestHandler,
+        event_queue: EventQueue,
+        graph: NetworkGraph,
+        knowledge_handlers: list[KnowledgeHandler]
+    ):
+        self.handler_context = handler_context
+        self.cache = cache
+        self.request_handler = request_handler
+        self.event_queue = event_queue
+        self.graph = graph
+        self.knowledge_handlers = knowledge_handlers
+    
+    def call_handler_chain(
+        self, 
+        handler_type: HandlerType,
+        kobj: KnowledgeObject
+    ) -> KnowledgeObject | StopChain:
+        """Calls handlers of provided type, chaining their inputs and outputs together.
+        
+        The knowledge object provided when this function is called will be passed to the first handler. A handler may return one of three types: 
+        - `KnowledgeObject` - to modify the knowledge object for the next handler in the chain
+        - `None` - to keep the same knowledge object for the next handler in the chain
+        - `STOP_CHAIN` - to stop the handler chain and immediately exit the processing pipeline
+        
+        Handlers will only be called in the chain if their handler and RID type match that of the inputted knowledge object. 
+        """
+        
+        for handler in self.knowledge_handlers:
+            if handler_type != handler.handler_type: 
+                continue
+            
+            if handler.rid_types and type(kobj.rid) not in handler.rid_types:
+                continue
+            
+            if handler.event_types and kobj.event_type not in handler.event_types:
+                continue
+            
+            log.debug(f"Calling {handler_type} handler '{handler.func.__name__}'")
+            
+            resp = handler(
+                ctx=self.handler_context,
+                kobj=kobj.model_copy()
+            )
+            
+            # stops handler chain execution
+            if resp is STOP_CHAIN:
+                log.debug(f"Handler chain stopped by {handler.func.__name__}")
+                return STOP_CHAIN
+            
+            # kobj unmodified
+            elif resp is None:
+                continue
+            
+            # kobj modified by handler
+            elif isinstance(resp, KnowledgeObject):
+                kobj = resp
+                log.debug(f"Knowledge object modified by {handler.func.__name__}")
+            
+            else:
+                raise ValueError(f"Handler {handler.func.__name__} returned invalid response '{resp}'")
+            
+        return kobj
+    
+    def process(self, kobj: KnowledgeObject):
+        """Sends knowledge object through knowledge processing pipeline.
+        
+        Handler chains are called in between major events in the 
+        pipeline, indicated by their handler type. Each handler type is 
+        guaranteed to have access to certain knowledge, and may affect a 
+        subsequent action in the pipeline. The five handler types are as 
+        follows:
+        - RID - provided RID; if event type is `FORGET`, this handler 
+        decides whether to delete the knowledge from the cache by 
+        setting the normalized event type to `FORGET`, otherwise this 
+        handler decides whether to validate the manifest (and fetch it 
+        if not provided). After processing, if event type is `FORGET`, 
+        the manifest and contents will be retrieved from the local cache, 
+        and indicate the last state of the knowledge before it was 
+        deleted.
+        - Manifest - provided RID, manifest; decides whether to validate 
+        the bundle (and fetch it if not provided).
+        - Bundle - provided RID, manifest, contents (bundle); decides 
+        whether to write knowledge to the cache by setting the 
+        normalized event type to `NEW` or `UPDATE`.
+        - Network - provided RID, manifest, contents (bundle); decides 
+        which nodes (if any) to broadcast an event about this knowledge 
+        to.
+        - Final - provided RID, manifests, contents (bundle); final 
+        action taken after network broadcast.
+        
+        The pipeline may be stopped by any point by a single handler 
+        returning the `STOP_CHAIN` sentinel. In that case, the process 
+        will exit immediately. Further handlers of that type and later 
+        handler chains will not be called.
+        """
+        
+        log.debug(f"Handling {kobj!r}")
+        kobj = self.call_handler_chain(HandlerType.RID, kobj)
+        if kobj is STOP_CHAIN: return
+        
+        if kobj.event_type == EventType.FORGET:
+            bundle = self.cache.read(kobj.rid)
+            if not bundle:
+                log.debug("Local bundle not found")
+                return
+            
+            # the bundle (to be deleted) attached to kobj for downstream analysis
+            log.debug("Adding local bundle (to be deleted) to knowledge object")
+            kobj.manifest = bundle.manifest
+            kobj.contents = bundle.contents
+            
+        else:
+            # attempt to retrieve manifest
+            if not kobj.manifest:
+                log.debug("Manifest not found")
+                if not kobj.source:
+                    return
+            
+                log.debug("Attempting to fetch remote manifest from source")
+                try:
+                    payload = self.request_handler.fetch_manifests(
+                        node=kobj.source,
+                        rids=[kobj.rid])
+                except RequestError:
+                    log.debug("Failed to find manifest")
+                    return
+                
+                kobj.manifest = payload.manifests[0]
+                
+            kobj = self.call_handler_chain(HandlerType.Manifest, kobj)
+            if kobj is STOP_CHAIN: return
+            
+            # attempt to retrieve bundle
+            if not kobj.contents:
+                log.debug("Bundle not found")
+                if kobj.source is None:
+                    return
+                
+                log.debug("Attempting to fetch remote bundle from source")
+                try:
+                    payload = self.request_handler.fetch_bundles(
+                        node=kobj.source,
+                        rids=[kobj.rid]
+                    )
+                except RequestError:
+                    log.debug("Failed to find bundle")
+                    return
+                
+                bundle = payload.bundles[0]
+                
+                if kobj.manifest != bundle.manifest:
+                    log.warning("Retrieved bundle contains a different manifest")
+                
+                kobj.manifest = bundle.manifest
+                kobj.contents = bundle.contents
+                
+        kobj = self.call_handler_chain(HandlerType.Bundle, kobj)
+        if kobj is STOP_CHAIN: return
+            
+        if kobj.normalized_event_type in (EventType.UPDATE, EventType.NEW):
+            log.info(f"Writing to cache: {kobj!r}")
+            self.cache.write(kobj.bundle)
+            
+        elif kobj.normalized_event_type == EventType.FORGET:
+            log.info(f"Deleting from cache: {kobj!r}")
+            self.cache.delete(kobj.rid)
+            
+        else:
+            log.debug("Normalized event type was not set, no cache or network operations will occur")
+            return
+        
+        if type(kobj.rid) in (KoiNetNode, KoiNetEdge):
+            log.debug("Change to node or edge, regenerating network graph")
+            self.graph.generate()
+        
+        kobj = self.call_handler_chain(HandlerType.Network, kobj)
+        if kobj is STOP_CHAIN: return
+        
+        if kobj.network_targets:
+            log.debug(f"Broadcasting event to {len(kobj.network_targets)} network target(s)")
+        else:
+            log.debug("No network targets set")
+        
+        for node in kobj.network_targets:
+            self.event_queue.push(kobj.normalized_event, node)
+        
+        kobj = self.call_handler_chain(HandlerType.Final, kobj)

koi_net/protocol/api_models.py

@@ -0,0 +1,67 @@
+"""Pydantic models for request and response objects in the KOI-net API."""
+
+from typing import Annotated, Literal
+from pydantic import BaseModel, Field
+from rid_lib import RID, RIDType
+from rid_lib.ext import Bundle, Manifest
+from .event import Event
+from .errors import ErrorType
+
+
+# REQUEST MODELS
+
+class PollEvents(BaseModel):
+    type: Literal["poll_events"] = Field("poll_events")
+    limit: int = 0
+    
+class FetchRids(BaseModel):
+    type: Literal["fetch_rids"] = Field("fetch_rids")
+    rid_types: list[RIDType] = []
+    
+class FetchManifests(BaseModel):
+    type: Literal["fetch_manifests"] = Field("fetch_manifests")
+    rid_types: list[RIDType] = []
+    rids: list[RID] = []
+    
+class FetchBundles(BaseModel):
+    type: Literal["fetch_bundles"] = Field("fetch_bundles")
+    rids: list[RID]
+    
+
+# RESPONSE/PAYLOAD MODELS
+
+class RidsPayload(BaseModel):
+    type: Literal["rids_payload"] = Field("rids_payload")
+    rids: list[RID]
+
+class ManifestsPayload(BaseModel):
+    type: Literal["manifests_payload"] = Field("manifests_payload")
+    manifests: list[Manifest]
+    not_found: list[RID] = []
+    
+class BundlesPayload(BaseModel):
+    type: Literal["bundles_payload"] = Field("bundles_payload")
+    bundles: list[Bundle]
+    not_found: list[RID] = []
+    deferred: list[RID] = []
+    
+class EventsPayload(BaseModel):
+    type: Literal["events_payload"] = Field("events_payload")
+    events: list[Event]
+    
+
+# ERROR MODELS
+
+class ErrorResponse(BaseModel):
+    type: Literal["error_response"] = Field("error_response")
+    error: ErrorType
+
+# TYPES
+
+type RequestModels = EventsPayload | PollEvents | FetchRids | FetchManifests | FetchBundles
+type ResponseModels = RidsPayload | ManifestsPayload | BundlesPayload | EventsPayload | ErrorResponse
+
+type ApiModels = Annotated[
+    RequestModels | ResponseModels,
+    Field(discriminator="type")
+]

koi_net/protocol/consts.py

@@ -0,0 +1,7 @@
+"""API paths for KOI-net protocol."""
+
+BROADCAST_EVENTS_PATH = "/events/broadcast"
+POLL_EVENTS_PATH      = "/events/poll"
+FETCH_RIDS_PATH       = "/rids/fetch"
+FETCH_MANIFESTS_PATH  = "/manifests/fetch"
+FETCH_BUNDLES_PATH    = "/bundles/fetch"

koi_net/protocol/edge.py

@@ -0,0 +1,50 @@
+from enum import StrEnum
+from pydantic import BaseModel
+from rid_lib import RIDType
+from rid_lib.ext.bundle import Bundle
+from rid_lib.ext.utils import sha256_hash
+from rid_lib.types import KoiNetEdge, KoiNetNode
+
+
+class EdgeStatus(StrEnum):
+    PROPOSED = "PROPOSED"
+    APPROVED = "APPROVED"
+    
+class EdgeType(StrEnum):
+    WEBHOOK = "WEBHOOK"
+    POLL = "POLL"
+
+class EdgeProfile(BaseModel):
+    source: KoiNetNode
+    target: KoiNetNode
+    edge_type: EdgeType
+    status: EdgeStatus
+    rid_types: list[RIDType]
+
+
+def generate_edge_bundle(
+    source: KoiNetNode,
+    target: KoiNetNode,
+    rid_types: list[RIDType],
+    edge_type: EdgeType
+) -> Bundle:
+    """Returns edge bundle."""
+    
+    edge_rid = KoiNetEdge(sha256_hash(
+        str(source) + str(target)
+    ))
+    
+    edge_profile = EdgeProfile(
+        source=source,
+        target=target,
+        rid_types=rid_types,
+        edge_type=edge_type,
+        status=EdgeStatus.PROPOSED
+    )
+    
+    edge_bundle = Bundle.generate(
+        edge_rid,
+        edge_profile.model_dump()
+    )
+    
+    return edge_bundle

koi_net/protocol/envelope.py

@@ -0,0 +1,65 @@
+import structlog
+from typing import Generic, TypeVar
+from pydantic import BaseModel, ConfigDict
+from rid_lib.types import KoiNetNode
+
+from .secure import PrivateKey, PublicKey
+from .api_models import RequestModels, ResponseModels
+
+log = structlog.stdlib.get_logger()
+
+
+T = TypeVar("T", bound=RequestModels | ResponseModels)
+
+class SignedEnvelope(BaseModel, Generic[T]):
+    payload: T
+    source_node: KoiNetNode
+    target_node: KoiNetNode
+    signature: str
+    
+    model_config = ConfigDict(exclude_none=True)
+    
+    def verify_with(self, pub_key: PublicKey):
+        """Verifies signed envelope with public key.
+        
+        Raises `cryptography.exceptions.InvalidSignature` on failure.
+        """
+        
+        # IMPORTANT: calling `model_dump()` loses all typing! when converting between SignedEnvelope and UnsignedEnvelope, use the Pydantic classes, not the dictionary form
+        
+        unsigned_envelope = UnsignedEnvelope[T](
+            payload=self.payload,
+            source_node=self.source_node,
+            target_node=self.target_node 
+        )
+        
+        log.debug(f"Verifying envelope: {unsigned_envelope.model_dump_json(exclude_none=True)}")
+
+        pub_key.verify(
+            self.signature,
+            unsigned_envelope.model_dump_json(exclude_none=True).encode()
+        )
+
+class UnsignedEnvelope(BaseModel, Generic[T]):
+    payload: T
+    source_node: KoiNetNode
+    target_node: KoiNetNode
+    
+    model_config = ConfigDict(exclude_none=True)
+    
+    def sign_with(self, priv_key: PrivateKey) -> SignedEnvelope[T]:
+        """Signs with private key and returns `SignedEnvelope`."""
+        
+        log.debug(f"Signing envelope: {self.model_dump_json(exclude_none=True)}")
+        log.debug(f"Type: [{type(self.payload)}]")
+        
+        signature = priv_key.sign(
+            self.model_dump_json(exclude_none=True).encode()
+        )
+        
+        return SignedEnvelope(
+            payload=self.payload,
+            source_node=self.source_node,
+            target_node=self.target_node,
+            signature=signature
+        )

koi_net/protocol/errors.py

@@ -0,0 +1,24 @@
+"""Defines KOI-net protocol errors."""
+
+from enum import StrEnum
+from ..exceptions import (
+    ProtocolError,
+    UnknownNodeError,
+    InvalidKeyError,
+    InvalidSignatureError,
+    InvalidTargetError
+)
+
+
+class ErrorType(StrEnum):
+    UnknownNode = "unknown_node"
+    InvalidKey = "invalid_key"
+    InvalidSignature = "invalid_signature"
+    InvalidTarget = "invalid_target"
+
+EXCEPTION_TO_ERROR_TYPE: dict[ProtocolError, ErrorType] = {
+    UnknownNodeError: ErrorType.UnknownNode,
+    InvalidKeyError: ErrorType.InvalidKey,
+    InvalidSignatureError: ErrorType.InvalidSignature,
+    InvalidTargetError: ErrorType.InvalidTarget
+}

koi_net/protocol/event.py

@@ -0,0 +1,51 @@
+from enum import StrEnum
+from pydantic import BaseModel
+from rid_lib import RID
+from rid_lib.ext import Manifest, Bundle
+
+
+class EventType(StrEnum):
+    NEW = "NEW"
+    UPDATE = "UPDATE"
+    FORGET = "FORGET"
+
+class Event(BaseModel):
+    rid: RID
+    event_type: EventType
+    manifest: Manifest | None = None
+    contents: dict | None = None
+    
+    def __repr__(self):
+        return f"<Event '{self.rid}' event type: '{self.event_type}'>"
+    
+    @classmethod
+    def from_bundle(cls, event_type: EventType, bundle: Bundle):
+        return cls(
+            rid=bundle.manifest.rid,
+            event_type=event_type,
+            manifest=bundle.manifest,
+            contents=bundle.contents
+        )
+        
+    @classmethod
+    def from_manifest(cls, event_type: EventType, manifest: Manifest):
+        return cls(
+            rid=manifest.rid,
+            event_type=event_type,
+            manifest=manifest
+        )
+        
+    @classmethod
+    def from_rid(cls, event_type: EventType, rid: RID):
+        return cls(
+            rid=rid,
+            event_type=event_type
+        )
+        
+    @property
+    def bundle(self):
+        if self.manifest is not None and self.contents is not None:
+            return Bundle(
+                manifest=self.manifest,
+                contents=self.contents
+            )

koi_net/protocol/model_map.py

@@ -0,0 +1,62 @@
+from typing import NamedTuple
+from pydantic import BaseModel
+from .envelope import SignedEnvelope
+from .consts import (
+    BROADCAST_EVENTS_PATH, 
+    POLL_EVENTS_PATH,
+    FETCH_BUNDLES_PATH, 
+    FETCH_MANIFESTS_PATH, 
+    FETCH_RIDS_PATH
+)
+from .api_models import (
+    EventsPayload,
+    PollEvents,
+    FetchBundles,
+    BundlesPayload,
+    FetchManifests,
+    ManifestsPayload,
+    FetchRids,
+    RidsPayload
+)
+
+
+class Models(NamedTuple):
+    request: type[BaseModel]
+    response: type[BaseModel] | None
+    request_envelope: type[SignedEnvelope]
+    response_envelope: type[SignedEnvelope] | None
+
+
+# maps API paths to request and response models
+API_MODEL_MAP: dict[str, Models] = {
+    BROADCAST_EVENTS_PATH: Models(
+        request=EventsPayload,
+        response=None,
+        request_envelope=SignedEnvelope[EventsPayload],
+        response_envelope=None
+    ),
+    POLL_EVENTS_PATH: Models(
+        request=PollEvents,
+        response=EventsPayload,
+        request_envelope=SignedEnvelope[PollEvents],
+        response_envelope=SignedEnvelope[EventsPayload]
+    ),
+    FETCH_BUNDLES_PATH: Models(
+        request=FetchBundles,
+        response=BundlesPayload,
+        request_envelope=SignedEnvelope[FetchBundles],
+        response_envelope=SignedEnvelope[BundlesPayload]
+    ),
+    FETCH_MANIFESTS_PATH: Models(
+        request=FetchManifests,
+        response=ManifestsPayload,
+        request_envelope=SignedEnvelope[FetchManifests],
+        response_envelope=SignedEnvelope[ManifestsPayload]
+    ),
+    FETCH_RIDS_PATH: Models(
+        request=FetchRids,
+        response=RidsPayload,
+        request_envelope=SignedEnvelope[FetchRids],
+        response_envelope=SignedEnvelope[RidsPayload]
+    )
+}

koi_net/protocol/node.py

@@ -0,0 +1,18 @@
+from enum import StrEnum
+from pydantic import BaseModel
+from rid_lib import RIDType
+
+
+class NodeType(StrEnum):
+    FULL = "FULL"
+    PARTIAL = "PARTIAL"
+
+class NodeProvides(BaseModel):
+    event: list[RIDType] = []
+    state: list[RIDType] = []
+
+class NodeProfile(BaseModel):
+    base_url: str | None = None
+    node_type: NodeType
+    provides: NodeProvides = NodeProvides()
+    public_key: str | None = None