koi-net 1.1.0b8__py3-none-any.whl → 1.2.0b2__py3-none-any.whl

koi_net/processor/{knowledge_pipeline.py → pipeline.py}

@@ -1,11 +1,9 @@
-import logging
-from typing import Callable
-from rid_lib.core import RIDType
+import structlog
 from rid_lib.types import KoiNetEdge, KoiNetNode
 from rid_lib.ext import Cache
 from ..protocol.event import EventType
 from ..network.request_handler import RequestHandler
-from ..network.event_queue import NetworkEventQueue
+from ..network.event_queue import EventQueue
 from ..network.graph import NetworkGraph
 from ..identity import NodeIdentity
 from .handler import (
@@ -15,55 +13,36 @@ from .handler import (
     StopChain
 )
 from .knowledge_object import KnowledgeObject
+from ..context import HandlerContext
 
-from typing import TYPE_CHECKING
-if TYPE_CHECKING:
-    from ..context import HandlerContext
-
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class KnowledgePipeline:
-    handler_context: "HandlerContext"
+    handler_context: HandlerContext
     cache: Cache
     identity: NodeIdentity
     request_handler: RequestHandler
-    event_queue: NetworkEventQueue
+    event_queue: EventQueue
     graph: NetworkGraph
-    handlers: list[KnowledgeHandler]
+    knowledge_handlers: list[KnowledgeHandler]
     
     def __init__(
         self, 
-        handler_context: "HandlerContext",
+        handler_context: HandlerContext,
         cache: Cache, 
         request_handler: RequestHandler,
-        event_queue: NetworkEventQueue,
+        event_queue: EventQueue,
         graph: NetworkGraph,
-        default_handlers: list[KnowledgeHandler] = []
+        knowledge_handlers: list[KnowledgeHandler] = []
     ):
         self.handler_context = handler_context
         self.cache = cache
         self.request_handler = request_handler
         self.event_queue = event_queue
         self.graph = graph
-        self.handlers = default_handlers
+        self.knowledge_handlers = knowledge_handlers
     
-    def add_handler(self, handler: KnowledgeHandler):
-        self.handlers.append(handler)
-            
-    def register_handler(
-        self,
-        handler_type: HandlerType,
-        rid_types: list[RIDType] | None = None,
-        event_types: list[EventType | None] | None = None
-    ):
-        """Assigns decorated function as handler for this processor."""
-        def decorator(func: Callable) -> Callable:
-            handler = KnowledgeHandler(func, handler_type, rid_types, event_types)
-            self.add_handler(handler)
-            return func
-        return decorator
-            
     def call_handler_chain(
         self, 
         handler_type: HandlerType,
@@ -79,7 +58,7 @@ class KnowledgePipeline:
         Handlers will only be called in the chain if their handler and RID type match that of the inputted knowledge object. 
         """
         
-        for handler in self.handlers:
+        for handler in self.knowledge_handlers:
             if handler_type != handler.handler_type: 
                 continue
             
@@ -89,7 +68,7 @@ class KnowledgePipeline:
             if handler.event_types and kobj.event_type not in handler.event_types:
                 continue
             
-            logger.debug(f"Calling {handler_type} handler '{handler.func.__name__}'")
+            log.debug(f"Calling {handler_type} handler '{handler.func.__name__}'")
             
             resp = handler.func(
                 ctx=self.handler_context,
@@ -98,7 +77,7 @@ class KnowledgePipeline:
             
             # stops handler chain execution
             if resp is STOP_CHAIN:
-                logger.debug(f"Handler chain stopped by {handler.func.__name__}")
+                log.debug(f"Handler chain stopped by {handler.func.__name__}")
                 return STOP_CHAIN
             # kobj unmodified
             elif resp is None:
@@ -106,7 +85,7 @@ class KnowledgePipeline:
             # kobj modified by handler
             elif isinstance(resp, KnowledgeObject):
                 kobj = resp
-                logger.debug(f"Knowledge object modified by {handler.func.__name__}")
+                log.debug(f"Knowledge object modified by {handler.func.__name__}")
             else:
                 raise ValueError(f"Handler {handler.func.__name__} returned invalid response '{resp}'")
                     
@@ -125,36 +104,36 @@ class KnowledgePipeline:
         The pipeline may be stopped by any point by a single handler returning the `STOP_CHAIN` sentinel. In that case, the process will exit immediately. Further handlers of that type and later handler chains will not be called.
         """
         
-        logger.debug(f"Handling {kobj!r}")
+        log.debug(f"Handling {kobj!r}")
         kobj = self.call_handler_chain(HandlerType.RID, kobj)
         if kobj is STOP_CHAIN: return
         
         if kobj.event_type == EventType.FORGET:
             bundle = self.cache.read(kobj.rid)
             if not bundle: 
-                logger.debug("Local bundle not found")
+                log.debug("Local bundle not found")
                 return
             
             # the bundle (to be deleted) attached to kobj for downstream analysis
-            logger.debug("Adding local bundle (to be deleted) to knowledge object")
+            log.debug("Adding local bundle (to be deleted) to knowledge object")
             kobj.manifest = bundle.manifest
             kobj.contents = bundle.contents
             
         else:
             # attempt to retrieve manifest
             if not kobj.manifest:
-                logger.debug("Manifest not found")
+                log.debug("Manifest not found")
                 if not kobj.source:
                     return
             
-                logger.debug("Attempting to fetch remote manifest from source")
+                log.debug("Attempting to fetch remote manifest from source")
                 payload = self.request_handler.fetch_manifests(
                     node=kobj.source,
                     rids=[kobj.rid]
                 )
                     
                 if not payload.manifests:
-                    logger.debug("Failed to find manifest")
+                    log.debug("Failed to find manifest")
                     return
                 
                 kobj.manifest = payload.manifests[0]
@@ -164,24 +143,24 @@ class KnowledgePipeline:
             
             # attempt to retrieve bundle
             if not kobj.bundle:
-                logger.debug("Bundle not found")
+                log.debug("Bundle not found")
                 if kobj.source is None:
                     return
                 
-                logger.debug("Attempting to fetch remote bundle from source")
+                log.debug("Attempting to fetch remote bundle from source")
                 payload = self.request_handler.fetch_bundles(
                     node=kobj.source,
                     rids=[kobj.rid]
                 )
                 
                 if not payload.bundles:
-                    logger.debug("Failed to find bundle")
+                    log.debug("Failed to find bundle")
                     return
                 
                 bundle = payload.bundles[0]                    
                 
                 if kobj.manifest != bundle.manifest:
-                    logger.warning("Retrieved bundle contains a different manifest")
+                    log.warning("Retrieved bundle contains a different manifest")
                 
                 kobj.manifest = bundle.manifest
                 kobj.contents = bundle.contents                
@@ -190,31 +169,30 @@ class KnowledgePipeline:
         if kobj is STOP_CHAIN: return
             
         if kobj.normalized_event_type in (EventType.UPDATE, EventType.NEW):
-            logger.info(f"Writing to cache: {kobj!r}")
+            log.info(f"Writing to cache: {kobj!r}")
             self.cache.write(kobj.bundle)
             
         elif kobj.normalized_event_type == EventType.FORGET:
-            logger.info(f"Deleting from cache: {kobj!r}")
+            log.info(f"Deleting from cache: {kobj!r}")
             self.cache.delete(kobj.rid)
             
         else:
-            logger.debug("Normalized event type was never set, no cache or network operations will occur")
+            log.debug("Normalized event type was never set, no cache or network operations will occur")
             return
         
         if type(kobj.rid) in (KoiNetNode, KoiNetEdge):
-            logger.debug("Change to node or edge, regenerating network graph")
+            log.debug("Change to node or edge, regenerating network graph")
             self.graph.generate()
         
         kobj = self.call_handler_chain(HandlerType.Network, kobj)
         if kobj is STOP_CHAIN: return
         
         if kobj.network_targets:
-            logger.debug(f"Broadcasting event to {len(kobj.network_targets)} network target(s)")
+            log.debug(f"Broadcasting event to {len(kobj.network_targets)} network target(s)")
         else:
-            logger.debug("No network targets set")
+            log.debug("No network targets set")
         
         for node in kobj.network_targets:
             self.event_queue.push_event_to(kobj.normalized_event, node)
-            self.event_queue.flush_webhook_queue(node)
         
         kobj = self.call_handler_chain(HandlerType.Final, kobj)

koi_net/protocol/api_models.py

@@ -1,6 +1,6 @@
-"""Pydantic models for request and response/payload objects in the KOI-net API."""
+"""Pydantic models for request and response objects in the KOI-net API."""
 
-from typing import Literal
+from typing import Annotated, Literal
 from pydantic import BaseModel, Field
 from rid_lib import RID, RIDType
 from rid_lib.ext import Bundle, Manifest
@@ -59,4 +59,8 @@ class ErrorResponse(BaseModel):
 # TYPES
 
 type RequestModels = EventsPayload | PollEvents | FetchRids | FetchManifests | FetchBundles
-type ResponseModels = RidsPayload | ManifestsPayload | BundlesPayload | EventsPayload | ErrorResponse
+type ResponseModels = RidsPayload | ManifestsPayload | BundlesPayload | EventsPayload | ErrorResponse
+type ApiModels = Annotated[
+    RequestModels | ResponseModels,
+    Field(discriminator="type")
+]

koi_net/protocol/envelope.py

@@ -1,4 +1,4 @@
-import logging
+import structlog
 from typing import Generic, TypeVar
 from pydantic import BaseModel, ConfigDict
 from rid_lib.types import KoiNetNode
@@ -6,8 +6,7 @@ from rid_lib.types import KoiNetNode
 from .secure import PrivateKey, PublicKey
 from .api_models import RequestModels, ResponseModels
 
-
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 T = TypeVar("T", bound=RequestModels | ResponseModels)
@@ -28,7 +27,7 @@ class SignedEnvelope(BaseModel, Generic[T]):
             target_node=self.target_node 
         )
         
-        logger.debug(f"Verifying envelope: {unsigned_envelope.model_dump_json(exclude_none=True)}")
+        log.debug(f"Verifying envelope: {unsigned_envelope.model_dump_json(exclude_none=True)}")
                 
         pub_key.verify(
             self.signature,
@@ -43,8 +42,8 @@ class UnsignedEnvelope(BaseModel, Generic[T]):
     target_node: KoiNetNode
     
     def sign_with(self, priv_key: PrivateKey) -> SignedEnvelope[T]:
-        logger.debug(f"Signing envelope: {self.model_dump_json(exclude_none=True)}")
-        logger.debug(f"Type: [{type(self.payload)}]")
+        log.debug(f"Signing envelope: {self.model_dump_json(exclude_none=True)}")
+        log.debug(f"Type: [{type(self.payload)}]")
         
         signature = priv_key.sign(
             self.model_dump_json(exclude_none=True).encode()

koi_net/protocol/model_map.py

@@ -0,0 +1,61 @@
+from typing import NamedTuple
+from pydantic import BaseModel
+from .envelope import SignedEnvelope
+from .consts import (
+    BROADCAST_EVENTS_PATH, 
+    POLL_EVENTS_PATH,
+    FETCH_BUNDLES_PATH, 
+    FETCH_MANIFESTS_PATH, 
+    FETCH_RIDS_PATH
+)
+from .api_models import (
+    EventsPayload,
+    PollEvents,
+    FetchBundles,
+    BundlesPayload,
+    FetchManifests,
+    ManifestsPayload,
+    FetchRids,
+    RidsPayload
+)
+
+
+class Models(NamedTuple):
+    request: type[BaseModel]
+    response: type[BaseModel] | None
+    request_envelope: type[SignedEnvelope]
+    response_envelope: type[SignedEnvelope] | None
+
+
+API_MODEL_MAP: dict[str, Models] = {
+    BROADCAST_EVENTS_PATH: Models(
+        request=EventsPayload,
+        response=None,
+        request_envelope=SignedEnvelope[EventsPayload],
+        response_envelope=None
+    ),
+    POLL_EVENTS_PATH: Models(
+        request=PollEvents,
+        response=EventsPayload,
+        request_envelope=SignedEnvelope[PollEvents],
+        response_envelope=SignedEnvelope[EventsPayload]
+    ),
+    FETCH_BUNDLES_PATH: Models(
+        request=FetchBundles,
+        response=BundlesPayload,
+        request_envelope=SignedEnvelope[FetchBundles],
+        response_envelope=SignedEnvelope[BundlesPayload]
+    ),
+    FETCH_MANIFESTS_PATH: Models(
+        request=FetchManifests,
+        response=ManifestsPayload,
+        request_envelope=SignedEnvelope[FetchManifests],
+        response_envelope=SignedEnvelope[ManifestsPayload]
+    ),
+    FETCH_RIDS_PATH: Models(
+        request=FetchRids,
+        response=RidsPayload,
+        request_envelope=SignedEnvelope[FetchRids],
+        response_envelope=SignedEnvelope[RidsPayload]
+    )
+}

koi_net/protocol/node.py

@@ -1,5 +1,5 @@
 from enum import StrEnum
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 from rid_lib import RIDType
 
 
@@ -8,8 +8,8 @@ class NodeType(StrEnum):
     PARTIAL = "PARTIAL"
 
 class NodeProvides(BaseModel):
-    event: list[RIDType] = []
-    state: list[RIDType] = []
+    event: list[RIDType] = Field(default_factory=list)
+    state: list[RIDType] = Field(default_factory=list)
 
 class NodeProfile(BaseModel):
     base_url: str | None = None

koi_net/protocol/secure.py

@@ -1,4 +1,4 @@
-import logging
+import structlog
 from base64 import b64decode, b64encode
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.asymmetric import ec
@@ -9,7 +9,7 @@ from cryptography.hazmat.primitives.asymmetric.utils import (
     encode_dss_signature
 )
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 def der_to_raw_signature(der_signature: bytes, curve=ec.SECP256R1()) -> bytes:
@@ -91,9 +91,9 @@ class PrivateKey:
         
         signature = b64encode(raw_signature_bytes).decode()
         
-        logger.debug(f"Signing message with [{self.public_key().to_der()}]")
-        logger.debug(f"hash: {hashed_message}")
-        logger.debug(f"signature: {signature}")
+        log.debug(f"Signing message with [{self.public_key().to_der()}]")
+        log.debug(f"hash: {hashed_message}")
+        log.debug(f"signature: {signature}")
         
         return signature
                 
@@ -144,9 +144,9 @@ class PublicKey:
         # print()
         # print(message.decode())
 
-        # logger.debug(f"Verifying message with [{self.to_der()}]")
-        # logger.debug(f"hash: {hashed_message}")
-        # logger.debug(f"signature: {signature}")
+        # log.debug(f"Verifying message with [{self.to_der()}]")
+        # log.debug(f"hash: {hashed_message}")
+        # log.debug(f"signature: {signature}")
         
         raw_signature_bytes = b64decode(signature)
         der_signature_bytes = raw_to_der_signature(raw_signature_bytes)

koi_net/secure.py

@@ -1,13 +1,14 @@
-import logging
+import structlog
 from functools import wraps
 
 import cryptography.exceptions
-from rid_lib.ext import Bundle
+from rid_lib.ext import Bundle, Cache
 from rid_lib.ext.utils import sha256_hash
+from rid_lib.types import KoiNetNode
 from .identity import NodeIdentity
 from .protocol.envelope import UnsignedEnvelope, SignedEnvelope
 from .protocol.secure import PublicKey
-from .protocol.api_models import EventsPayload
+from .protocol.api_models import ApiModels, EventsPayload
 from .protocol.event import EventType
 from .protocol.node import NodeProfile
 from .protocol.secure import PrivateKey
@@ -17,31 +18,32 @@ from .protocol.errors import (
     InvalidSignatureError,
     InvalidTargetError
 )
-from .effector import Effector
 from .config import NodeConfig
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class Secure:
+    """Subsystem handling secure protocol logic."""
     identity: NodeIdentity
-    effector: Effector
+    cache: Cache
     config: NodeConfig
     priv_key: PrivateKey
     
     def __init__(
         self, 
         identity: NodeIdentity, 
-        effector: Effector,
+        cache: Cache,
         config: NodeConfig
     ):
         self.identity = identity
-        self.effector = effector
+        self.cache = cache
         self.config = config
 
         self.priv_key = self._load_priv_key()
         
     def _load_priv_key(self) -> PrivateKey:
+        """Loads private key from PEM file path in config."""
         with open(self.config.koi_net.private_key_pem_path, "r") as f:
             priv_key_pem = f.read()
         
@@ -51,6 +53,14 @@ class Secure:
         )
         
     def _handle_unknown_node(self, envelope: SignedEnvelope) -> Bundle | None:
+        """Attempts to find node profile in proided envelope.
+        
+        If an unknown node sends an envelope, it may still be able to be
+        validated if that envelope contains their node profile. This is
+        essential for allowing unknown nodes to handshake and introduce
+        themselves. Only an `EventsPayload` contain a `NEW` event for a 
+        node profile for the source node is permissible.
+        """
         if type(envelope.payload) != EventsPayload:
             return None
             
@@ -64,7 +74,10 @@ class Secure:
             return event.bundle
         return None
         
-    def create_envelope(self, payload, target) -> SignedEnvelope:
+    def create_envelope(
+        self, payload: ApiModels, target: KoiNetNode
+    ) -> SignedEnvelope:
+        """Returns signed envelope to target from provided payload."""
         return UnsignedEnvelope(
             payload=payload,
             source_node=self.identity.rid,
@@ -72,8 +85,10 @@ class Secure:
         ).sign_with(self.priv_key)
         
     def validate_envelope(self, envelope: SignedEnvelope):
+        """Validates signed envelope from another node."""
+        
         node_bundle = (
-            self.effector.deref(envelope.source_node) or
+            self.cache.read(envelope.source_node) or
             self._handle_unknown_node(envelope)
         )
         
@@ -98,17 +113,22 @@ class Secure:
             raise InvalidTargetError(f"Envelope target {envelope.target_node!r} is not me")
         
     def envelope_handler(self, func):
+        """Wrapper function validates envelopes for server endpoints.
+        
+        Validates incoming envelope and passes payload to endpoint
+        handler. Resulting payload is returned as a signed envelope.
+        """
         @wraps(func)
         async def wrapper(req: SignedEnvelope, *args, **kwargs) -> SignedEnvelope | None:
-            logger.info("Validating envelope")
+            log.info("Validating envelope")
             
             self.validate_envelope(req)            
-            logger.info("Calling endpoint handler")
+            log.info("Calling endpoint handler")
             
             result = await func(req, *args, **kwargs)            
             
             if result is not None:
-                logger.info("Creating response envelope")
+                log.info("Creating response envelope")
                 return self.create_envelope(
                     payload=result,
                     target=req.source_node

koi_net/sentry.py

@@ -0,0 +1,13 @@
+# import sentry_sdk
+# from sentry_sdk.integrations.logging import LoggingIntegration
+
+# sentry_sdk.init(
+#     dsn="https://7bbafef3c7dbd652506db3cb2aca9f98@o4510149352357888.ingest.us.sentry.io/4510149355765760",
+#     # Add data like request headers and IP for users,
+#     # see https://docs.sentry.io/platforms/python/data-management/data-collected/ for more info
+#     send_default_pii=True,
+#     enable_logs=True,
+#     integrations=[
+#         LoggingIntegration(sentry_logs_level=None)
+#     ]
+# )