koi-net 1.2.4__py3-none-any.whl

koi_net/exceptions.py

@@ -0,0 +1,107 @@
+"""KOI-net library exceptions.
+
+Exception hierarchy map:
+- `KoiNetError`
+  - `BuildError`
+  - `RequestError`
+    - `ClientError`
+      - `SelfRequestError`
+      - `PartialNodeQueryError`
+      - `NodeNotFoundError`
+    - `TransportError`
+    - `ServerError`
+      - `RemoteProtocolError`
+        - `RemoteUnknownNodeError`
+        - `RemoteInvalidKeyError`
+        - `RemoteInvalidSignatureError`
+        - `RemoteInvalidTargetError`
+  - `ProtocolError`
+    - `UnknownNodeError`
+    - `InvalidKeyError`
+    - `InvalidSignatureError`
+    - `InvalidTargetError`
+"""
+
+
+# BASE EXCEPTION
+class KoiNetError(Exception):
+    """Base exception."""
+    pass
+
+# BUILD ERRORS
+class BuildError(KoiNetError):
+    """Raised when errors occur in build process."""
+    pass
+
+# NETWORK REQUEST ERRORS
+class RequestError(KoiNetError):
+    """Base for network request errors."""
+    pass
+
+# CLIENT ERRORS
+class ClientError(RequestError):
+    """Raised when this node makes an invalid request."""
+    pass
+
+class SelfRequestError(ClientError):
+    """Raised when this node tries to request itself."""
+    pass
+
+class PartialNodeQueryError(ClientError):
+    """Raised when this node attempts to query a partial node."""
+    pass
+
+class NodeNotFoundError(ClientError):
+    """Raised when this node cannot find a node's URL."""
+    pass
+
+class TransportError(RequestError):
+    """Raised when a transport error occurs during a request."""
+    pass
+
+# SERVER ERRORS
+class ServerError(RequestError):
+    """Raised when an server error occurs during a request."""
+    pass
+
+# PROTOCOL ERRORS
+class RemoteProtocolError(ServerError):
+    """Base for protocol errors raised by peer node."""
+    pass
+
+class RemoteUnknownNodeError(RemoteProtocolError):
+    """Raised by peer node when this node is unknown."""
+    pass
+    
+class RemoteInvalidKeyError(RemoteProtocolError):
+    """Raised by peer node when this node's public key doesn't match their RID."""
+    pass
+    
+class RemoteInvalidSignatureError(RemoteProtocolError):
+    """Raised by peer node when this node's envelope signature is invalid."""
+    pass
+
+class RemoteInvalidTargetError(RemoteProtocolError):
+    """Raised by peer node when this node's envelope target is not it's RID."""
+    pass
+
+
+class ProtocolError(KoiNetError):
+    """Base for protocol errors raised by this node."""
+    pass
+
+class UnknownNodeError(ProtocolError):
+    """Raised when peer node is unknown."""
+    pass
+    
+class InvalidKeyError(ProtocolError):
+    """Raised when peer node's public key doesn't match their RID."""
+    pass
+    
+class InvalidSignatureError(ProtocolError):
+    """Raised when peer node's envelope signature is invalid."""
+    pass
+
+class InvalidTargetError(ProtocolError):
+    """Raised when peer node's target is not this node."""
+    pass

koi_net/identity.py

@@ -0,0 +1,20 @@
+from rid_lib.types import KoiNetNode
+from .config.core import NodeConfig
+from .protocol.node import NodeProfile
+
+
+class NodeIdentity:
+    """Represents a node's identity (RID, profile)."""
+    
+    config: NodeConfig
+    
+    def __init__(self, config: NodeConfig):
+        self.config = config
+        
+    @property
+    def rid(self) -> KoiNetNode:
+        return self.config.koi_net.node_rid
+    
+    @property
+    def profile(self) -> NodeProfile:
+        return self.config.koi_net.node_profile

koi_net/log_system.py

@@ -0,0 +1,133 @@
+"""Configures and initializes the logging system."""
+
+import sys
+import logging
+from logging.handlers import RotatingFileHandler
+from datetime import datetime
+
+import structlog
+import colorama
+
+LOG_FILE_PATH = "log.ndjson"
+MAX_LOG_SIZE = 10 * 1024 ** 2 # 10MB
+MAX_LOG_BACKUPS = 5
+LOG_ENCODING = "utf-8"
+
+
+shared_log_processors = [
+    structlog.stdlib.add_logger_name,
+    structlog.stdlib.add_log_level,
+    structlog.stdlib.PositionalArgumentsFormatter(),
+    structlog.processors.TimeStamper(fmt="iso"),
+    structlog.processors.UnicodeDecoder(),
+    structlog.processors.CallsiteParameterAdder({
+        structlog.processors.CallsiteParameter.MODULE,
+        structlog.processors.CallsiteParameter.FUNC_NAME
+    }),
+]
+
+console_renderer = structlog.dev.ConsoleRenderer(
+    columns=[
+        # Render the timestamp without the key name in yellow.
+        structlog.dev.Column(
+            "timestamp",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=None,
+                value_style=colorama.Style.DIM,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=lambda t: datetime.fromisoformat(t).strftime("%Y-%m-%d %H:%M:%S"),
+            ),
+        ),
+        structlog.dev.Column(
+            "level",
+            structlog.dev.LogLevelColumnFormatter(
+                level_styles={
+                    level: colorama.Style.BRIGHT + color
+                    for level, color in {
+                        "critical": colorama.Fore.RED,
+                        "exception": colorama.Fore.RED,
+                        "error": colorama.Fore.RED,
+                        "warn": colorama.Fore.YELLOW,
+                        "warning": colorama.Fore.YELLOW,
+                        "info": colorama.Fore.GREEN,
+                        "debug": colorama.Fore.GREEN,
+                        "notset": colorama.Back.RED,
+                    }.items()
+                },
+                reset_style=colorama.Style.RESET_ALL,
+                width=9
+            )
+        ),
+        # Render the event without the key name in bright magenta.
+        
+        # Default formatter for all keys not explicitly mentioned. The key is
+        # cyan, the value is green.
+        structlog.dev.Column(
+            "path",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=None,
+                value_style=colorama.Fore.MAGENTA,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=str,
+                width=30
+            ),
+        ),
+        structlog.dev.Column(
+            "event",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=None,
+                value_style=colorama.Fore.WHITE,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=str,
+                width=30
+            ),
+        ),
+        structlog.dev.Column(
+            "",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=colorama.Fore.BLUE,
+                value_style=colorama.Fore.GREEN,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=str,
+            ),
+        )
+    ]
+)
+
+
+console_handler = logging.StreamHandler(sys.stdout)
+console_handler.setFormatter(
+    structlog.stdlib.ProcessorFormatter(
+        processor=console_renderer,
+        foreign_pre_chain=shared_log_processors
+    )
+)
+
+file_handler = RotatingFileHandler(
+    filename=LOG_FILE_PATH,
+    maxBytes=MAX_LOG_SIZE,
+    backupCount=MAX_LOG_BACKUPS,
+    encoding=LOG_ENCODING
+)
+
+file_handler.setFormatter(
+    structlog.stdlib.ProcessorFormatter(
+        processor=structlog.processors.JSONRenderer(),
+        foreign_pre_chain=shared_log_processors
+    )
+)
+
+logging.basicConfig(
+    level=logging.DEBUG, 
+    handlers=[
+        file_handler,
+        console_handler
+    ])
+
+structlog.configure(
+    processors=shared_log_processors + [
+        structlog.stdlib.ProcessorFormatter.wrap_for_formatter],
+    wrapper_class=structlog.stdlib.BoundLogger,
+    logger_factory=structlog.stdlib.LoggerFactory(),
+    cache_logger_on_first_use=True,
+)

koi_net/network/error_handler.py

@@ -0,0 +1,63 @@
+import structlog
+from rid_lib.types import KoiNetNode
+from ..behaviors.handshaker import Handshaker
+from ..protocol.errors import ErrorType
+from ..protocol.event import EventType
+from ..processor.kobj_queue import KobjQueue
+
+log = structlog.stdlib.get_logger()
+
+
+class ErrorHandler:
+    """Handles network and protocol errors that may occur during requests."""
+    timeout_counter: dict[KoiNetNode, int]
+    kobj_queue: KobjQueue
+    
+    def __init__(
+        self, 
+        kobj_queue: KobjQueue,
+        handshaker: Handshaker
+    ):
+        self.kobj_queue = kobj_queue
+        self.handshaker = handshaker
+        self.timeout_counter = {}
+    
+    def reset_timeout_counter(self, node: KoiNetNode):
+        """Reset's a node timeout counter to zero."""
+        self.timeout_counter[node] = 0
+    
+    def handle_connection_error(self, node: KoiNetNode):
+        """Drops nodes after timing out three times.
+        
+        TODO: Need a better heuristic for network state. For example, if
+        a node lost connection to the internet, it would quickly forget
+        all other nodes.
+        """
+        self.timeout_counter.setdefault(node, 0)
+        self.timeout_counter[node] += 1
+        
+        log.debug(f"{node} has timed out {self.timeout_counter[node]} time(s)")
+        
+        if self.timeout_counter[node] > 3:
+            log.debug(f"Exceeded time out limit, forgetting node")
+            self.kobj_queue.push(rid=node, event_type=EventType.FORGET)
+        
+    def handle_protocol_error(
+        self, 
+        error_type: ErrorType, 
+        node: KoiNetNode
+    ):
+        """Handles protocol errors that occur during network requests.
+        
+        Attempts handshake when this node is unknown to target.
+        """
+        
+        log.info(f"Handling protocol error {error_type} for node {node!r}")
+        match error_type:
+            case ErrorType.UnknownNode:
+                log.info("Peer doesn't know me, attempting handshake...")
+                self.handshaker.handshake_with(node)
+                
+            case ErrorType.InvalidKey: ...
+            case ErrorType.InvalidSignature: ...
+            case ErrorType.InvalidTarget: ...

koi_net/network/event_buffer.py

@@ -0,0 +1,91 @@
+import time
+from contextlib import contextmanager
+from typing import Generator
+from rid_lib.types import KoiNetNode
+
+from ..protocol.event import Event
+
+
+class EventBuffer:
+    """Stores outgoing events sent to other nodes."""
+
+    buffers: dict[KoiNetNode, list[Event]]
+    start_time: dict[KoiNetNode, float]
+    
+    def __init__(self):
+        self.buffers = {}
+        self.start_time = {}
+        
+    def push(self, node: KoiNetNode, event: Event):
+        """Pushes event to specified node.
+        
+        Sets start time to now if unset.
+        """
+        
+        self.start_time.setdefault(node, time.time())
+        
+        event_buf = self.buffers.setdefault(node, [])
+        event_buf.append(event)
+        
+    def buf_len(self, node: KoiNetNode):
+        """Returns the length of a node's event buffer."""
+        return len(self.buffers.get(node, []))
+        
+    def flush(self, node: KoiNetNode, limit: int = 0) -> list[Event]:
+        """Flushes all (or limit) events for a node.
+        
+        Resets start time.
+        """
+        self.start_time.pop(node, None)
+        
+        if node not in self.buffers:
+            return []
+        
+        event_buf = self.buffers[node]
+        
+        if limit and len(event_buf) > limit:
+            flushed_events = event_buf[:limit]
+            self.buffers[node] = event_buf[limit:]
+        else:
+            flushed_events = event_buf.copy()
+            del self.buffers[node]
+        
+        return flushed_events
+    
+    @contextmanager
+    def safe_flush(
+        self, 
+        node: KoiNetNode, 
+        limit: int = 0,
+        force_flush: bool = False
+    ) -> Generator[list[Event], None, None]:
+        """Context managed safe flush, only commits on successful exit.
+        
+        Exceptions will result in buffer rollback to the previous state.
+        """
+        
+        self.start_time.pop(node, None)
+        
+        if node not in self.buffers:
+            yield []
+            return
+        
+        event_buf = self.buffers[node].copy()
+        in_place = limit and len(event_buf) > limit
+        
+        try:
+            if in_place:
+                yield event_buf[:limit]
+                self.buffers[node] = event_buf[limit:]
+            else:
+                yield event_buf.copy()
+                self.buffers.pop(node, None)
+        
+        except Exception:
+            # if force, flushes buffers and reraises exception
+            if force_flush:
+                if in_place:
+                    self.buffers[node] = event_buf[limit:]
+                else:
+                    self.buffers.pop(node, None)
+            raise

koi_net/network/event_queue.py

@@ -0,0 +1,31 @@
+import structlog
+from queue import Queue
+
+from rid_lib.types import KoiNetNode
+from pydantic import BaseModel
+
+from ..protocol.event import Event
+
+log = structlog.stdlib.get_logger()
+
+
+class QueuedEvent(BaseModel):
+    event: Event
+    target: KoiNetNode
+
+class EventQueue:
+    """Queue for outgoing network events."""
+    q: Queue[QueuedEvent]
+    
+    def __init__(self):
+        self.q = Queue()
+    
+    def push(self, event: Event, target: KoiNetNode):
+        """Pushes event to queue of specified node.
+        
+        Event will be sent to webhook or poll queue by the event worker
+        depending on the node type and edge type of the specified node.
+        """
+        
+        self.q.put(QueuedEvent(target=target, event=event))
+    

koi_net/network/graph.py

@@ -0,0 +1,123 @@
+import structlog
+from typing import Literal
+import networkx as nx
+from rid_lib import RIDType
+from rid_lib.ext import Cache
+from rid_lib.types import KoiNetEdge, KoiNetNode
+from ..identity import NodeIdentity
+from ..protocol.edge import EdgeProfile, EdgeStatus
+
+log = structlog.stdlib.get_logger()
+
+
+class NetworkGraph:
+    """Graph functions for this node's view of its network."""
+    
+    cache: Cache
+    identity: NodeIdentity
+    dg: nx.DiGraph
+    
+    def __init__(self, cache: Cache, identity: NodeIdentity):
+        self.cache = cache
+        self.dg = nx.DiGraph()
+        self.identity = identity
+    
+    def start(self):
+        self.generate()
+        
+    def generate(self):
+        """Generates directed graph from cached KOI nodes and edges."""
+        log.debug("Generating network graph")
+        self.dg.clear()
+        for rid in self.cache.list_rids():
+            if type(rid) == KoiNetNode:                
+                self.dg.add_node(rid)
+                log.debug(f"Added node {rid!r}")
+                
+            elif type(rid) == KoiNetEdge:
+                edge_bundle = self.cache.read(rid)
+                if not edge_bundle:
+                    log.warning(f"Failed to load {rid!r}")
+                    continue
+                edge_profile = edge_bundle.validate_contents(EdgeProfile)
+                self.dg.add_edge(edge_profile.source, edge_profile.target, rid=rid)
+                log.debug(f"Added edge {rid!r} ({edge_profile.source} -> {edge_profile.target})")
+        log.debug("Done")
+        
+    def get_edge(
+        self, 
+        source: KoiNetNode, 
+        target: KoiNetNode
+    ) -> KoiNetEdge | None:
+        """Returns edge RID given the RIDs of a source and target node."""
+        if (source, target) in self.dg.edges:
+            edge_data = self.dg.get_edge_data(source, target)
+            if edge_data:
+                return edge_data.get("rid")
+
+        return None
+
+    def get_edges(
+        self,
+        direction: Literal["in", "out"] | None = None,
+    ) -> list[KoiNetEdge]:
+        """Returns edges this node belongs to.
+        
+        All edges returned by default, specify `direction` to restrict 
+        to incoming or outgoing edges only.
+        """
+        
+        edges = []
+        if (direction is None or direction == "out") and self.dg.out_edges:
+            out_edges = self.dg.out_edges(self.identity.rid)
+            edges.extend(out_edges)
+        
+        if (direction is None or direction == "in") and self.dg.in_edges:
+            in_edges = self.dg.in_edges(self.identity.rid)
+            edges.extend(in_edges)
+        
+        edge_rids = []
+        for edge in edges:
+            edge_data = self.dg.get_edge_data(*edge)
+            if not edge_data: continue
+            edge_rid = edge_data.get("rid")
+            if not edge_rid: continue
+            edge_rids.append(edge_rid)
+       
+        return edge_rids
+    
+    def get_neighbors(
+        self,
+        direction: Literal["in", "out"] | None = None,
+        status: EdgeStatus | None = None,
+        allowed_type: RIDType | None = None
+    ) -> list[KoiNetNode]:
+        """Returns neighboring nodes this node shares an edge with.
+        
+        All neighboring nodes returned by default, specify `direction` 
+        to restrict to neighbors connected by incoming or outgoing edges
+        only.
+        """
+        
+        neighbors = set()
+        for edge_rid in self.get_edges(direction):
+            edge_bundle = self.cache.read(edge_rid)
+            
+            if not edge_bundle: 
+                log.warning(f"Failed to find edge {edge_rid!r} in cache")
+                continue
+            
+            edge_profile = edge_bundle.validate_contents(EdgeProfile)
+                        
+            if status and edge_profile.status != status:
+                continue
+            
+            if allowed_type and allowed_type not in edge_profile.rid_types:
+                continue
+            
+            if edge_profile.target == self.identity.rid:
+                neighbors.add(edge_profile.source)
+            elif edge_profile.source == self.identity.rid:
+                neighbors.add(edge_profile.target)
+            
+        return list(neighbors)