koi-net 1.0.0__py3-none-any.whl

koi_net/__init__.py

@@ -0,0 +1 @@
+from .core import NodeInterface

koi_net/config.py

@@ -0,0 +1,101 @@
+import os
+from typing import TypeVar
+from ruamel.yaml import YAML
+from koi_net.protocol.node import NodeProfile
+from rid_lib.types import KoiNetNode
+from pydantic import BaseModel, Field, PrivateAttr
+from dotenv import load_dotenv
+
+
+class ServerConfig(BaseModel):
+    host: str | None = "127.0.0.1"
+    port: int | None = 8000
+    path: str | None = "/koi-net"
+    
+    @property
+    def url(self) -> str:
+        return f"http://{self.host}:{self.port}{self.path or ''}"
+
+class KoiNetConfig(BaseModel):
+    node_name: str
+    node_rid: KoiNetNode | None = None
+    node_profile: NodeProfile
+    
+    cache_directory_path: str | None = ".rid_cache"
+    event_queues_path: str | None = "event_queues.json"
+
+    first_contact: str | None = None
+
+class EnvConfig(BaseModel):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        load_dotenv()
+    
+    def __getattribute__(self, name):
+        value = super().__getattribute__(name)
+        if name in type(self).model_fields:
+            env_val = os.getenv(value)
+            if env_val is None:
+                raise ValueError(f"Required environment variable {value} not set")
+            return env_val
+        return value
+
+class NodeConfig(BaseModel):
+    server: ServerConfig | None = Field(default_factory=ServerConfig)
+    koi_net: KoiNetConfig
+    _file_path: str = PrivateAttr(default="config.yaml")
+    _file_content: str | None = PrivateAttr(default=None)
+    
+    @classmethod
+    def load_from_yaml(
+        cls, 
+        file_path: str = "config.yaml", 
+        generate_missing: bool = True
+    ):
+        yaml = YAML()
+        
+        try:
+            with open(file_path, "r") as f:
+                file_content = f.read()
+            config_data = yaml.load(file_content)
+            config = cls.model_validate(config_data)
+            config._file_content = file_content
+            
+        except FileNotFoundError:
+            # empty_fields = {}
+            # for name, field in cls.model_fields.items():
+                
+            #     if field.default is None or field.default_factory is None:
+            # print(empty_fields)
+            config = cls()
+            
+            
+        config._file_path = file_path
+        
+        if generate_missing:            
+            config.koi_net.node_rid = (
+                config.koi_net.node_rid or KoiNetNode.generate(config.koi_net.node_name)
+            )   
+            config.koi_net.node_profile.base_url = (
+                config.koi_net.node_profile.base_url or config.server.url
+            )
+                
+            config.save_to_yaml()
+                    
+        return config
+    
+    def save_to_yaml(self):
+        yaml = YAML()
+        
+        with open(self._file_path, "w") as f:
+            try:
+                config_data = self.model_dump(mode="json")
+                yaml.dump(config_data, f)
+            except Exception as e:
+                if self._file_content:
+                    f.seek(0)
+                    f.truncate()
+                    f.write(self._file_content)
+                raise e
+                
+ConfigType = TypeVar("ConfigType", bound=NodeConfig)

koi_net/core.py

@@ -0,0 +1,126 @@
+import logging
+from typing import Generic
+import httpx
+from rid_lib.ext import Cache, Bundle
+from .network import NetworkInterface
+from .processor import ProcessorInterface
+from .processor import default_handlers
+from .processor.handler import KnowledgeHandler
+from .identity import NodeIdentity
+from .protocol.event import Event, EventType
+from .config import ConfigType
+
+logger = logging.getLogger(__name__)
+
+
+
+class NodeInterface(Generic[ConfigType]):
+    config: ConfigType
+    cache: Cache
+    identity: NodeIdentity
+    network: NetworkInterface
+    processor: ProcessorInterface
+    
+    use_kobj_processor_thread: bool
+    
+    def __init__(
+        self, 
+        config: ConfigType,
+        use_kobj_processor_thread: bool = False,
+        
+        handlers: list[KnowledgeHandler] | None = None,
+        
+        cache: Cache | None = None,
+        network: NetworkInterface | None = None,
+        processor: ProcessorInterface | None = None
+    ):
+        self.config: ConfigType = config
+        self.cache = cache or Cache(
+            self.config.koi_net.cache_directory_path)
+        
+        self.identity = NodeIdentity(
+            config=self.config,
+            cache=self.cache)
+        
+        self.network = network or NetworkInterface(
+            config=self.config,
+            cache=self.cache, 
+            identity=self.identity
+        )
+        
+        # pull all handlers defined in default_handlers module
+        if handlers is None:
+            handlers = [
+                obj for obj in vars(default_handlers).values() 
+                if isinstance(obj, KnowledgeHandler)
+            ]
+
+        self.use_kobj_processor_thread = use_kobj_processor_thread
+        self.processor = processor or ProcessorInterface(
+            config=self.config,
+            cache=self.cache, 
+            network=self.network, 
+            identity=self.identity, 
+            use_kobj_processor_thread=self.use_kobj_processor_thread,
+            default_handlers=handlers
+        )
+            
+    def start(self) -> None:
+        """Starts a node, call this method first.
+        
+        Starts the processor thread (if enabled). Loads event queues into memory. Generates network graph from nodes and edges in cache. Processes any state changes of node bundle. Initiates handshake with first contact (if provided) if node doesn't have any neighbors.
+        """
+        if self.use_kobj_processor_thread:
+            logger.info("Starting processor worker thread")
+            self.processor.worker_thread.start()
+        
+        self.network._load_event_queues()
+        self.network.graph.generate()
+        
+        self.processor.handle(
+            bundle=Bundle.generate(
+                rid=self.identity.rid, 
+                contents=self.identity.profile.model_dump()
+            )
+        )
+        
+        logger.debug("Waiting for kobj queue to empty")
+        if self.use_kobj_processor_thread:
+            self.processor.kobj_queue.join()
+        else:
+            self.processor.flush_kobj_queue()
+        logger.debug("Done")
+    
+        if not self.network.graph.get_neighbors() and self.config.koi_net.first_contact:
+            logger.debug(f"I don't have any neighbors, reaching out to first contact {self.config.koi_net.first_contact}")
+            
+            events = [
+                Event.from_rid(EventType.FORGET, self.identity.rid),
+                Event.from_bundle(EventType.NEW, self.identity.bundle)
+            ]
+            
+            try:
+                self.network.request_handler.broadcast_events(
+                    url=self.config.koi_net.first_contact,
+                    events=events
+                )
+                
+            except httpx.ConnectError:
+                logger.warning("Failed to reach first contact")
+                return
+            
+                        
+    def stop(self):
+        """Stops a node, call this method last.
+        
+        Finishes processing knowledge object queue. Saves event queues to storage.
+        """
+        logger.info("Stopping node...")
+        
+        if self.use_kobj_processor_thread:
+            logger.info(f"Waiting for kobj queue to empty ({self.processor.kobj_queue.unfinished_tasks} tasks remaining)")
+            self.processor.kobj_queue.join()
+        else:
+            self.processor.flush_kobj_queue()
+        
+        self.network._save_event_queues()

koi_net/identity.py

@@ -0,0 +1,42 @@
+import logging
+from rid_lib.ext.bundle import Bundle
+from rid_lib.ext.cache import Cache
+from rid_lib.types.koi_net_node import KoiNetNode
+
+from .config import NodeConfig
+from .protocol.node import NodeProfile
+
+logger = logging.getLogger(__name__)
+
+    
+class NodeIdentity:
+    """Represents a node's identity (RID, profile, bundle)."""
+    
+    config: NodeConfig    
+    cache: Cache
+    
+    def __init__(
+        self,
+        config: NodeConfig,
+        cache: Cache
+    ):
+        """Initializes node identity from a name and profile.
+        
+        Attempts to read identity from storage. If it doesn't already exist, a new RID is generated from the provided name, and that RID and profile are written to storage. Changes to the name or profile will update the stored identity.
+        
+        WARNING: If the name is changed, the RID will be overwritten which will have consequences for the rest of the network.
+        """
+        self.config = config
+        self.cache = cache
+        
+    @property
+    def rid(self) -> KoiNetNode:
+        return self.config.koi_net.node_rid
+    
+    @property
+    def profile(self) -> NodeProfile:
+        return self.config.koi_net.node_profile 
+    
+    @property
+    def bundle(self) -> Bundle:
+        return self.cache.read(self.rid)

koi_net/network/__init__.py

@@ -0,0 +1 @@
+from .interface import NetworkInterface

koi_net/network/graph.py

@@ -0,0 +1,127 @@
+import logging
+from typing import Literal
+import networkx as nx
+from rid_lib import RIDType
+from rid_lib.ext import Cache
+from rid_lib.types import KoiNetEdge, KoiNetNode
+from ..identity import NodeIdentity
+from ..protocol.edge import EdgeProfile, EdgeStatus
+from ..protocol.node import NodeProfile
+
+logger = logging.getLogger(__name__)
+
+
+class NetworkGraph:
+    """Graph functions for this node's view of its network."""
+    
+    cache: Cache
+    identity: NodeIdentity
+    dg: nx.DiGraph
+    
+    def __init__(self, cache: Cache, identity: NodeIdentity):
+        self.cache = cache
+        self.dg = nx.DiGraph()
+        self.identity = identity
+        
+    def generate(self):
+        """Generates directed graph from cached KOI nodes and edges."""
+        logger.debug("Generating network graph")
+        self.dg.clear()
+        for rid in self.cache.list_rids():
+            if type(rid) == KoiNetNode:                
+                self.dg.add_node(rid)
+                logger.debug(f"Added node {rid}")
+                
+            elif type(rid) == KoiNetEdge:
+                edge_profile = self.get_edge_profile(rid)
+                if not edge_profile:
+                    logger.warning(f"Failed to load {rid!r}")
+                    continue
+                self.dg.add_edge(edge_profile.source, edge_profile.target, rid=rid)
+                logger.debug(f"Added edge {rid} ({edge_profile.source} -> {edge_profile.target})")
+        logger.debug("Done")
+        
+    def get_node_profile(self, rid: KoiNetNode) -> NodeProfile | None:
+        """Returns node profile given its RID."""
+        bundle = self.cache.read(rid)
+        if bundle:
+            return bundle.validate_contents(NodeProfile)
+        
+    def get_edge_profile(
+        self, 
+        rid: KoiNetEdge | None = None,
+        source: KoiNetNode | None = None, 
+        target: KoiNetNode | None = None,
+    ) -> EdgeProfile | None:
+        """Returns edge profile given its RID, or source and target node RIDs."""
+        if source and target:
+            if (source, target) not in self.dg.edges: return
+            edge_data = self.dg.get_edge_data(source, target)
+            if not edge_data: return
+            rid = edge_data.get("rid")
+            if not rid: return
+        elif not rid:
+            raise ValueError("Either 'rid' or 'source' and 'target' must be provided")
+        
+        bundle = self.cache.read(rid)
+        if bundle:
+            return bundle.validate_contents(EdgeProfile)
+        
+    def get_edges(
+        self,
+        direction: Literal["in", "out"] | None = None,
+    ) -> list[KoiNetEdge]:
+        """Returns edges this node belongs to.
+        
+        All edges returned by default, specify `direction` to restrict to incoming or outgoing edges only."""
+                
+        edges = []
+        if direction != "in" and self.dg.out_edges:
+            out_edges = self.dg.out_edges(self.identity.rid)
+            edges.extend([e for e in out_edges])
+                
+        if direction != "out" and self.dg.in_edges:
+            in_edges = self.dg.in_edges(self.identity.rid)
+            edges.extend([e for e in in_edges])
+                    
+        edge_rids = []
+        for edge in edges:
+            edge_data = self.dg.get_edge_data(*edge)
+            if not edge_data: continue
+            edge_rid = edge_data.get("rid")
+            if not edge_rid: continue
+            edge_rids.append(edge_rid)
+       
+        return edge_rids
+    
+    def get_neighbors(
+        self,
+        direction: Literal["in", "out"] | None = None,
+        status: EdgeStatus | None = None,
+        allowed_type: RIDType | None = None
+    ) -> list[KoiNetNode]:
+        """Returns neighboring nodes this node shares an edge with.
+        
+        All neighboring nodes returned by default, specify `direction` to restrict to neighbors connected by incoming or outgoing edges only."""
+        
+        neighbors = []
+        for edge_rid in self.get_edges(direction):
+            edge_profile = self.get_edge_profile(edge_rid)
+            
+            if not edge_profile: 
+                logger.warning(f"Failed to find edge {edge_rid!r} in cache")
+                continue
+                        
+            if status and edge_profile.status != status:
+                continue
+            
+            if allowed_type and allowed_type not in edge_profile.rid_types:
+                continue
+            
+            if edge_profile.target == self.identity.rid:
+                neighbors.append(edge_profile.source)
+            elif edge_profile.source == self.identity.rid:
+                neighbors.append(edge_profile.target)
+                
+        return list(neighbors)
+        

koi_net/network/interface.py

@@ -0,0 +1,276 @@
+import logging
+from queue import Queue
+from typing import Generic
+import httpx
+from pydantic import BaseModel
+from rid_lib import RID
+from rid_lib.core import RIDType
+from rid_lib.ext import Cache
+from rid_lib.types import KoiNetNode
+
+from .graph import NetworkGraph
+from .request_handler import RequestHandler
+from .response_handler import ResponseHandler
+from ..protocol.node import NodeType
+from ..protocol.edge import EdgeType
+from ..protocol.event import Event
+from ..identity import NodeIdentity
+from ..config import ConfigType
+
+logger = logging.getLogger(__name__)
+
+
+class EventQueueModel(BaseModel):
+    webhook: dict[KoiNetNode, list[Event]]
+    poll: dict[KoiNetNode, list[Event]]
+
+type EventQueue = dict[RID, Queue[Event]]
+
+class NetworkInterface(Generic[ConfigType]):
+    """A collection of functions and classes to interact with the KOI network."""
+    
+    config: ConfigType    
+    identity: NodeIdentity
+    cache: Cache
+    graph: NetworkGraph
+    request_handler: RequestHandler
+    response_handler: ResponseHandler
+    poll_event_queue: EventQueue
+    webhook_event_queue: EventQueue
+    
+    def __init__(
+        self, 
+        config: ConfigType,
+        cache: Cache, 
+        identity: NodeIdentity
+    ):
+        self.config = config
+        self.identity = identity
+        self.cache = cache
+        self.graph = NetworkGraph(cache, identity)
+        self.request_handler = RequestHandler(cache, self.graph)
+        self.response_handler = ResponseHandler(cache)
+        
+        self.poll_event_queue = dict()
+        self.webhook_event_queue = dict()
+        self._load_event_queues()
+    
+    def _load_event_queues(self):
+        """Loads event queues from storage."""
+        try:
+            with open(self.config.koi_net.event_queues_path, "r") as f:
+                queues = EventQueueModel.model_validate_json(f.read())
+            
+            for node in queues.poll.keys():
+                for event in queues.poll[node]:
+                    queue = self.poll_event_queue.setdefault(node, Queue())
+                    queue.put(event)
+            
+            for node in queues.webhook.keys():
+                for event in queues.webhook[node]:
+                    queue = self.webhook_event_queue.setdefault(node, Queue())
+                    queue.put(event)
+                                
+        except FileNotFoundError:
+            return
+        
+    def _save_event_queues(self):
+        """Writes event queues to storage."""
+        events_model = EventQueueModel(
+            poll={
+                node: list(queue.queue) 
+                for node, queue in self.poll_event_queue.items()
+                if not queue.empty()
+            },
+            webhook={
+                node: list(queue.queue) 
+                for node, queue in self.webhook_event_queue.items()
+                if not queue.empty()
+            }
+        )
+        
+        if len(events_model.poll) == 0 and len(events_model.webhook) == 0:
+            return
+        
+        with open(self.config.koi_net.event_queues_path, "w") as f:
+            f.write(events_model.model_dump_json(indent=2))
+    
+    def push_event_to(self, event: Event, node: KoiNetNode, flush=False):
+        """Pushes event to queue of specified node.
+        
+        Event will be sent to webhook or poll queue depending on the node type and edge type of the specified node. If `flush` is set to `True`, the webhook queued will be flushed after pushing the event.
+        """
+        logger.debug(f"Pushing event {event.event_type} {event.rid} to {node}")
+            
+        node_profile = self.graph.get_node_profile(node)
+        if not node_profile:
+            logger.warning(f"Node {node!r} unknown to me")
+        
+        # if there's an edge from me to the target node, override broadcast type
+        edge_profile = self.graph.get_edge_profile(
+            source=self.identity.rid,
+            target=node
+        )
+        
+        if edge_profile:
+            if edge_profile.edge_type == EdgeType.WEBHOOK:
+                event_queue = self.webhook_event_queue
+            elif edge_profile.edge_type == EdgeType.POLL:
+                event_queue = self.poll_event_queue
+        else:
+            if node_profile.node_type == NodeType.FULL:
+                event_queue = self.webhook_event_queue
+            elif node_profile.node_type == NodeType.PARTIAL:
+                event_queue = self.poll_event_queue
+        
+        queue = event_queue.setdefault(node, Queue())
+        queue.put(event)
+                
+        if flush and event_queue is self.webhook_event_queue:
+            self.flush_webhook_queue(node)
+            
+    def _flush_queue(self, event_queue: EventQueue, node: KoiNetNode) -> list[Event]:
+        """Flushes a node's queue, returning list of events."""
+        queue = event_queue.get(node)
+        events = list()
+        if queue:
+            while not queue.empty():
+                event = queue.get()
+                logger.debug(f"Dequeued {event.event_type} '{event.rid}'")
+                events.append(event)
+        
+        return events
+    
+    def flush_poll_queue(self, node: KoiNetNode) -> list[Event]:
+        """Flushes a node's poll queue, returning list of events."""
+        logger.debug(f"Flushing poll queue for {node}")
+        return self._flush_queue(self.poll_event_queue, node)
+    
+    def flush_webhook_queue(self, node: KoiNetNode):
+        """Flushes a node's webhook queue, and broadcasts events.
+        
+        If node profile is unknown, or node type is not `FULL`, this operation will fail silently. If the remote node cannot be reached, all events will be requeued.
+        """
+        
+        logger.debug(f"Flushing webhook queue for {node}")
+        
+        node_profile = self.graph.get_node_profile(node)
+        
+        if not node_profile:
+            logger.warning(f"{node!r} not found")
+            return
+        
+        if node_profile.node_type != NodeType.FULL:
+            logger.warning(f"{node!r} is a partial node!")
+            return
+        
+        events = self._flush_queue(self.webhook_event_queue, node)
+        if not events: return
+        
+        logger.debug(f"Broadcasting {len(events)} events")
+        
+        try:  
+            self.request_handler.broadcast_events(node, events=events)
+            return True
+        except httpx.ConnectError:
+            logger.warning("Broadcast failed")
+            for event in events:
+                self.push_event_to(event, node)
+            return False
+            
+    def get_state_providers(self, rid_type: RIDType) -> list[KoiNetNode]:
+        """Returns list of node RIDs which provide state for the specified RID type."""
+        
+        logger.debug(f"Looking for state providers of '{rid_type}'")
+        provider_nodes = []
+        for node_rid in self.cache.list_rids(rid_types=[KoiNetNode]):
+            node = self.graph.get_node_profile(node_rid)
+                        
+            if node.node_type == NodeType.FULL and rid_type in node.provides.state:
+                logger.debug(f"Found provider '{node_rid}'")
+                provider_nodes.append(node_rid)
+        
+        if not provider_nodes:
+            logger.debug("Failed to find providers")
+        return provider_nodes
+            
+    def fetch_remote_bundle(self, rid: RID):
+        """Attempts to fetch a bundle by RID from known peer nodes."""
+        
+        logger.debug(f"Fetching remote bundle '{rid}'")
+        remote_bundle = None
+        for node_rid in self.get_state_providers(type(rid)):
+            payload = self.request_handler.fetch_bundles(
+                node=node_rid, rids=[rid])
+            
+            if payload.bundles:
+                remote_bundle = payload.bundles[0]
+                logger.debug(f"Got bundle from '{node_rid}'")
+                break
+        
+        if not remote_bundle:
+            logger.warning("Failed to fetch remote bundle")
+            
+        return remote_bundle
+    
+    def fetch_remote_manifest(self, rid: RID):
+        """Attempts to fetch a manifest by RID from known peer nodes."""
+        
+        logger.debug(f"Fetching remote manifest '{rid}'")
+        remote_manifest = None
+        for node_rid in self.get_state_providers(type(rid)):
+            payload = self.request_handler.fetch_manifests(
+                node=node_rid, rids=[rid])
+            
+            if payload.manifests:
+                remote_manifest = payload.manifests[0]
+                logger.debug(f"Got bundle from '{node_rid}'")
+                break
+        
+        if not remote_manifest:
+            logger.warning("Failed to fetch remote bundle")
+            
+        return remote_manifest
+    
+    def poll_neighbors(self) -> list[Event]:
+        """Polls all neighboring nodes and returns compiled list of events.
+        
+        If this node has no neighbors, it will instead attempt to poll the provided first contact URL.
+        """
+        
+        neighbors = self.graph.get_neighbors()
+        
+        if not neighbors and self.config.koi_net.first_contact:
+            logger.debug("No neighbors found, polling first contact")
+            try:
+                payload = self.request_handler.poll_events(
+                    url=self.config.koi_net.first_contact, 
+                    rid=self.identity.rid
+                )
+                if payload.events:
+                    logger.debug(f"Received {len(payload.events)} events from '{self.config.koi_net.first_contact}'")
+                return payload.events
+            except httpx.ConnectError:
+                logger.debug(f"Failed to reach first contact '{self.config.koi_net.first_contact}'")
+        
+        events = []
+        for node_rid in neighbors:
+            node = self.graph.get_node_profile(node_rid)
+            if not node: continue
+            if node.node_type != NodeType.FULL: continue
+            
+            try:
+                payload = self.request_handler.poll_events(
+                    node=node_rid, 
+                    rid=self.identity.rid
+                )
+                if payload.events:
+                    logger.debug(f"Received {len(payload.events)} events from {node_rid!r}")
+                events.extend(payload.events)
+            except httpx.ConnectError:
+                logger.debug(f"Failed to reach node '{node_rid}'")
+                continue
+            
+        return events                
+        
+