koi-net 1.0.0b1__py3-none-any.whl

koi_net/__init__.py

@@ -0,0 +1 @@
+from .core import NodeInterface

koi_net/core.py

@@ -0,0 +1,86 @@
+import logging
+import httpx
+from rid_lib.ext import Cache, Bundle
+from .network import NetworkInterface
+from .processor import ProcessorInterface
+from .processor import default_handlers as _default_handlers
+from .processor.handler import KnowledgeHandler
+from .identity import NodeIdentity
+from .protocol.node import NodeProfile
+from .protocol.event import Event, EventType
+
+logger = logging.getLogger(__name__)
+
+class NodeInterface:
+    def __init__(
+        self, 
+        name: str,
+        profile: NodeProfile,
+        identity_file_path: str = "identity.json",
+        first_contact: str | None = None,
+        default_handlers: list[KnowledgeHandler] | None = None,
+        cache: Cache | None = None,
+        network: NetworkInterface | None = None,
+        processor: ProcessorInterface | None = None
+    ):
+        self.cache = cache or Cache(directory_path=f"{name}_cache")
+        self.identity = NodeIdentity(
+            name=name,
+            profile=profile,
+            cache=self.cache,
+            file_path=identity_file_path
+        )
+        self.first_contact = first_contact
+        self.network = network or NetworkInterface(
+            file_path=f"{self.identity.rid.name}_event_queues.json", 
+            first_contact=self.first_contact,
+            cache=self.cache, 
+            identity=self.identity
+        )
+        
+        # pull all handlers defined in default_handlers module
+        if not default_handlers:
+            default_handlers = [
+                obj for obj in vars(_default_handlers).values() 
+                if isinstance(obj, KnowledgeHandler)
+            ]
+        
+        self.processor = processor or ProcessorInterface(
+            cache=self.cache, 
+            network=self.network, 
+            identity=self.identity, 
+            default_handlers=default_handlers
+        )
+    
+    def initialize(self):
+        self.network.graph.generate()
+        
+        self.processor.handle(
+            bundle=Bundle.generate(
+                rid=self.identity.rid, 
+                contents=self.identity.profile.model_dump()
+            ),
+            flush=True
+        )
+    
+        if not self.network.graph.get_neighbors() and self.first_contact:
+            logger.info(f"I don't have any neighbors, reaching out to first contact {self.first_contact}")
+            
+            events = [
+                Event.from_rid(EventType.FORGET, self.identity.rid),
+                Event.from_bundle(EventType.NEW, self.identity.bundle)
+            ]
+            
+            try:
+                self.network.request_handler.broadcast_events(
+                    url=self.first_contact,
+                    events=events
+                )
+                
+            except httpx.ConnectError:
+                logger.info("Failed to reach first contact")
+                return
+            
+                        
+    def finalize(self):
+        self.network.save_event_queues()

koi_net/identity.py

@@ -0,0 +1,62 @@
+import logging
+from pydantic import BaseModel
+from rid_lib.ext.bundle import Bundle
+from rid_lib.ext.cache import Cache
+from rid_lib.types.koi_net_node import KoiNetNode
+from .protocol.node import NodeProfile
+
+logger = logging.getLogger(__name__)
+
+
+class NodeIdentityModel(BaseModel):
+    rid: KoiNetNode
+    profile: NodeProfile
+    
+class NodeIdentity:
+    _identity: NodeIdentityModel
+    file_path: str
+    cache: Cache
+    
+    def __init__(
+        self,  
+        name: str,
+        profile: NodeProfile,
+        cache: Cache,
+        file_path: str = "identity.json"
+    ):
+        self.cache = cache
+        self.file_path = file_path
+        
+        self._identity = None
+        try:
+            with open(file_path, "r") as f:
+                self._identity = NodeIdentityModel.model_validate_json(f.read())
+                    
+        except FileNotFoundError:
+            pass
+        
+        if self._identity:
+            if self._identity.rid.name != name:
+                logger.warning("Node name changed which will change this node's RID, if you really want to do this manually delete the identity JSON file")
+            if self._identity.profile != profile:
+                self._identity.profile = profile
+        else:
+            self._identity = NodeIdentityModel( 
+                rid=KoiNetNode.generate(name), 
+                profile=profile,
+            )
+                    
+        with open(file_path, "w") as f:
+            f.write(self._identity.model_dump_json(indent=2))
+        
+    @property
+    def rid(self) -> KoiNetNode:
+        return self._identity.rid
+    
+    @property
+    def profile(self) -> NodeProfile:
+        return self._identity.profile    
+    
+    @property
+    def bundle(self) -> Bundle:
+        return self.cache.read(self.rid)

koi_net/network/__init__.py

@@ -0,0 +1 @@
+from .interface import NetworkInterface

koi_net/network/graph.py

@@ -0,0 +1,112 @@
+import logging
+from typing import Literal
+import networkx as nx
+from rid_lib import RIDType
+from rid_lib.ext import Cache
+from rid_lib.types import KoiNetEdge, KoiNetNode
+from ..identity import NodeIdentity
+from ..protocol.edge import EdgeProfile, EdgeStatus
+from ..protocol.node import NodeProfile
+
+logger = logging.getLogger(__name__)
+
+
+class NetworkGraph:
+    def __init__(self, cache: Cache, identity: NodeIdentity):
+        self.cache = cache
+        self.dg = nx.DiGraph()
+        self.identity = identity
+        
+    def generate(self):
+        logger.info("Generating network graph")
+        self.dg.clear()
+        for rid in self.cache.list_rids():
+            if type(rid) == KoiNetNode:                
+                self.dg.add_node(rid)
+                logger.info(f"Added node {rid}")
+                
+            elif type(rid) == KoiNetEdge:
+                edge_profile = self.get_edge_profile(rid)
+                if not edge_profile:
+                    logger.warning(f"Failed to load {rid!r}")
+                    continue
+                self.dg.add_edge(edge_profile.source, edge_profile.target, rid=rid)
+                logger.info(f"Added edge {rid} ({edge_profile.source} -> {edge_profile.target})")
+        logger.info("Done")
+        
+    def get_node_profile(self, rid: KoiNetNode) -> NodeProfile | None:
+        bundle = self.cache.read(rid)
+        if bundle:
+            return bundle.validate_contents(NodeProfile)
+        
+    def get_edge_profile(
+        self, 
+        rid: KoiNetEdge | None = None,
+        source: KoiNetNode | None = None, 
+        target: KoiNetNode | None = None,
+    ) -> EdgeProfile | None:
+        if source and target:
+            if (source, target) not in self.dg.edges: return
+            edge_data = self.dg.get_edge_data(source, target)
+            if not edge_data: return
+            rid = edge_data.get("rid")
+            if not rid: return
+        elif not rid:
+            raise ValueError("Either 'rid' or 'source' and 'target' must be provided")
+        
+        bundle = self.cache.read(rid)
+        if bundle:
+            return bundle.validate_contents(EdgeProfile)
+        
+    def get_edges(
+        self,
+        direction: Literal["in", "out"] | None = None,
+    ) -> list[KoiNetEdge]:
+        
+        edges = []
+        if direction != "in":
+            out_edges = self.dg.out_edges(self.identity.rid)
+            edges.extend([e for e in out_edges])
+                
+        if direction != "out":
+            in_edges = self.dg.in_edges(self.identity.rid)
+            edges.extend([e for e in in_edges])
+                    
+        edge_rids = []
+        for edge in edges:
+            edge_data = self.dg.get_edge_data(*edge)
+            if not edge_data: continue
+            edge_rid = edge_data.get("rid")
+            if not edge_rid: continue
+            edge_rids.append(edge_rid)
+       
+        return edge_rids
+    
+    def get_neighbors(
+        self,
+        direction: Literal["in", "out"] | None = None,
+        status: EdgeStatus | None = None,
+        allowed_type: RIDType | None = None
+    ) -> list[KoiNetNode]:
+        
+        neighbors = []
+        for edge_rid in self.get_edges(direction):
+            edge_profile = self.get_edge_profile(edge_rid)
+            
+            if not edge_profile: 
+                logger.warning(f"Failed to find edge {edge_rid!r} in cache")
+                continue
+                        
+            if status and edge_profile.status != status:
+                continue
+            
+            if allowed_type and allowed_type not in edge_profile.rid_types:
+                continue
+            
+            if edge_profile.target == self.identity.rid:
+                neighbors.append(edge_profile.source)
+            elif edge_profile.source == self.identity.rid:
+                neighbors.append(edge_profile.target)
+                
+        return list(neighbors)
+        

koi_net/network/interface.py

@@ -0,0 +1,249 @@
+import logging
+from queue import Queue
+import httpx
+from pydantic import BaseModel
+from rid_lib import RID
+from rid_lib.core import RIDType
+from rid_lib.ext import Cache
+from rid_lib.types import KoiNetNode
+from .graph import NetworkGraph
+from .request_handler import RequestHandler
+from .response_handler import ResponseHandler
+from ..protocol.node import NodeType
+from ..protocol.edge import EdgeType
+from ..protocol.event import Event
+from ..identity import NodeIdentity
+
+logger = logging.getLogger(__name__)
+
+
+class EventQueueModel(BaseModel):
+    webhook: dict[KoiNetNode, list[Event]]
+    poll: dict[KoiNetNode, list[Event]]
+
+type EventQueue = dict[RID, Queue[Event]]
+
+class NetworkInterface:
+    identity: NodeIdentity
+    cache: Cache
+    first_contact: str | None
+    graph: NetworkGraph
+    request_handler: RequestHandler
+    response_handler: ResponseHandler
+    poll_event_queue: EventQueue
+    webhook_event_queue: EventQueue
+    event_queues_file_path: str
+    
+    def __init__(
+        self, 
+        file_path: str,
+        first_contact: str | None,
+        cache: Cache, 
+        identity: NodeIdentity
+    ):
+        self.identity = identity
+        self.cache = cache
+        self.first_contact = first_contact
+        self.graph = NetworkGraph(cache, identity)
+        self.request_handler = RequestHandler(cache)
+        self.response_handler = ResponseHandler(cache)
+        self.event_queues_file_path = file_path
+        
+        self.poll_event_queue = dict()
+        self.webhook_event_queue = dict()
+        self.load_event_queues()
+    
+    def load_event_queues(self):
+        try:
+            with open(self.event_queues_file_path, "r") as f:
+                queues = EventQueueModel.model_validate_json(f.read())
+            
+            for node in queues.poll.keys():
+                for event in queues.poll[node]:
+                    queue = self.poll_event_queue.setdefault(node, Queue())
+                    queue.put(event)
+            
+            for node in queues.webhook.keys():
+                for event in queues.webhook[node]:
+                    queue = self.webhook_event_queue.setdefault(node, Queue())
+                    queue.put(event)
+                                
+        except FileNotFoundError:
+            return
+        
+    def save_event_queues(self):
+        events_model = EventQueueModel(
+            poll={
+                node: list(queue.queue) 
+                for node, queue in self.poll_event_queue.items()
+                if not queue.empty()
+            },
+            webhook={
+                node: list(queue.queue) 
+                for node, queue in self.webhook_event_queue.items()
+                if not queue.empty()
+            }
+        )
+        
+        if len(events_model.poll) == 0 and len(events_model.webhook) == 0:
+            return
+        
+        with open(self.event_queues_file_path, "w") as f:
+            f.write(events_model.model_dump_json(indent=2))
+    
+    def push_event_to(self, event: Event, node: KoiNetNode, flush=False):
+        logger.info(f"Pushing event {event.event_type} {event.rid} to {node}")
+            
+        node_profile = self.graph.get_node_profile(node)
+        if not node_profile:
+            logger.warning(f"Node {node!r} unknown to me")
+        
+        # if there's an edge from me to the target node, override broadcast type
+        edge_profile = self.graph.get_edge_profile(
+            source=self.identity.rid,
+            target=node
+        )
+        
+        if edge_profile:
+            if edge_profile.edge_type == EdgeType.WEBHOOK:
+                event_queue = self.webhook_event_queue
+            elif edge_profile.edge_type == EdgeType.POLL:
+                event_queue = self.poll_event_queue
+        else:
+            if node_profile.node_type == NodeType.FULL:
+                event_queue = self.webhook_event_queue
+            elif node_profile.node_type == NodeType.PARTIAL:
+                event_queue = self.poll_event_queue
+        
+        queue = event_queue.setdefault(node, Queue())
+        queue.put(event)
+                
+        if flush and event_queue is self.webhook_event_queue:
+            self.flush_webhook_queue(node)
+            
+    def flush_queue(self, event_queue: EventQueue, node: KoiNetNode) -> list[Event]:
+        queue = event_queue.get(node)
+        events = list()
+        if queue:
+            while not queue.empty():
+                event = queue.get()
+                logger.info(f"Dequeued {event.event_type} '{event.rid}'")
+                events.append(event)
+        
+        return events
+    
+    def flush_poll_queue(self, node: KoiNetNode) -> list[Event]:
+        logger.info(f"Flushing poll queue for {node}")
+        return self.flush_queue(self.poll_event_queue, node)
+    
+    def flush_webhook_queue(self, node: RID):
+        logger.info(f"Flushing webhook queue for {node}")
+        
+        node_profile = self.graph.get_node_profile(node)
+        
+        if not node_profile:
+            logger.warning(f"{node!r} not found")
+        
+        if node_profile.node_type != NodeType.FULL:
+            logger.warning(f"{node!r} is a partial node!")
+            return
+        
+        events = self.flush_queue(self.webhook_event_queue, node)
+        logger.info(f"Broadcasting {len(events)} events")
+        
+        try:  
+            self.request_handler.broadcast_events(node, events=events)
+        except httpx.ConnectError:
+            logger.warning("Broadcast failed, requeuing events")
+            for event in events:
+                self.push_event_to(event, node)
+    
+    def flush_all_webhook_queues(self):
+        for node in self.webhook_event_queue.keys():
+            self.flush_webhook_queue(node)
+            
+    def get_state_providers(self, rid_type: RIDType):
+        logger.info(f"Looking for state providers of '{rid_type}'")
+        provider_nodes = []
+        for node_rid in self.cache.list_rids(rid_types=[KoiNetNode]):
+            node = self.graph.get_node_profile(node_rid)
+                        
+            if node.node_type == NodeType.FULL and rid_type in node.provides.state:
+                logger.info(f"Found provider '{node_rid}'")
+                provider_nodes.append(node_rid)
+        
+        if not provider_nodes:
+            logger.info("Failed to find providers")
+        return provider_nodes
+            
+    def fetch_remote_bundle(self, rid: RID):
+        logger.info(f"Fetching remote bundle '{rid}'")
+        remote_bundle = None
+        for node_rid in self.get_state_providers(type(rid)):
+            payload = self.request_handler.fetch_bundles(
+                node=node_rid, rids=[rid])
+            
+            if payload.manifests:
+                remote_bundle = payload.manifests[0]
+                logger.info(f"Got bundle from '{node_rid}'")
+                break
+        
+        if not remote_bundle:
+            logger.warning("Failed to fetch remote bundle")
+            
+        return remote_bundle
+    
+    def fetch_remote_manifest(self, rid: RID):
+        logger.info(f"Fetching remote manifest '{rid}'")
+        remote_manifest = None
+        for node_rid in self.get_state_providers(type(rid)):
+            payload = self.request_handler.fetch_manifests(
+                node=node_rid, rids=[rid])
+            
+            if payload.manifests:
+                remote_manifest = payload.manifests[0]
+                logger.info(f"Got bundle from '{node_rid}'")
+                break
+        
+        if not remote_manifest:
+            logger.warning("Failed to fetch remote bundle")
+            
+        return remote_manifest
+    
+    def poll_neighbors(self) -> list[Event]:
+        neighbors = self.graph.get_neighbors()
+        
+        if not neighbors:
+            logger.info("No neighbors found, polling first contact")
+            try:
+                payload = self.request_handler.poll_events(
+                    url=self.first_contact, 
+                    rid=self.identity.rid
+                )
+                if payload.events:
+                    logger.info(f"Received {len(payload.events)} events from '{self.first_contact}'")
+                return payload.events
+            except httpx.ConnectError:
+                logger.info(f"Failed to reach first contact '{self.first_contact}'")
+        
+        events = []
+        for node_rid in neighbors:
+            node = self.graph.get_node_profile(node_rid)
+            if not node: continue
+            if node.node_type != NodeType.FULL: continue
+            
+            try:
+                payload = self.request_handler.poll_events(
+                    node=node_rid, 
+                    rid=self.identity.rid
+                )
+                if payload.events:
+                    logger.info(f"Received {len(payload.events)} events from {node_rid!r}")
+                events.extend(payload.events)
+            except httpx.ConnectError:
+                logger.info(f"Failed to reach node '{node_rid}'")
+                continue
+            
+        return events                
+        
+        

koi_net/network/request_handler.py

@@ -0,0 +1,105 @@
+import logging
+import httpx
+from pydantic import BaseModel
+from rid_lib import RID
+from rid_lib.ext import Cache
+from rid_lib.types.koi_net_node import KoiNetNode
+from ..protocol.api_models import (
+    RidsPayload,
+    ManifestsPayload,
+    BundlesPayload,
+    EventsPayload,
+    FetchRids,
+    FetchManifests,
+    FetchBundles,
+    PollEvents
+)
+from ..protocol.consts import (
+    BROADCAST_EVENTS_PATH,
+    POLL_EVENTS_PATH,
+    FETCH_RIDS_PATH,
+    FETCH_MANIFESTS_PATH,
+    FETCH_BUNDLES_PATH
+)
+from ..protocol.node import NodeProfile, NodeType
+
+
+logger = logging.getLogger(__name__)
+
+
+class RequestHandler:
+    cache: Cache
+    
+    def __init__(self, cache: Cache):
+        self.cache = cache
+                
+    def make_request(self, url, request: BaseModel) -> httpx.Response:
+        logger.info(f"Making request to {url}")
+        resp = httpx.post(
+            url=url,
+            data=request.model_dump_json()
+        )
+        return resp
+            
+    def get_url(self, node_rid: KoiNetNode, url: str) -> str:
+        if not node_rid and not url:
+            raise ValueError("One of 'node_rid' and 'url' must be provided")
+        
+        if node_rid:
+            # can't access get_node rn
+            bundle = self.cache.read(node_rid)
+            node = NodeProfile.model_validate(bundle.contents)
+            if node.node_type != NodeType.FULL:
+                raise Exception("Can't query partial node")
+            logger.info(f"Resolved {node_rid!r} to {node.base_url}")
+            return node.base_url
+        else:
+            return url
+    
+    def broadcast_events(
+        self, node: RID = None, url: str = None, **kwargs
+    ) -> None:
+        self.make_request(
+            self.get_url(node, url) + BROADCAST_EVENTS_PATH,
+            EventsPayload.model_validate(kwargs)
+        )
+        
+    def poll_events(
+        self, node: RID = None, url: str = None, **kwargs
+    ) -> EventsPayload:      
+        resp = self.make_request(
+            self.get_url(node, url) + POLL_EVENTS_PATH,
+            PollEvents.model_validate(kwargs)
+        )
+        
+        return EventsPayload.model_validate_json(resp.text)
+    
+    def fetch_rids(
+        self, node: RID = None, url: str = None, **kwargs
+    ) -> RidsPayload:      
+        resp = self.make_request(
+            self.get_url(node, url) + FETCH_RIDS_PATH,
+            FetchRids.model_validate(kwargs)
+        )
+        
+        return RidsPayload.model_validate_json(resp.text)
+        
+    def fetch_manifests(
+        self, node: RID = None, url: str = None, **kwargs
+    ) -> ManifestsPayload:    
+        resp = self.make_request(
+            self.get_url(node, url) + FETCH_MANIFESTS_PATH,
+            FetchManifests.model_validate(kwargs)
+        )
+        
+        return ManifestsPayload.model_validate_json(resp.text)
+        
+    def fetch_bundles(
+        self, node: RID = None, url: str = None, **kwargs
+    ) -> BundlesPayload:        
+        resp = self.make_request(
+            self.get_url(node, url) + FETCH_BUNDLES_PATH,
+            FetchBundles.model_validate(kwargs)
+        )
+        
+        return BundlesPayload.model_validate_json(resp.text)

koi_net/network/response_handler.py

@@ -0,0 +1,57 @@
+import logging
+from rid_lib import RID
+from rid_lib.ext import Manifest, Cache
+from rid_lib.ext.bundle import Bundle
+from ..protocol.api_models import (
+    RidsPayload,
+    ManifestsPayload,
+    BundlesPayload,
+    FetchRids,
+    FetchManifests,
+    FetchBundles,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ResponseHandler:
+    cache: Cache
+    
+    def __init__(self, cache: Cache):
+        self.cache = cache
+        
+    def fetch_rids(self, req: FetchRids) -> RidsPayload:
+        logger.info(f"Request to fetch rids, allowed types {req.rid_types}")
+        rids = self.cache.list_rids(req.rid_types)
+        
+        return RidsPayload(rids=rids)
+        
+    def fetch_manifests(self, req: FetchManifests) -> ManifestsPayload:
+        logger.info(f"Request to fetch manifests, allowed types {req.rid_types}, rids {req.rids}")
+        
+        manifests: list[Manifest] = []
+        not_found: list[RID] = []
+        
+        for rid in (req.rids or self.cache.list_rids(req.rid_types)):
+            bundle = self.cache.read(rid)
+            if bundle:
+                manifests.append(bundle.manifest)
+            else:
+                not_found.append(rid)
+        
+        return ManifestsPayload(manifests=manifests, not_found=not_found)
+        
+    def fetch_bundles(self, req: FetchBundles) -> BundlesPayload:
+        logger.info(f"Request to fetch bundles, requested rids {req.rids}")
+        
+        bundles: list[Bundle] = []
+        not_found: list[RID] = []
+
+        for rid in req.rids:
+            bundle = self.cache.read(rid)
+            if bundle:
+                bundles.append(bundle)
+            else:
+                not_found.append(rid)
+            
+        return BundlesPayload(manifests=bundles, not_found=not_found)

koi_net/processor/__init__.py

@@ -0,0 +1 @@
+from .interface import ProcessorInterface