koi-net 1.0.0b18__tar.gz → 1.1.0b1__tar.gz

{koi_net-1.0.0b18 → koi_net-1.1.0b1}/.gitignore

@@ -1,6 +1,8 @@
 rid-lib
 __pycache__
 *.json
+*.pem
+*.yaml
 venv
 .env
 prototypes

{koi_net-1.0.0b18 → koi_net-1.1.0b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: koi-net
-Version: 1.0.0b18
+Version: 1.1.0b1
 Summary: Implementation of KOI-net protocol in Python
 Project-URL: Homepage, https://github.com/BlockScience/koi-net/
 Author-email: Luke Miller <luke@block.science>
@@ -27,11 +27,12 @@ License: MIT License
         SOFTWARE.
 License-File: LICENSE
 Requires-Python: >=3.10
+Requires-Dist: cryptography>=45.0.3
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: networkx>=3.4.2
 Requires-Dist: pydantic>=2.10.6
 Requires-Dist: python-dotenv>=1.1.0
-Requires-Dist: rid-lib>=3.2.3
+Requires-Dist: rid-lib>=3.2.7
 Requires-Dist: ruamel-yaml>=0.18.10
 Provides-Extra: dev
 Requires-Dist: build; extra == 'dev'

{koi_net-1.0.0b18 → koi_net-1.1.0b1}/examples/basic_coordinator_node.py

@@ -8,13 +8,7 @@ from fastapi import FastAPI
 from rid_lib.types import KoiNetNode, KoiNetEdge
 from koi_net import NodeInterface
 from koi_net.config import NodeConfig, KoiNetConfig
-from koi_net.processor.handler import HandlerType
-from koi_net.processor.knowledge_object import KnowledgeObject, KnowledgeSource
-from koi_net.protocol.edge import EdgeType
-from koi_net.protocol.event import Event, EventType
-from koi_net.protocol.helpers import generate_edge_bundle
 from koi_net.protocol.node import NodeProfile, NodeType, NodeProvides
-from koi_net.processor import ProcessorInterface
 from koi_net.protocol.api_models import (
     PollEvents,
     FetchRids,
@@ -41,6 +35,8 @@ logging.basicConfig(
     handlers=[RichHandler()]
 )
 
+# TODO: UPDATE OR REMOVE, THIS EXAMPLE IS OUT OF DATE
+
 logging.getLogger("koi_net").setLevel(logging.DEBUG)
 
 
@@ -69,32 +65,6 @@ node = NodeInterface(
 logger = logging.getLogger(__name__)
 
 
-@node.processor.register_handler(HandlerType.Network, rid_types=[KoiNetNode])
-def handshake_handler(proc: ProcessorInterface, kobj: KnowledgeObject):    
-    logger.info("Handling node handshake")
-
-    # only respond if node declares itself as NEW
-    if kobj.event_type != EventType.NEW:
-        return
-        
-    logger.info("Sharing this node's bundle with peer")
-    proc.network.push_event_to(
-        event=Event.from_bundle(EventType.NEW, proc.identity.bundle),
-        node=kobj.rid,
-        flush=True
-    )
-    
-    logger.info("Proposing new edge")    
-    # defer handling of proposed edge
-    proc.handle(bundle=generate_edge_bundle(
-        source=kobj.rid,
-        target=proc.identity.rid,
-        edge_type=EdgeType.WEBHOOK,
-        rid_types=[KoiNetNode, KoiNetEdge]
-    ))
-
-
-
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     node.start()
@@ -117,20 +87,20 @@ def broadcast_events(req: EventsPayload):
 @app.post(POLL_EVENTS_PATH)
 def poll_events(req: PollEvents) -> EventsPayload:
     logger.info(f"Request to {POLL_EVENTS_PATH}")
-    events = node.network.flush_poll_queue(req.rid)
+    events = node.event_queue.flush_poll_queue(req.rid)
     return EventsPayload(events=events)
 
 @app.post(FETCH_RIDS_PATH)
 def fetch_rids(req: FetchRids) -> RidsPayload:
-    return node.network.response_handler.fetch_rids(req)
+    return node.response_handler.fetch_rids(req)
 
 @app.post(FETCH_MANIFESTS_PATH)
 def fetch_manifests(req: FetchManifests) -> ManifestsPayload:
-    return node.network.response_handler.fetch_manifests(req)
+    return node.response_handler.fetch_manifests(req)
 
 @app.post(FETCH_BUNDLES_PATH)
 def fetch_bundles(req: FetchBundles) -> BundlesPayload:
-    return node.network.response_handler.fetch_bundles(req)
+    return node.response_handler.fetch_bundles(req)
     
 if __name__ == "__main__":
     openapi_spec = app.openapi()

{koi_net-1.0.0b18 → koi_net-1.1.0b1}/examples/basic_partial_node.py

@@ -3,9 +3,8 @@ import logging
 from pydantic import Field
 from rich.logging import RichHandler
 from koi_net import NodeInterface
-from koi_net.processor.knowledge_object import KnowledgeSource
 from koi_net.protocol.node import NodeProfile, NodeType
-from koi_net.config import NodeConfig, KoiNetConfig
+from koi_net.config import NodeConfig, KoiNetConfig, NodeContact
 
 logging.basicConfig(
     level=logging.INFO,
@@ -18,29 +17,32 @@ logging.getLogger("koi_net").setLevel(logging.DEBUG)
 logger = logging.getLogger(__name__)
 
 
-class CoordinatorNodeConfig(NodeConfig):
+class PartialNodeConfig(NodeConfig):
     koi_net: KoiNetConfig | None = Field(default_factory = lambda:
         KoiNetConfig(
-            node_name="coordinator",
+            node_name="partial",
             node_profile=NodeProfile(
-                node_type=NodeType.FULL
+                node_type=NodeType.PARTIAL
             ),
             cache_directory_path=".basic_partial_rid_cache",
-            event_queues_path="basic_partial_event_queues.json",
-            first_contact="http://127.0.0.1:8000/koi-net"
+            event_queues_path="basic_partial_event_queues.json"
         )
     )
 
 
 node = NodeInterface(
-    config=CoordinatorNodeConfig.load_from_yaml("basical_partial_config.yaml")
+    config=PartialNodeConfig.load_from_yaml("basic_partial_config.yaml")
 )
 
+
 node.start()
 
 while True:
-    for event in node.network.poll_neighbors():
-        node.processor.handle(event=event, source=KnowledgeSource.External)
+    neighbors = node.resolver.poll_neighbors()
+    for node_rid in neighbors:
+        events = neighbors[node_rid]
+        for event in events:
+            node.processor.handle(event=event, source=node_rid)
     node.processor.flush_kobj_queue()
     
     time.sleep(5)

{koi_net-1.0.0b18 → koi_net-1.1.0b1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "koi-net"
-version = "1.0.0-beta.18"
+version = "1.1.0-beta.1"
 description = "Implementation of KOI-net protocol in Python"
 authors = [
     {name = "Luke Miller", email = "luke@block.science"}
@@ -13,12 +13,13 @@ readme = "README.md"
 requires-python = ">=3.10"
 license = {file = "LICENSE"}
 dependencies = [
-    "rid-lib>=3.2.3",
+    "rid-lib>=3.2.7",
     "networkx>=3.4.2",
     "httpx>=0.28.1",
     "pydantic>=2.10.6",
     "ruamel.yaml>=0.18.10",
-    "python-dotenv>=1.1.0"
+    "python-dotenv>=1.1.0",
+    "cryptography>=45.0.3"
 ]
 
 [project.optional-dependencies]

{koi_net-1.0.0b18 → koi_net-1.1.0b1}/src/koi_net/config.py

@@ -1,32 +1,42 @@
 import os
-from typing import TypeVar
 from ruamel.yaml import YAML
-from koi_net.protocol.node import NodeProfile
-from rid_lib.types import KoiNetNode
 from pydantic import BaseModel, Field, PrivateAttr
 from dotenv import load_dotenv
+from rid_lib.ext.utils import sha256_hash
+from rid_lib.types import KoiNetNode
+from .protocol.secure import PrivateKey
+from .protocol.node import NodeProfile, NodeType
 
 
 class ServerConfig(BaseModel):
-    host: str | None = "127.0.0.1"
-    port: int | None = 8000
+    host: str = "127.0.0.1"
+    port: int = 8000
     path: str | None = "/koi-net"
     
     @property
     def url(self) -> str:
         return f"http://{self.host}:{self.port}{self.path or ''}"
 
+class NodeContact(BaseModel):
+    rid: KoiNetNode | None = None
+    url: str | None = None
+
 class KoiNetConfig(BaseModel):
     node_name: str
     node_rid: KoiNetNode | None = None
     node_profile: NodeProfile
     
-    cache_directory_path: str | None = ".rid_cache"
-    event_queues_path: str | None = "event_queues.json"
-
-    first_contact: str | None = None
+    cache_directory_path: str = ".rid_cache"
+    event_queues_path: str = "event_queues.json"
+    private_key_pem_path: str = "priv_key.pem"
+    
+    first_contact: NodeContact = Field(default_factory=NodeContact)
+    
+    _priv_key: PrivateKey | None = PrivateAttr(default=None)
 
 class EnvConfig(BaseModel):
+    priv_key_password: str | None = "PRIV_KEY_PASSWORD"
+    
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         load_dotenv()
@@ -41,8 +51,10 @@ class EnvConfig(BaseModel):
         return value
 
 class NodeConfig(BaseModel):
-    server: ServerConfig | None = Field(default_factory=ServerConfig)
+    server: ServerConfig = Field(default_factory=ServerConfig)
     koi_net: KoiNetConfig
+    env: EnvConfig = Field(default_factory=EnvConfig)
+    
     _file_path: str = PrivateAttr(default="config.yaml")
     _file_content: str | None = PrivateAttr(default=None)
     
@@ -72,13 +84,27 @@ class NodeConfig(BaseModel):
             
         config._file_path = file_path
         
-        if generate_missing:            
-            config.koi_net.node_rid = (
-                config.koi_net.node_rid or KoiNetNode.generate(config.koi_net.node_name)
-            )   
-            config.koi_net.node_profile.base_url = (
-                config.koi_net.node_profile.base_url or config.server.url
-            )
+        if generate_missing:
+            if not config.koi_net.node_rid:
+                priv_key = PrivateKey.generate()
+                pub_key = priv_key.public_key()
+                
+                config.koi_net.node_rid = KoiNetNode(
+                    config.koi_net.node_name,
+                    sha256_hash(pub_key.to_der())
+                )
+                
+                with open(config.koi_net.private_key_pem_path, "w") as f:
+                    f.write(
+                        priv_key.to_pem(config.env.priv_key_password)
+                    )
+                
+                config.koi_net.node_profile.public_key = pub_key.to_der()
+            
+            if config.koi_net.node_profile.node_type == NodeType.FULL:
+                config.koi_net.node_profile.base_url = (
+                    config.koi_net.node_profile.base_url or config.server.url
+                )
                 
             config.save_to_yaml()
                     
@@ -98,4 +124,3 @@ class NodeConfig(BaseModel):
                     f.write(self._file_content)
                 raise e
                 
-ConfigType = TypeVar("ConfigType", bound=NodeConfig)

koi_net-1.1.0b1/src/koi_net/context.py

@@ -0,0 +1,55 @@
+
+from koi_net.effector import Effector
+from rid_lib.ext import Cache
+from .network.graph import NetworkGraph
+from .network.event_queue import NetworkEventQueue
+from .network.request_handler import RequestHandler
+from .identity import NodeIdentity
+from .processor.interface import ProcessorInterface
+
+
+class ActionContext:
+    identity: NodeIdentity
+    effector: Effector
+
+    def __init__(
+        self,
+        identity: NodeIdentity,
+        effector: Effector
+    ):
+        self.identity = identity
+        self.effector = effector
+    
+
+class HandlerContext:
+    identity: NodeIdentity
+    cache: Cache
+    event_queue: NetworkEventQueue
+    graph: NetworkGraph
+    request_handler: RequestHandler
+    effector: Effector
+    _processor: ProcessorInterface | None
+    
+    def __init__(
+        self,
+        identity: NodeIdentity,
+        cache: Cache,
+        event_queue: NetworkEventQueue,
+        graph: NetworkGraph,
+        request_handler: RequestHandler,
+        effector: Effector
+    ):
+        self.identity = identity
+        self.cache = cache
+        self.event_queue = event_queue
+        self.graph = graph
+        self.request_handler = request_handler
+        self.effector = effector
+        self._processor = None
+        
+    def set_processor(self, processor: ProcessorInterface):
+        self._processor = processor
+        
+    @property
+    def handle(self):
+        return self._processor.handle

koi_net-1.1.0b1/src/koi_net/core.py

@@ -0,0 +1,192 @@
+import logging
+import httpx
+from rid_lib.ext import Cache
+from .network.resolver import NetworkResolver
+from .network.event_queue import NetworkEventQueue
+from .network.graph import NetworkGraph
+from .network.request_handler import RequestHandler
+from .network.response_handler import ResponseHandler
+from .network.error_handler import ErrorHandler
+from .network.behavior import Actor
+from .processor.interface import ProcessorInterface
+from .processor import default_handlers
+from .processor.handler import KnowledgeHandler
+from .processor.knowledge_pipeline import KnowledgePipeline
+from .identity import NodeIdentity
+from .secure import Secure
+from .config import NodeConfig
+from .context import HandlerContext, ActionContext
+from .effector import Effector
+from . import default_actions
+
+logger = logging.getLogger(__name__)
+
+
+
+class NodeInterface:
+    config: NodeConfig
+    cache: Cache
+    identity: NodeIdentity
+    resolver: NetworkResolver
+    event_queue: NetworkEventQueue
+    graph: NetworkGraph
+    processor: ProcessorInterface
+    secure: Secure
+    
+    use_kobj_processor_thread: bool
+    
+    def __init__(
+        self, 
+        config: NodeConfig,
+        use_kobj_processor_thread: bool = False,
+        
+        handlers: list[KnowledgeHandler] | None = None,
+        
+        cache: Cache | None = None,
+        processor: ProcessorInterface | None = None
+    ):
+        self.config = config
+        self.cache = cache or Cache(
+            directory_path=self.config.koi_net.cache_directory_path
+        )
+        
+        self.identity = NodeIdentity(config=self.config)
+        
+        self.effector = Effector(cache=self.cache)
+
+        self.graph = NetworkGraph(
+            cache=self.cache, 
+            identity=self.identity
+        )
+        
+        self.secure = Secure(
+            identity=self.identity, 
+            effector=self.effector, 
+            config=self.config
+        )
+        
+        self.request_handler = RequestHandler(
+            effector=self.effector, 
+            identity=self.identity,
+            secure=self.secure
+        )
+        
+        self.response_handler = ResponseHandler(self.cache, self.effector)
+        
+        self.resolver = NetworkResolver(
+            config=self.config,
+            cache=self.cache, 
+            identity=self.identity,
+            graph=self.graph,
+            request_handler=self.request_handler,
+            effector=self.effector
+        )
+        
+        self.event_queue = NetworkEventQueue(
+            config=self.config,
+            cache=self.cache, 
+            identity=self.identity,
+            graph=self.graph,
+            request_handler=self.request_handler,
+            effector=self.effector
+        )
+        
+        self.actor = Actor(
+            identity=self.identity,
+            effector=self.effector,
+            event_queue=self.event_queue
+        )
+        
+        # pull all handlers defined in default_handlers module
+        if handlers is None:
+            handlers = [
+                obj for obj in vars(default_handlers).values() 
+                if isinstance(obj, KnowledgeHandler)
+            ]
+
+        self.use_kobj_processor_thread = use_kobj_processor_thread
+        
+        self.action_context = ActionContext(
+            identity=self.identity,
+            effector=self.effector
+        )
+        
+        self.handler_context = HandlerContext(
+            identity=self.identity,
+            cache=self.cache,
+            event_queue=self.event_queue,
+            graph=self.graph,
+            request_handler=self.request_handler,
+            effector=self.effector
+        )
+        
+        self.pipeline = KnowledgePipeline(
+            handler_context=self.handler_context,
+            cache=self.cache,
+            request_handler=self.request_handler,
+            event_queue=self.event_queue,
+            graph=self.graph,
+            default_handlers=handlers
+        )
+        
+        self.processor = processor or ProcessorInterface(
+            pipeline=self.pipeline,
+            use_kobj_processor_thread=self.use_kobj_processor_thread
+        )
+        
+        self.error_handler = ErrorHandler(
+            processor=self.processor,
+            actor=self.actor
+        )
+        
+        self.request_handler.set_error_handler(self.error_handler)
+        
+        self.handler_context.set_processor(self.processor)
+        
+        self.effector.set_processor(self.processor)
+        self.effector.set_resolver(self.resolver)
+        self.effector.set_action_context(self.action_context)
+         
+            
+    def start(self) -> None:
+        """Starts a node, call this method first.
+        
+        Starts the processor thread (if enabled). Loads event queues into memory. Generates network graph from nodes and edges in cache. Processes any state changes of node bundle. Initiates handshake with first contact (if provided) if node doesn't have any neighbors.
+        """
+        if self.use_kobj_processor_thread:
+            logger.info("Starting processor worker thread")
+            self.processor.worker_thread.start()
+        
+        # self.network._load_event_queues()
+        self.graph.generate()
+        
+        # refresh to reflect changes (if any) in config.yaml
+        self.effector.deref(self.identity.rid, refresh_cache=True)
+                
+        logger.debug("Waiting for kobj queue to empty")
+        if self.use_kobj_processor_thread:
+            self.processor.kobj_queue.join()
+        else:
+            self.processor.flush_kobj_queue()
+        logger.debug("Done")
+    
+        if not self.graph.get_neighbors() and self.config.koi_net.first_contact.rid:
+            logger.debug(f"I don't have any neighbors, reaching out to first contact {self.config.koi_net.first_contact.rid!r}")
+            
+            self.actor.handshake_with(self.config.koi_net.first_contact.rid)
+            
+                        
+    def stop(self):
+        """Stops a node, call this method last.
+        
+        Finishes processing knowledge object queue. Saves event queues to storage.
+        """
+        logger.info("Stopping node...")
+        
+        if self.use_kobj_processor_thread:
+            logger.info(f"Waiting for kobj queue to empty ({self.processor.kobj_queue.unfinished_tasks} tasks remaining)")
+            self.processor.kobj_queue.join()
+        else:
+            self.processor.flush_kobj_queue()
+        
+        # self.network._save_event_queues()

koi_net-1.1.0b1/src/koi_net/default_actions.py

@@ -0,0 +1,15 @@
+from .context import ActionContext
+from rid_lib.types import KoiNetNode
+from rid_lib.ext import Bundle
+from .effector import Effector
+
+
+@Effector.register_default_action(KoiNetNode)
+def dereference_koi_node(ctx: ActionContext, rid: KoiNetNode) -> Bundle:
+    if rid != ctx.identity.rid:
+        return
+    
+    return Bundle.generate(
+        rid=ctx.identity.rid,
+        contents=ctx.identity.profile.model_dump()
+    )

koi_net-1.1.0b1/src/koi_net/effector.py

@@ -0,0 +1,139 @@
+import logging
+from typing import Callable
+from enum import StrEnum
+from rid_lib.ext import Cache, Bundle
+from rid_lib.core import RID, RIDType
+from rid_lib.types import KoiNetNode
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .network.resolver import NetworkResolver
+    from .processor.interface import ProcessorInterface
+    from .context import ActionContext
+
+logger = logging.getLogger(__name__)
+
+
+class BundleSource(StrEnum):
+    CACHE = "CACHE"
+    ACTION = "ACTION"
+
+class Effector:
+    cache: Cache
+    resolver: "NetworkResolver | None"
+    processor: "ProcessorInterface | None"
+    action_context: "ActionContext | None"
+    _action_table: dict[
+        type[RID], 
+        Callable[
+            ["ActionContext", RID], 
+            Bundle | None
+        ]
+    ] = dict()
+    
+    def __init__(
+        self, 
+        cache: Cache,
+    ):
+        self.cache = cache
+        self.resolver = None
+        self.processor = None
+        self.action_context = None
+        self._action_table = self.__class__._action_table.copy()
+        
+    def set_processor(self, processor: "ProcessorInterface"):
+        self.processor = processor
+        
+    def set_resolver(self, resolver: "NetworkResolver"):
+        self.resolver = resolver
+        
+    def set_action_context(self, action_context: "ActionContext"):
+        self.action_context = action_context
+        
+    @classmethod
+    def register_default_action(cls, rid_type: RIDType):
+        def decorator(func: Callable) -> Callable:
+            cls._action_table[rid_type] = func
+            return func
+        return decorator
+        
+    def register_action(self, rid_type: RIDType):
+        def decorator(func: Callable) -> Callable:
+            self._action_table[rid_type] = func
+            return func
+        return decorator
+    
+    def _try_cache(self, rid: RID) -> tuple[Bundle, BundleSource] | None:
+        bundle = self.cache.read(rid)
+        
+        if bundle:
+            logger.debug("Cache hit")
+            return bundle, BundleSource.CACHE
+        else:
+            logger.debug("Cache miss")
+            return None
+                    
+    def _try_action(self, rid: RID) -> tuple[Bundle, BundleSource] | None:
+        if type(rid) not in self._action_table:
+            logger.debug("No action available")
+            return None
+        
+        logger.debug("Action available")
+        func = self._action_table[type(rid)]
+        bundle = func(
+            ctx=self.action_context, 
+            rid=rid
+        )
+        
+        if bundle:
+            logger.debug("Action hit")
+            return bundle, BundleSource.ACTION
+        else:
+            logger.debug("Action miss")
+            return None
+
+        
+    def _try_network(self, rid: RID) -> tuple[Bundle, KoiNetNode] | None:
+        bundle, source = self.resolver.fetch_remote_bundle(rid)
+        
+        if bundle:
+            logger.debug("Network hit")
+            return bundle, source
+        else:
+            logger.debug("Network miss")
+            return None
+        
+    
+    def deref(
+        self, 
+        rid: RID,
+        refresh_cache: bool = False,
+        use_network: bool = False,
+        handle_result: bool = True
+    ) -> Bundle | None:
+        logger.debug(f"Dereferencing {rid!r}")
+        
+        bundle, source = (
+            # if `refresh_cache`, skip try cache
+            not refresh_cache and self._try_cache(rid) or 
+            self._try_action(rid) or
+            use_network and self._try_network(rid) or
+            # if not found, bundle and source set to None
+            (None, None) 
+        )
+        
+        if (
+            handle_result 
+            and bundle is not None 
+            and source != BundleSource.CACHE
+        ):            
+            self.processor.handle(
+                bundle=bundle, 
+                source=source if type(source) is KoiNetNode else None
+            )
+
+            # TODO: refactor for general solution, param to write through to cache before continuing
+            # like `self.processor.kobj_queue.join()``
+
+        return bundle