koi-net 1.2.0b1__py3-none-any.whl → 1.2.0b3__py3-none-any.whl

koi_net/{poller.py → entrypoints/poller.py}

@@ -1,27 +1,29 @@
 
 import time
-import logging
-from .processor.kobj_queue import KobjQueue
-from .lifecycle import NodeLifecycle
-from .network.resolver import NetworkResolver
-from .config import NodeConfig
+import structlog
 
-logger = logging.getLogger(__name__)
+from .base import EntryPoint
+from ..processor.kobj_queue import KobjQueue
+from ..lifecycle import NodeLifecycle
+from ..network.resolver import NetworkResolver
+from ..config.partial_node import PartialNodeConfig
 
+log = structlog.stdlib.get_logger()
 
-class NodePoller:
+
+class NodePoller(EntryPoint):
     """Manages polling based event loop for partial nodes."""
     kobj_queue: KobjQueue
     lifecycle: NodeLifecycle
     resolver: NetworkResolver
-    config: NodeConfig
+    config: PartialNodeConfig
     
     def __init__(
         self,
-        kobj_queue: KobjQueue,
+        config: PartialNodeConfig,
         lifecycle: NodeLifecycle,
+        kobj_queue: KobjQueue,
         resolver: NetworkResolver,
-        config: NodeConfig
     ):
         self.kobj_queue = kobj_queue
         self.lifecycle = lifecycle
@@ -33,7 +35,7 @@ class NodePoller:
         neighbors = self.resolver.poll_neighbors()
         for node_rid in neighbors:
             for event in neighbors[node_rid]:
-                self.kobj_queue.put_kobj(event=event, source=node_rid)
+                self.kobj_queue.push(event=event, source=node_rid)
 
     def run(self):
         """Runs polling event loop."""
@@ -42,6 +44,6 @@ class NodePoller:
                 start_time = time.time()
                 self.poll()
                 elapsed = time.time() - start_time
-                sleep_time = self.config.koi_net.polling_interval - elapsed
+                sleep_time = self.config.poller.polling_interval - elapsed
                 if sleep_time > 0:
                     time.sleep(sleep_time)

koi_net/entrypoints/server.py

@@ -0,0 +1,94 @@
+import structlog
+import uvicorn
+from contextlib import asynccontextmanager
+from fastapi import FastAPI, APIRouter
+from fastapi.responses import JSONResponse
+
+from .base import EntryPoint
+from ..network.response_handler import ResponseHandler
+from ..protocol.model_map import API_MODEL_MAP
+from ..protocol.api_models import ErrorResponse
+from ..protocol.errors import ProtocolError
+from ..lifecycle import NodeLifecycle
+from ..config.full_node import FullNodeConfig
+
+log = structlog.stdlib.get_logger()
+
+
+class NodeServer(EntryPoint):
+    """Manages FastAPI server and event handling for full nodes."""
+    config: FullNodeConfig
+    lifecycle: NodeLifecycle
+    response_handler: ResponseHandler
+    app: FastAPI
+    router: APIRouter
+    
+    def __init__(
+        self,
+        config: FullNodeConfig,
+        lifecycle: NodeLifecycle,
+        response_handler: ResponseHandler,
+    ):
+        self.config = config
+        self.lifecycle = lifecycle
+        self.response_handler = response_handler
+        self._build_app()
+        
+    def _build_app(self):
+        """Builds FastAPI app and adds endpoints."""
+        @asynccontextmanager
+        async def lifespan(*args, **kwargs):
+            async with self.lifecycle.async_run():
+                yield
+        
+        self.app = FastAPI(
+            lifespan=lifespan, 
+            title="KOI-net Protocol API",
+            version="1.0.0"
+        )
+        
+        self.app.add_exception_handler(ProtocolError, self.protocol_error_handler)
+        
+        self.router = APIRouter(prefix="/koi-net")
+        
+        for path, models in API_MODEL_MAP.items():
+            def create_endpoint(path: str):
+                async def endpoint(req):
+                    return self.response_handler.handle_response(path, req)
+                
+                # programmatically setting type hint annotations for FastAPI's model validation 
+                endpoint.__annotations__ = {
+                    "req": models.request_envelope,
+                    "return": models.response_envelope
+                }
+                
+                return endpoint
+            
+            self.router.add_api_route(
+                path=path,
+                endpoint=create_endpoint(path),
+                methods=["POST"],
+                response_model_exclude_none=True
+            )
+        
+        self.app.include_router(self.router)
+        
+    def protocol_error_handler(self, request, exc: ProtocolError):
+        """Catches `ProtocolError` and returns as `ErrorResponse`."""
+        log.info(f"caught protocol error: {exc}")
+        resp = ErrorResponse(error=exc.error_type)
+        log.info(f"returning error response: {resp}")
+        return JSONResponse(
+            status_code=400,
+            content=resp.model_dump(mode="json")
+        )
+    
+    def run(self):
+        """Starts FastAPI server and event handler."""
+        uvicorn.run(
+            app=self.app,
+            host=self.config.server.host,
+            port=self.config.server.port,
+            log_config=None,
+            access_log=False
+        )

koi_net/handshaker.py

@@ -1,11 +1,11 @@
-from logging import getLogger
+import structlog
 from rid_lib.ext import Cache
 from rid_lib.types import KoiNetNode
 from koi_net.identity import NodeIdentity
 from koi_net.network.event_queue import EventQueue
 from .protocol.event import Event, EventType
 
-logger = getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class Handshaker:
@@ -24,14 +24,14 @@ class Handshaker:
         Pushes successive `FORGET` and `NEW` events to target node to
         reset the target's cache in case it already knew this node. 
         """
-        logger.debug(f"Initiating handshake with {target}")
-        self.event_queue.push_event_to(
+        log.debug(f"Initiating handshake with {target}")
+        self.event_queue.push(
             Event.from_rid(
                 event_type=EventType.FORGET, 
                 rid=self.identity.rid),
             target=target
         )
-        self.event_queue.push_event_to(
+        self.event_queue.push(
             event=Event.from_bundle(
                 event_type=EventType.NEW, 
                 bundle=self.cache.read(self.identity.rid)),

koi_net/identity.py

@@ -1,10 +1,9 @@
-import logging
+import structlog
 from rid_lib.types.koi_net_node import KoiNetNode
-from .config import NodeConfig
+from .config.core import NodeConfig
 from .protocol.node import NodeProfile
 
-
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
     
 class NodeIdentity:

koi_net/lifecycle.py

@@ -1,22 +1,22 @@
-import logging
+import structlog
 from contextlib import contextmanager, asynccontextmanager
 
 from rid_lib.ext import Bundle, Cache
 from rid_lib.types import KoiNetNode
 
-from koi_net.behaviors import Behaviors
-from koi_net.handshaker import Handshaker
-from koi_net.kobj_worker import KnowledgeProcessingWorker
-from koi_net.models import END
-from koi_net.network.event_queue import EventQueue
-from koi_net.processor.event_worker import EventProcessingWorker
-
-from .config import NodeConfig
+from .handshaker import Handshaker
+from .network.request_handler import RequestHandler
+from .workers.kobj_worker import KnowledgeProcessingWorker
+from .network.event_queue import EventQueue
+from .workers import EventProcessingWorker
+from .protocol.api_models import ErrorResponse
+from .workers.base import STOP_WORKER
+from .config.core import NodeConfig
 from .processor.kobj_queue import KobjQueue
 from .network.graph import NetworkGraph
 from .identity import NodeIdentity
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class NodeLifecycle:
@@ -31,7 +31,7 @@ class NodeLifecycle:
     event_worker: EventProcessingWorker
     cache: Cache
     handshaker: Handshaker
-    behaviors: Behaviors
+    request_handler: RequestHandler
     
     def __init__(
         self,
@@ -44,7 +44,7 @@ class NodeLifecycle:
         event_worker: EventProcessingWorker,
         cache: Cache,
         handshaker: Handshaker,
-        behaviors: Behaviors
+        request_handler: RequestHandler
     ):
         self.config = config
         self.identity = identity
@@ -55,32 +55,32 @@ class NodeLifecycle:
         self.event_worker = event_worker
         self.cache = cache
         self.handshaker = handshaker
-        self.behaviors = behaviors
+        self.request_handler = request_handler
         
     @contextmanager
     def run(self):
         """Synchronous context manager for node startup and shutdown."""
         try:
-            logger.info("Starting node lifecycle...")
+            log.info("Starting node lifecycle...")
             self.start()
             yield
         except KeyboardInterrupt:
-            logger.info("Keyboard interrupt!")
+            log.info("Keyboard interrupt!")
         finally:
-            logger.info("Stopping node lifecycle...")
+            log.info("Stopping node lifecycle...")
             self.stop()
 
     @asynccontextmanager
     async def async_run(self):
         """Asynchronous context manager for node startup and shutdown."""
         try:
-            logger.info("Starting async node lifecycle...")
+            log.info("Starting async node lifecycle...")
             self.start()
             yield
         except KeyboardInterrupt:
-            logger.info("Keyboard interrupt!")
+            log.info("Keyboard interrupt!")
         finally:
-            logger.info("Stopping async node lifecycle...")
+            log.info("Stopping async node lifecycle...")
             self.stop()
     
     def start(self):
@@ -91,7 +91,7 @@ class NodeLifecycle:
         of node bundle. Initiates handshake with first contact if node 
         doesn't have any neighbors. Catches up with coordinator state.
         """
-        logger.info("Starting processor worker thread")
+        log.info("Starting processor worker thread")
         
         self.kobj_worker.thread.start()
         self.event_worker.thread.start()
@@ -99,35 +99,43 @@ class NodeLifecycle:
         
         # refresh to reflect changes (if any) in config.yaml
         
-        self.kobj_queue.put_kobj(bundle=Bundle.generate(
+        self.kobj_queue.push(bundle=Bundle.generate(
             rid=self.identity.rid,
             contents=self.identity.profile.model_dump()
         ))
         
-        logger.debug("Waiting for kobj queue to empty")
+        log.debug("Waiting for kobj queue to empty")
         self.kobj_queue.q.join()
         
-        # TODO: FACTOR OUT BEHAVIOR
-        
         coordinators = self.graph.get_neighbors(direction="in", allowed_type=KoiNetNode)
         
-        if len(coordinators) == 0 and self.config.koi_net.first_contact.rid:
-            logger.debug(f"I don't have any edges with coordinators, reaching out to first contact {self.config.koi_net.first_contact.rid!r}")
+        if len(coordinators) > 0:
+            for coordinator in coordinators:
+                payload = self.request_handler.fetch_manifests(
+                    node=coordinator,
+                    rid_types=[KoiNetNode]
+                )
+                if type(payload) is ErrorResponse:
+                    continue
+                
+                for manifest in payload.manifests:
+                    self.kobj_queue.push(
+                        manifest=manifest,
+                        source=coordinator
+                    )
+                    
+        elif self.config.koi_net.first_contact.rid:
+            log.debug(f"I don't have any edges with coordinators, reaching out to first contact {self.config.koi_net.first_contact.rid!r}")
             
             self.handshaker.handshake_with(self.config.koi_net.first_contact.rid)
         
-        
-        
-        for coordinator in self.behaviors.identify_coordinators():
-            self.behaviors.catch_up_with(coordinator, rid_types=[KoiNetNode])
-        
 
     def stop(self):
         """Stops a node.
         
         Finishes processing knowledge object queue.
         """        
-        logger.info(f"Waiting for kobj queue to empty ({self.kobj_queue.q.unfinished_tasks} tasks remaining)")
+        log.info(f"Waiting for kobj queue to empty ({self.kobj_queue.q.unfinished_tasks} tasks remaining)")
         
-        self.kobj_queue.q.put(END)
-        self.event_queue.q.put(END)
+        self.kobj_queue.q.put(STOP_WORKER)
+        self.event_queue.q.put(STOP_WORKER)

koi_net/logger.py

@@ -0,0 +1,176 @@
+from datetime import datetime
+import logging
+from logging.handlers import RotatingFileHandler
+import colorama
+import structlog
+import sys
+# from sentry_sdk import logger as sentry_logger
+
+
+def my_processor(_, __, event: dict):
+    # print(_, __, event)
+    event["path"] = event["module"] + "." + event["func_name"]
+    return event
+
+# def sentry_processor(_, method, event: dict):
+#     print(event)
+#     if method == "critical":
+#         sentry_logger.fatal(
+#             event["event"],
+#             attributes=event
+#         )
+#     elif method == "info":
+#         sentry_logger.info(
+#             event["event"],
+#             attributes=event
+#         )
+#     elif method == "debug":
+#         sentry_logger.debug(
+#             event["event"],
+#             attributes=event
+#         )
+#     return event
+
+console_renderer = structlog.dev.ConsoleRenderer(
+    columns=[
+        # Render the timestamp without the key name in yellow.
+        structlog.dev.Column(
+            "timestamp",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=None,
+                value_style=colorama.Style.DIM,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=lambda t: datetime.fromisoformat(t).strftime("%Y-%m-%d %H:%M:%S"),
+            ),
+        ),
+        structlog.dev.Column(
+            "level",
+            structlog.dev.LogLevelColumnFormatter(
+                level_styles={
+                    level: colorama.Style.BRIGHT + color
+                    for level, color in {
+                        "critical": colorama.Fore.RED,
+                        "exception": colorama.Fore.RED,
+                        "error": colorama.Fore.RED,
+                        "warn": colorama.Fore.YELLOW,
+                        "warning": colorama.Fore.YELLOW,
+                        "info": colorama.Fore.GREEN,
+                        "debug": colorama.Fore.GREEN,
+                        "notset": colorama.Back.RED,
+                    }.items()
+                },
+                reset_style=colorama.Style.RESET_ALL,
+                width=9
+            )
+        ),
+        # Render the event without the key name in bright magenta.
+        
+        # Default formatter for all keys not explicitly mentioned. The key is
+        # cyan, the value is green.
+        structlog.dev.Column(
+            "path",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=None,
+                value_style=colorama.Fore.MAGENTA,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=str,
+                width=30
+            ),
+        ),
+        # structlog.dev.Column(
+        #     "func_name",
+        #     structlog.dev.KeyValueColumnFormatter(
+        #         key_style=None,
+        #         value_style=colorama.Fore.MAGENTA,
+        #         reset_style=colorama.Style.RESET_ALL,
+        #         value_repr=str,
+        #         prefix="(",
+        #         postfix=")",
+        #         width=15
+        #     ),
+        # ),
+        structlog.dev.Column(
+            "event",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=None,
+                value_style=colorama.Fore.WHITE,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=str,
+                width=30
+            ),
+        ),
+        structlog.dev.Column(
+            "",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=colorama.Fore.BLUE,
+                value_style=colorama.Fore.GREEN,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=str,
+            ),
+        )
+    ]
+)
+
+structlog.configure(
+    processors=[
+        # If log level is too low, abort pipeline and throw away log entry.
+        structlog.stdlib.filter_by_level,
+        # Add the name of the logger to event dict.
+        structlog.stdlib.add_logger_name,
+        # Add log level to event dict.
+        structlog.stdlib.add_log_level,
+        # Perform %-style formatting.
+        structlog.stdlib.PositionalArgumentsFormatter(),
+        # Add a timestamp in ISO 8601 format.
+        structlog.processors.TimeStamper(fmt="iso"),
+        # If the "stack_info" key in the event dict is true, remove it and
+        # render the current stack trace in the "stack" key.
+        structlog.processors.StackInfoRenderer(),
+        # If the "exc_info" key in the event dict is either true or a
+        # sys.exc_info() tuple, remove "exc_info" and render the exception
+        # with traceback into the "exception" key.
+        # structlog.processors.format_exc_info,
+        # If some value is in bytes, decode it to a Unicode str.
+        structlog.processors.UnicodeDecoder(),
+        # Add callsite parameters.
+        structlog.processors.CallsiteParameterAdder(
+            {
+                structlog.processors.CallsiteParameter.MODULE,
+                structlog.processors.CallsiteParameter.FUNC_NAME,
+                # structlog.processors.CallsiteParameter.LINENO,
+            }
+        ),
+        my_processor,
+        # Render the final event dict as JSON.
+        # sentry_processor,
+        console_renderer
+        # structlog.processors.JSONRenderer()
+        
+    ],
+    # `wrapper_class` is the bound logger that you get back from
+    # get_logger(). This one imitates the API of `logging.Logger`.
+    wrapper_class=structlog.stdlib.BoundLogger,
+    # `logger_factory` is used to create wrapped loggers that are used for
+    # OUTPUT. This one returns a `logging.Logger`. The final value (a JSON
+    # string) from the final processor (`JSONRenderer`) will be passed to
+    # the method of the same name as that you've called on the bound logger.
+    logger_factory=structlog.stdlib.LoggerFactory(),
+    # Effectively freeze configuration after creating the first bound
+    # logger.
+    cache_logger_on_first_use=True,
+)
+
+file_handler = RotatingFileHandler(
+    filename="app.log",
+    maxBytes=10 * 1024 * 1024,
+    backupCount=5,
+    encoding="utf-8"
+)
+
+logging.basicConfig(
+    format="%(message)s",
+    stream=sys.stdout,
+    level=logging.INFO,
+)
+
+# log = structlog.stdlib.get_logger()

koi_net/network/error_handler.py

@@ -1,11 +1,11 @@
-from logging import getLogger
+import structlog
 from koi_net.handshaker import Handshaker
 from koi_net.protocol.errors import ErrorType
 from koi_net.protocol.event import EventType
 from rid_lib.types import KoiNetNode
 from ..processor.kobj_queue import KobjQueue
 
-logger = getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class ErrorHandler:
@@ -27,11 +27,11 @@ class ErrorHandler:
         self.timeout_counter.setdefault(node, 0)
         self.timeout_counter[node] += 1
         
-        logger.debug(f"{node} has timed out {self.timeout_counter[node]} time(s)")
+        log.debug(f"{node} has timed out {self.timeout_counter[node]} time(s)")
         
         if self.timeout_counter[node] > 3:
-            logger.debug(f"Exceeded time out limit, forgetting node")
-            self.kobj_queue.put_kobj(rid=node, event_type=EventType.FORGET)
+            log.debug(f"Exceeded time out limit, forgetting node")
+            self.kobj_queue.push(rid=node, event_type=EventType.FORGET)
             # do something
         
         
@@ -41,10 +41,10 @@ class ErrorHandler:
         node: KoiNetNode
     ):
         """Attempts handshake when this node is unknown to target."""
-        logger.info(f"Handling protocol error {error_type} for node {node!r}")
+        log.info(f"Handling protocol error {error_type} for node {node!r}")
         match error_type:
             case ErrorType.UnknownNode:
-                logger.info("Peer doesn't know me, attempting handshake...")
+                log.info("Peer doesn't know me, attempting handshake...")
                 self.handshaker.handshake_with(node)
                 
             case ErrorType.InvalidKey: ...

koi_net/network/event_queue.py

@@ -1,14 +1,18 @@
-import logging
+import structlog
 from queue import Queue
 
 from rid_lib.types import KoiNetNode
+from pydantic import BaseModel
 
-from ..models import QueuedEvent
 from ..protocol.event import Event
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
+class QueuedEvent(BaseModel):
+    event: Event
+    target: KoiNetNode
+
 class EventQueue:
     """Handles out going network event queues."""
     q: Queue[QueuedEvent]
@@ -16,13 +20,11 @@ class EventQueue:
     def __init__(self):
         self.q = Queue()
     
-    def push_event_to(self, event: Event, target: KoiNetNode):
+    def push(self, event: Event, target: KoiNetNode):
         """Pushes event to queue of specified node.
         
         Event will be sent to webhook or poll queue depending on the 
-        node type and edge type of the specified node. If `flush` is set 
-        to `True`, the webhook queued will be flushed after pushing the 
-        event.
+        node type and edge type of the specified node.
         """
         
         self.q.put(QueuedEvent(target=target, event=event))

koi_net/network/graph.py

@@ -1,4 +1,4 @@
-import logging
+import structlog
 from typing import Literal
 import networkx as nx
 from rid_lib import RIDType
@@ -7,7 +7,7 @@ from rid_lib.types import KoiNetEdge, KoiNetNode
 from ..identity import NodeIdentity
 from ..protocol.edge import EdgeProfile, EdgeStatus
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class NetworkGraph:
@@ -24,22 +24,22 @@ class NetworkGraph:
         
     def generate(self):
         """Generates directed graph from cached KOI nodes and edges."""
-        logger.debug("Generating network graph")
+        log.debug("Generating network graph")
         self.dg.clear()
         for rid in self.cache.list_rids():
             if type(rid) == KoiNetNode:                
                 self.dg.add_node(rid)
-                logger.debug(f"Added node {rid!r}")
+                log.debug(f"Added node {rid!r}")
                 
             elif type(rid) == KoiNetEdge:
                 edge_bundle = self.cache.read(rid)
                 if not edge_bundle:
-                    logger.warning(f"Failed to load {rid!r}")
+                    log.warning(f"Failed to load {rid!r}")
                     continue
                 edge_profile = edge_bundle.validate_contents(EdgeProfile)
                 self.dg.add_edge(edge_profile.source, edge_profile.target, rid=rid)
-                logger.debug(f"Added edge {rid!r} ({edge_profile.source} -> {edge_profile.target})")
-        logger.debug("Done")
+                log.debug(f"Added edge {rid!r} ({edge_profile.source} -> {edge_profile.target})")
+        log.debug("Done")
         
     def get_edge(self, source: KoiNetNode, target: KoiNetNode,) -> KoiNetEdge | None:
         """Returns edge RID given the RIDs of a source and target node."""
@@ -97,7 +97,7 @@ class NetworkGraph:
             edge_bundle = self.cache.read(edge_rid)
             
             if not edge_bundle: 
-                logger.warning(f"Failed to find edge {edge_rid!r} in cache")
+                log.warning(f"Failed to find edge {edge_rid!r} in cache")
                 continue
             
             edge_profile = edge_bundle.validate_contents(EdgeProfile)