koi-net 1.2.0b1__py3-none-any.whl → 1.2.0b2__py3-none-any.whl

koi_net/lifecycle.py

@@ -1,22 +1,22 @@
-import logging
+import structlog
 from contextlib import contextmanager, asynccontextmanager
 
 from rid_lib.ext import Bundle, Cache
 from rid_lib.types import KoiNetNode
 
-from koi_net.behaviors import Behaviors
-from koi_net.handshaker import Handshaker
-from koi_net.kobj_worker import KnowledgeProcessingWorker
-from koi_net.models import END
-from koi_net.network.event_queue import EventQueue
-from koi_net.processor.event_worker import EventProcessingWorker
-
+from .handshaker import Handshaker
+from .network.request_handler import RequestHandler
+from .processor.kobj_worker import KnowledgeProcessingWorker
+from .network.event_queue import EventQueue
+from .processor.event_worker import EventProcessingWorker
+from .protocol.api_models import ErrorResponse
+from .interfaces.worker import STOP_WORKER
 from .config import NodeConfig
 from .processor.kobj_queue import KobjQueue
 from .network.graph import NetworkGraph
 from .identity import NodeIdentity
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class NodeLifecycle:
@@ -31,7 +31,7 @@ class NodeLifecycle:
     event_worker: EventProcessingWorker
     cache: Cache
     handshaker: Handshaker
-    behaviors: Behaviors
+    request_handler: RequestHandler
     
     def __init__(
         self,
@@ -44,7 +44,7 @@ class NodeLifecycle:
         event_worker: EventProcessingWorker,
         cache: Cache,
         handshaker: Handshaker,
-        behaviors: Behaviors
+        request_handler: RequestHandler
     ):
         self.config = config
         self.identity = identity
@@ -55,32 +55,32 @@ class NodeLifecycle:
         self.event_worker = event_worker
         self.cache = cache
         self.handshaker = handshaker
-        self.behaviors = behaviors
+        self.request_handler = request_handler
         
     @contextmanager
     def run(self):
         """Synchronous context manager for node startup and shutdown."""
         try:
-            logger.info("Starting node lifecycle...")
+            log.info("Starting node lifecycle...")
             self.start()
             yield
         except KeyboardInterrupt:
-            logger.info("Keyboard interrupt!")
+            log.info("Keyboard interrupt!")
         finally:
-            logger.info("Stopping node lifecycle...")
+            log.info("Stopping node lifecycle...")
             self.stop()
 
     @asynccontextmanager
     async def async_run(self):
         """Asynchronous context manager for node startup and shutdown."""
         try:
-            logger.info("Starting async node lifecycle...")
+            log.info("Starting async node lifecycle...")
             self.start()
             yield
         except KeyboardInterrupt:
-            logger.info("Keyboard interrupt!")
+            log.info("Keyboard interrupt!")
         finally:
-            logger.info("Stopping async node lifecycle...")
+            log.info("Stopping async node lifecycle...")
             self.stop()
     
     def start(self):
@@ -91,7 +91,7 @@ class NodeLifecycle:
         of node bundle. Initiates handshake with first contact if node 
         doesn't have any neighbors. Catches up with coordinator state.
         """
-        logger.info("Starting processor worker thread")
+        log.info("Starting processor worker thread")
         
         self.kobj_worker.thread.start()
         self.event_worker.thread.start()
@@ -104,30 +104,38 @@ class NodeLifecycle:
             contents=self.identity.profile.model_dump()
         ))
         
-        logger.debug("Waiting for kobj queue to empty")
+        log.debug("Waiting for kobj queue to empty")
         self.kobj_queue.q.join()
         
-        # TODO: FACTOR OUT BEHAVIOR
-        
         coordinators = self.graph.get_neighbors(direction="in", allowed_type=KoiNetNode)
         
-        if len(coordinators) == 0 and self.config.koi_net.first_contact.rid:
-            logger.debug(f"I don't have any edges with coordinators, reaching out to first contact {self.config.koi_net.first_contact.rid!r}")
+        if len(coordinators) > 0:
+            for coordinator in coordinators:
+                payload = self.request_handler.fetch_manifests(
+                    node=coordinator,
+                    rid_types=[KoiNetNode]
+                )
+                if type(payload) is ErrorResponse:
+                    continue
+                
+                for manifest in payload.manifests:
+                    self.kobj_queue.put_kobj(
+                        manifest=manifest,
+                        source=coordinator
+                    )
+                    
+        elif self.config.koi_net.first_contact.rid:
+            log.debug(f"I don't have any edges with coordinators, reaching out to first contact {self.config.koi_net.first_contact.rid!r}")
             
             self.handshaker.handshake_with(self.config.koi_net.first_contact.rid)
         
-        
-        
-        for coordinator in self.behaviors.identify_coordinators():
-            self.behaviors.catch_up_with(coordinator, rid_types=[KoiNetNode])
-        
 
     def stop(self):
         """Stops a node.
         
         Finishes processing knowledge object queue.
         """        
-        logger.info(f"Waiting for kobj queue to empty ({self.kobj_queue.q.unfinished_tasks} tasks remaining)")
+        log.info(f"Waiting for kobj queue to empty ({self.kobj_queue.q.unfinished_tasks} tasks remaining)")
         
-        self.kobj_queue.q.put(END)
-        self.event_queue.q.put(END)
+        self.kobj_queue.q.put(STOP_WORKER)
+        self.event_queue.q.put(STOP_WORKER)

koi_net/logger.py

@@ -0,0 +1,176 @@
+from datetime import datetime
+import logging
+from logging.handlers import RotatingFileHandler
+import colorama
+import structlog
+import sys
+# from sentry_sdk import logger as sentry_logger
+
+
+def my_processor(_, __, event: dict):
+    # print(_, __, event)
+    event["path"] = event["module"] + "." + event["func_name"]
+    return event
+
+# def sentry_processor(_, method, event: dict):
+#     print(event)
+#     if method == "critical":
+#         sentry_logger.fatal(
+#             event["event"],
+#             attributes=event
+#         )
+#     elif method == "info":
+#         sentry_logger.info(
+#             event["event"],
+#             attributes=event
+#         )
+#     elif method == "debug":
+#         sentry_logger.debug(
+#             event["event"],
+#             attributes=event
+#         )
+#     return event
+
+console_renderer = structlog.dev.ConsoleRenderer(
+    columns=[
+        # Render the timestamp without the key name in yellow.
+        structlog.dev.Column(
+            "timestamp",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=None,
+                value_style=colorama.Style.DIM,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=lambda t: datetime.fromisoformat(t).strftime("%Y-%m-%d %H:%M:%S"),
+            ),
+        ),
+        structlog.dev.Column(
+            "level",
+            structlog.dev.LogLevelColumnFormatter(
+                level_styles={
+                    level: colorama.Style.BRIGHT + color
+                    for level, color in {
+                        "critical": colorama.Fore.RED,
+                        "exception": colorama.Fore.RED,
+                        "error": colorama.Fore.RED,
+                        "warn": colorama.Fore.YELLOW,
+                        "warning": colorama.Fore.YELLOW,
+                        "info": colorama.Fore.GREEN,
+                        "debug": colorama.Fore.GREEN,
+                        "notset": colorama.Back.RED,
+                    }.items()
+                },
+                reset_style=colorama.Style.RESET_ALL,
+                width=9
+            )
+        ),
+        # Render the event without the key name in bright magenta.
+        
+        # Default formatter for all keys not explicitly mentioned. The key is
+        # cyan, the value is green.
+        structlog.dev.Column(
+            "path",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=None,
+                value_style=colorama.Fore.MAGENTA,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=str,
+                width=30
+            ),
+        ),
+        # structlog.dev.Column(
+        #     "func_name",
+        #     structlog.dev.KeyValueColumnFormatter(
+        #         key_style=None,
+        #         value_style=colorama.Fore.MAGENTA,
+        #         reset_style=colorama.Style.RESET_ALL,
+        #         value_repr=str,
+        #         prefix="(",
+        #         postfix=")",
+        #         width=15
+        #     ),
+        # ),
+        structlog.dev.Column(
+            "event",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=None,
+                value_style=colorama.Fore.WHITE,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=str,
+                width=30
+            ),
+        ),
+        structlog.dev.Column(
+            "",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=colorama.Fore.BLUE,
+                value_style=colorama.Fore.GREEN,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=str,
+            ),
+        )
+    ]
+)
+
+structlog.configure(
+    processors=[
+        # If log level is too low, abort pipeline and throw away log entry.
+        structlog.stdlib.filter_by_level,
+        # Add the name of the logger to event dict.
+        structlog.stdlib.add_logger_name,
+        # Add log level to event dict.
+        structlog.stdlib.add_log_level,
+        # Perform %-style formatting.
+        structlog.stdlib.PositionalArgumentsFormatter(),
+        # Add a timestamp in ISO 8601 format.
+        structlog.processors.TimeStamper(fmt="iso"),
+        # If the "stack_info" key in the event dict is true, remove it and
+        # render the current stack trace in the "stack" key.
+        structlog.processors.StackInfoRenderer(),
+        # If the "exc_info" key in the event dict is either true or a
+        # sys.exc_info() tuple, remove "exc_info" and render the exception
+        # with traceback into the "exception" key.
+        # structlog.processors.format_exc_info,
+        # If some value is in bytes, decode it to a Unicode str.
+        structlog.processors.UnicodeDecoder(),
+        # Add callsite parameters.
+        structlog.processors.CallsiteParameterAdder(
+            {
+                structlog.processors.CallsiteParameter.MODULE,
+                structlog.processors.CallsiteParameter.FUNC_NAME,
+                # structlog.processors.CallsiteParameter.LINENO,
+            }
+        ),
+        my_processor,
+        # Render the final event dict as JSON.
+        # sentry_processor,
+        console_renderer
+        # structlog.processors.JSONRenderer()
+        
+    ],
+    # `wrapper_class` is the bound logger that you get back from
+    # get_logger(). This one imitates the API of `logging.Logger`.
+    wrapper_class=structlog.stdlib.BoundLogger,
+    # `logger_factory` is used to create wrapped loggers that are used for
+    # OUTPUT. This one returns a `logging.Logger`. The final value (a JSON
+    # string) from the final processor (`JSONRenderer`) will be passed to
+    # the method of the same name as that you've called on the bound logger.
+    logger_factory=structlog.stdlib.LoggerFactory(),
+    # Effectively freeze configuration after creating the first bound
+    # logger.
+    cache_logger_on_first_use=True,
+)
+
+file_handler = RotatingFileHandler(
+    filename="app.log",
+    maxBytes=10 * 1024 * 1024,
+    backupCount=5,
+    encoding="utf-8"
+)
+
+logging.basicConfig(
+    format="%(message)s",
+    stream=sys.stdout,
+    level=logging.INFO,
+)
+
+# log = structlog.stdlib.get_logger()

koi_net/network/error_handler.py

@@ -1,11 +1,11 @@
-from logging import getLogger
+import structlog
 from koi_net.handshaker import Handshaker
 from koi_net.protocol.errors import ErrorType
 from koi_net.protocol.event import EventType
 from rid_lib.types import KoiNetNode
 from ..processor.kobj_queue import KobjQueue
 
-logger = getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class ErrorHandler:
@@ -27,10 +27,10 @@ class ErrorHandler:
         self.timeout_counter.setdefault(node, 0)
         self.timeout_counter[node] += 1
         
-        logger.debug(f"{node} has timed out {self.timeout_counter[node]} time(s)")
+        log.debug(f"{node} has timed out {self.timeout_counter[node]} time(s)")
         
         if self.timeout_counter[node] > 3:
-            logger.debug(f"Exceeded time out limit, forgetting node")
+            log.debug(f"Exceeded time out limit, forgetting node")
             self.kobj_queue.put_kobj(rid=node, event_type=EventType.FORGET)
             # do something
         
@@ -41,10 +41,10 @@ class ErrorHandler:
         node: KoiNetNode
     ):
         """Attempts handshake when this node is unknown to target."""
-        logger.info(f"Handling protocol error {error_type} for node {node!r}")
+        log.info(f"Handling protocol error {error_type} for node {node!r}")
         match error_type:
             case ErrorType.UnknownNode:
-                logger.info("Peer doesn't know me, attempting handshake...")
+                log.info("Peer doesn't know me, attempting handshake...")
                 self.handshaker.handshake_with(node)
                 
             case ErrorType.InvalidKey: ...

koi_net/network/event_queue.py

@@ -1,14 +1,18 @@
-import logging
+import structlog
 from queue import Queue
 
 from rid_lib.types import KoiNetNode
+from pydantic import BaseModel
 
-from ..models import QueuedEvent
 from ..protocol.event import Event
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
+class QueuedEvent(BaseModel):
+    event: Event
+    target: KoiNetNode
+
 class EventQueue:
     """Handles out going network event queues."""
     q: Queue[QueuedEvent]
@@ -20,9 +24,7 @@ class EventQueue:
         """Pushes event to queue of specified node.
         
         Event will be sent to webhook or poll queue depending on the 
-        node type and edge type of the specified node. If `flush` is set 
-        to `True`, the webhook queued will be flushed after pushing the 
-        event.
+        node type and edge type of the specified node.
         """
         
         self.q.put(QueuedEvent(target=target, event=event))

koi_net/network/graph.py

@@ -1,4 +1,4 @@
-import logging
+import structlog
 from typing import Literal
 import networkx as nx
 from rid_lib import RIDType
@@ -7,7 +7,7 @@ from rid_lib.types import KoiNetEdge, KoiNetNode
 from ..identity import NodeIdentity
 from ..protocol.edge import EdgeProfile, EdgeStatus
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class NetworkGraph:
@@ -24,22 +24,22 @@ class NetworkGraph:
         
     def generate(self):
         """Generates directed graph from cached KOI nodes and edges."""
-        logger.debug("Generating network graph")
+        log.debug("Generating network graph")
         self.dg.clear()
         for rid in self.cache.list_rids():
             if type(rid) == KoiNetNode:                
                 self.dg.add_node(rid)
-                logger.debug(f"Added node {rid!r}")
+                log.debug(f"Added node {rid!r}")
                 
             elif type(rid) == KoiNetEdge:
                 edge_bundle = self.cache.read(rid)
                 if not edge_bundle:
-                    logger.warning(f"Failed to load {rid!r}")
+                    log.warning(f"Failed to load {rid!r}")
                     continue
                 edge_profile = edge_bundle.validate_contents(EdgeProfile)
                 self.dg.add_edge(edge_profile.source, edge_profile.target, rid=rid)
-                logger.debug(f"Added edge {rid!r} ({edge_profile.source} -> {edge_profile.target})")
-        logger.debug("Done")
+                log.debug(f"Added edge {rid!r} ({edge_profile.source} -> {edge_profile.target})")
+        log.debug("Done")
         
     def get_edge(self, source: KoiNetNode, target: KoiNetNode,) -> KoiNetEdge | None:
         """Returns edge RID given the RIDs of a source and target node."""
@@ -97,7 +97,7 @@ class NetworkGraph:
             edge_bundle = self.cache.read(edge_rid)
             
             if not edge_bundle: 
-                logger.warning(f"Failed to find edge {edge_rid!r} in cache")
+                log.warning(f"Failed to find edge {edge_rid!r} in cache")
                 continue
             
             edge_profile = edge_bundle.validate_contents(EdgeProfile)

koi_net/network/poll_event_buffer.py

@@ -0,0 +1,26 @@
+from rid_lib.types import KoiNetNode
+
+from koi_net.protocol.event import Event
+
+
+class PollEventBuffer:
+    buffers: dict[KoiNetNode, list[Event]]
+    
+    def __init__(self):
+        self.buffers = dict()
+        
+    def put(self, node: KoiNetNode, event: Event):
+        event_buf = self.buffers.setdefault(node, [])
+        event_buf.append(event)
+        
+    def flush(self, node: KoiNetNode, limit: int = 0):
+        event_buf = self.buffers.get(node, [])
+        
+        if limit and len(event_buf) > limit:
+            to_return = event_buf[:limit]
+            self.buffers[node] = event_buf[limit:]
+        else:
+            to_return = event_buf.copy()
+            self.buffers[node] = []
+        
+        return to_return

koi_net/network/request_handler.py

@@ -1,9 +1,11 @@
-import logging
+import structlog
 import httpx
 from rid_lib import RID
 from rid_lib.ext import Cache
 from rid_lib.types.koi_net_node import KoiNetNode
 
+from koi_net.protocol.model_map import API_MODEL_MAP
+
 from ..identity import NodeIdentity
 from ..protocol.api_models import (
     RidsPayload,
@@ -30,8 +32,7 @@ from ..protocol.node import NodeProfile, NodeType
 from ..secure import Secure
 from .error_handler import ErrorHandler
 
-
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 # Custom error types for request handling
@@ -74,7 +75,7 @@ class RequestHandler:
     def get_url(self, node_rid: KoiNetNode) -> str:
         """Retrieves URL of a node from its RID."""
         
-        logger.debug(f"Getting URL for {node_rid!r}")
+        log.debug(f"Getting URL for {node_rid!r}")
         node_url = None
         
         if node_rid == self.identity.rid:
@@ -84,20 +85,20 @@ class RequestHandler:
                 
         if node_bundle:
             node_profile = node_bundle.validate_contents(NodeProfile)
-            logger.debug(f"Found node profile: {node_profile}")
+            log.debug(f"Found node profile: {node_profile}")
             if node_profile.node_type != NodeType.FULL:
                 raise PartialNodeQueryError("Can't query partial node")
             node_url = node_profile.base_url
         
         else:
             if node_rid == self.identity.config.koi_net.first_contact.rid:
-                logger.debug("Found URL of first contact")
+                log.debug("Found URL of first contact")
                 node_url = self.identity.config.koi_net.first_contact.url
         
         if not node_url:
             raise NodeNotFoundError("Node not found")
         
-        logger.debug(f"Resolved {node_rid!r} to {node_url}")
+        log.debug(f"Resolved {node_rid!r} to {node_url}")
         return node_url
     
     def make_request(
@@ -108,7 +109,7 @@ class RequestHandler:
     ) -> ResponseModels | None:
         """Makes a request to a node."""
         url = self.get_url(node) + path
-        logger.info(f"Making request to {url}")
+        log.info(f"Making request to {url}")
     
         signed_envelope = self.secure.create_envelope(
             payload=request,
@@ -116,9 +117,12 @@ class RequestHandler:
         )
         
         try:
-            result = httpx.post(url, data=signed_envelope.model_dump_json(exclude_none=True))
+            result = httpx.post(
+                url, 
+                data=signed_envelope.model_dump_json(exclude_none=True)
+            )
         except httpx.ConnectError as err:
-            logger.debug("Failed to connect")
+            log.debug("Failed to connect")
             self.error_handler.handle_connection_error(node)
             raise err
         
@@ -127,20 +131,11 @@ class RequestHandler:
             self.error_handler.handle_protocol_error(resp.error, node)
             return resp
         
-        if path == BROADCAST_EVENTS_PATH:
-            return None
-        elif path == POLL_EVENTS_PATH:
-            EnvelopeModel = SignedEnvelope[EventsPayload]
-        elif path == FETCH_RIDS_PATH:
-            EnvelopeModel = SignedEnvelope[RidsPayload]
-        elif path == FETCH_MANIFESTS_PATH:
-            EnvelopeModel = SignedEnvelope[ManifestsPayload]
-        elif path == FETCH_BUNDLES_PATH:
-            EnvelopeModel = SignedEnvelope[BundlesPayload]
-        else:
-            raise UnknownPathError(f"Unknown path '{path}'")
+        resp_env_model = API_MODEL_MAP[path].response_envelope
+        if resp_env_model is None:
+            return
         
-        resp_envelope = EnvelopeModel.model_validate_json(result.text)
+        resp_envelope = resp_env_model.model_validate_json(result.text)
         self.secure.validate_envelope(resp_envelope)
         
         return resp_envelope.payload
@@ -157,7 +152,7 @@ class RequestHandler:
         """
         request = req or EventsPayload.model_validate(kwargs)
         self.make_request(node, BROADCAST_EVENTS_PATH, request)
-        logger.info(f"Broadcasted {len(request.events)} event(s) to {node!r}")
+        log.info(f"Broadcasted {len(request.events)} event(s) to {node!r}")
         
     def poll_events(
         self, 
@@ -172,7 +167,7 @@ class RequestHandler:
         request = req or PollEvents.model_validate(kwargs)
         resp = self.make_request(node, POLL_EVENTS_PATH, request)
         if type(resp) != ErrorResponse:
-            logger.info(f"Polled {len(resp.events)} events from {node!r}")
+            log.info(f"Polled {len(resp.events)} events from {node!r}")
         return resp
         
     def fetch_rids(
@@ -188,7 +183,7 @@ class RequestHandler:
         request = req or FetchRids.model_validate(kwargs)
         resp = self.make_request(node, FETCH_RIDS_PATH, request)
         if type(resp) != ErrorResponse:
-            logger.info(f"Fetched {len(resp.rids)} RID(s) from {node!r}")
+            log.info(f"Fetched {len(resp.rids)} RID(s) from {node!r}")
         return resp
                 
     def fetch_manifests(
@@ -204,7 +199,7 @@ class RequestHandler:
         request = req or FetchManifests.model_validate(kwargs)
         resp = self.make_request(node, FETCH_MANIFESTS_PATH, request)
         if type(resp) != ErrorResponse:
-            logger.info(f"Fetched {len(resp.manifests)} manifest(s) from {node!r}")
+            log.info(f"Fetched {len(resp.manifests)} manifest(s) from {node!r}")
         return resp
                 
     def fetch_bundles(
@@ -220,5 +215,5 @@ class RequestHandler:
         request = req or FetchBundles.model_validate(kwargs)
         resp = self.make_request(node, FETCH_BUNDLES_PATH, request)
         if type(resp) != ErrorResponse:
-            logger.info(f"Fetched {len(resp.bundles)} bundle(s) from {node!r}")
+            log.info(f"Fetched {len(resp.bundles)} bundle(s) from {node!r}")
         return resp