koi-net 1.1.0b8__py3-none-any.whl → 1.2.0b2__py3-none-any.whl

koi_net/effector.py

@@ -1,33 +1,43 @@
-import logging
+import structlog
 from typing import Callable
 from enum import StrEnum
 from rid_lib.ext import Cache, Bundle
 from rid_lib.core import RID, RIDType
 from rid_lib.types import KoiNetNode
+from .network.resolver import NetworkResolver
+from .processor.kobj_queue import KobjQueue
+from .identity import NodeIdentity
 
-from typing import TYPE_CHECKING
+log = structlog.stdlib.get_logger()
 
-if TYPE_CHECKING:
-    from .network.resolver import NetworkResolver
-    from .processor.interface import ProcessorInterface
-    from .context import ActionContext
 
-logger = logging.getLogger(__name__)
+class ActionContext:
+    """Provides action handlers access to other subsystems."""
+    
+    identity: NodeIdentity
 
+    def __init__(
+        self,
+        identity: NodeIdentity,
+    ):
+        self.identity = identity
+    
 
 class BundleSource(StrEnum):
     CACHE = "CACHE"
     ACTION = "ACTION"
 
 class Effector:
+    """Subsystem for dereferencing RIDs."""
+    
     cache: Cache
-    resolver: "NetworkResolver | None"
-    processor: "ProcessorInterface | None"
-    action_context: "ActionContext | None"
+    resolver: NetworkResolver
+    kobj_queue: KobjQueue | None
+    action_context: ActionContext | None
     _action_table: dict[
         type[RID], 
         Callable[
-            ["ActionContext", RID], 
+            [ActionContext, RID], 
             Bundle | None
         ]
     ] = dict()
@@ -35,22 +45,16 @@ class Effector:
     def __init__(
         self, 
         cache: Cache,
+        resolver: NetworkResolver,
+        kobj_queue: KobjQueue,
+        identity: NodeIdentity
     ):
         self.cache = cache
-        self.resolver = None
-        self.processor = None
-        self.action_context = None
-        self._action_table = self.__class__._action_table.copy()
-        
-    def set_processor(self, processor: "ProcessorInterface"):
-        self.processor = processor
-        
-    def set_resolver(self, resolver: "NetworkResolver"):
         self.resolver = resolver
-        
-    def set_action_context(self, action_context: "ActionContext"):
-        self.action_context = action_context
-        
+        self.kobj_queue = kobj_queue
+        self.action_context = ActionContext(identity)
+        self._action_table = self.__class__._action_table.copy()
+    
     @classmethod
     def register_default_action(cls, rid_type: RIDType):
         def decorator(func: Callable) -> Callable:
@@ -59,6 +63,16 @@ class Effector:
         return decorator
         
     def register_action(self, rid_type: RIDType):
+        """Registers a new dereference action for an RID type.
+        
+        Example:
+            This function should be used as a decorator on an action function::
+            
+                @node.register_action(KoiNetNode)
+                def deref_koi_net_node(ctx: ActionContext, rid: KoiNetNode):
+                    # return a Bundle or None
+                    return
+        """
         def decorator(func: Callable) -> Callable:
             self._action_table[rid_type] = func
             return func
@@ -68,18 +82,18 @@ class Effector:
         bundle = self.cache.read(rid)
         
         if bundle:
-            logger.debug("Cache hit")
+            log.debug("Cache hit")
             return bundle, BundleSource.CACHE
         else:
-            logger.debug("Cache miss")
+            log.debug("Cache miss")
             return None
                     
     def _try_action(self, rid: RID) -> tuple[Bundle, BundleSource] | None:
         if type(rid) not in self._action_table:
-            logger.debug("No action available")
+            log.debug("No action available")
             return None
         
-        logger.debug("Action available")
+        log.debug("Action available")
         func = self._action_table[type(rid)]
         bundle = func(
             ctx=self.action_context, 
@@ -87,10 +101,10 @@ class Effector:
         )
         
         if bundle:
-            logger.debug("Action hit")
+            log.debug("Action hit")
             return bundle, BundleSource.ACTION
         else:
-            logger.debug("Action miss")
+            log.debug("Action miss")
             return None
 
         
@@ -98,10 +112,10 @@ class Effector:
         bundle, source = self.resolver.fetch_remote_bundle(rid)
         
         if bundle:
-            logger.debug("Network hit")
+            log.debug("Network hit")
             return bundle, source
         else:
-            logger.debug("Network miss")
+            log.debug("Network miss")
             return None
         
     
@@ -112,7 +126,20 @@ class Effector:
         use_network: bool = False,
         handle_result: bool = True
     ) -> Bundle | None:
-        logger.debug(f"Dereferencing {rid!r}")
+        """Dereferences an RID.
+        
+        Attempts to dereference an RID by (in order) reading the cache, 
+        calling a bound action, or fetching from other nodes in the 
+        newtork.
+        
+        Args:
+            rid: RID to dereference
+            refresh_cache: skips cache read when `True` 
+            use_network: enables fetching from other nodes when `True`
+            handle_result: handles resulting bundle with knowledge pipeline when `True`
+        """
+        
+        log.debug(f"Dereferencing {rid!r}")
         
         bundle, source = (
             # if `refresh_cache`, skip try cache
@@ -128,7 +155,7 @@ class Effector:
             and bundle is not None 
             and source != BundleSource.CACHE
         ):            
-            self.processor.handle(
+            self.kobj_queue.put_kobj(
                 bundle=bundle, 
                 source=source if type(source) is KoiNetNode else None
             )

koi_net/handshaker.py

@@ -0,0 +1,39 @@
+import structlog
+from rid_lib.ext import Cache
+from rid_lib.types import KoiNetNode
+from koi_net.identity import NodeIdentity
+from koi_net.network.event_queue import EventQueue
+from .protocol.event import Event, EventType
+
+log = structlog.stdlib.get_logger()
+
+
+class Handshaker:
+    def __init__(
+        self, 
+        cache: Cache, 
+        identity: NodeIdentity, 
+        event_queue: EventQueue
+    ):
+        self.cache = cache
+        self.identity = identity
+        self.event_queue = event_queue
+        
+    def handshake_with(self, target: KoiNetNode):
+        """Initiates a handshake with target node.
+        Pushes successive `FORGET` and `NEW` events to target node to
+        reset the target's cache in case it already knew this node. 
+        """
+        log.debug(f"Initiating handshake with {target}")
+        self.event_queue.push_event_to(
+            Event.from_rid(
+                event_type=EventType.FORGET, 
+                rid=self.identity.rid),
+            target=target
+        )
+        self.event_queue.push_event_to(
+            event=Event.from_bundle(
+                event_type=EventType.NEW, 
+                bundle=self.cache.read(self.identity.rid)),
+            target=target
+        )

koi_net/identity.py

@@ -1,10 +1,9 @@
-import logging
+import structlog
 from rid_lib.types.koi_net_node import KoiNetNode
 from .config import NodeConfig
 from .protocol.node import NodeProfile
 
-
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
     
 class NodeIdentity:

koi_net/interfaces/entrypoint.py

@@ -0,0 +1,5 @@
+from typing import Protocol
+
+
+class EntryPoint(Protocol):
+    def run(self): ...

koi_net/interfaces/worker.py

@@ -0,0 +1,17 @@
+import threading
+
+
+class End:
+    """Class for a sentinel value by knowledge handlers."""
+    pass
+
+STOP_WORKER = End()
+
+class ThreadWorker:
+    thread: threading.Thread
+    
+    def __init__(self):
+        self.thread = threading.Thread(target=self.run)
+        
+    def run(self):
+        ...

koi_net/lifecycle.py

@@ -1,104 +1,141 @@
-import logging
+import structlog
 from contextlib import contextmanager, asynccontextmanager
 
+from rid_lib.ext import Bundle, Cache
 from rid_lib.types import KoiNetNode
 
-from .actor import Actor
-from .effector import Effector
+from .handshaker import Handshaker
+from .network.request_handler import RequestHandler
+from .processor.kobj_worker import KnowledgeProcessingWorker
+from .network.event_queue import EventQueue
+from .processor.event_worker import EventProcessingWorker
+from .protocol.api_models import ErrorResponse
+from .interfaces.worker import STOP_WORKER
 from .config import NodeConfig
-from .processor.interface import ProcessorInterface
+from .processor.kobj_queue import KobjQueue
 from .network.graph import NetworkGraph
 from .identity import NodeIdentity
 
-logger = logging.getLogger(__name__)
+log = structlog.stdlib.get_logger()
 
 
 class NodeLifecycle:
+    """Manages node startup and shutdown processes."""
+    
     config: NodeConfig
+    identity: NodeIdentity
     graph: NetworkGraph
-    processor: ProcessorInterface
-    effector: Effector
-    actor: Actor
+    kobj_queue: KobjQueue
+    kobj_worker: KnowledgeProcessingWorker
+    event_queue: EventQueue
+    event_worker: EventProcessingWorker
+    cache: Cache
+    handshaker: Handshaker
+    request_handler: RequestHandler
     
     def __init__(
         self,
         config: NodeConfig,
         identity: NodeIdentity,
         graph: NetworkGraph,
-        processor: ProcessorInterface,
-        effector: Effector,
-        actor: Actor,
-        use_kobj_processor_thread: bool
+        kobj_queue: KobjQueue,
+        kobj_worker: KnowledgeProcessingWorker,
+        event_queue: EventQueue,
+        event_worker: EventProcessingWorker,
+        cache: Cache,
+        handshaker: Handshaker,
+        request_handler: RequestHandler
     ):
         self.config = config
         self.identity = identity
         self.graph = graph
-        self.processor = processor
-        self.effector = effector
-        self.actor = actor
-        self.use_kobj_processor_thread = use_kobj_processor_thread
+        self.kobj_queue = kobj_queue
+        self.kobj_worker = kobj_worker
+        self.event_queue = event_queue
+        self.event_worker = event_worker
+        self.cache = cache
+        self.handshaker = handshaker
+        self.request_handler = request_handler
         
     @contextmanager
     def run(self):
+        """Synchronous context manager for node startup and shutdown."""
         try:
-            logger.info("Starting node lifecycle...")
+            log.info("Starting node lifecycle...")
             self.start()
             yield
         except KeyboardInterrupt:
-            logger.info("Keyboard interrupt!")
+            log.info("Keyboard interrupt!")
         finally:
-            logger.info("Stopping node lifecycle...")
+            log.info("Stopping node lifecycle...")
             self.stop()
 
     @asynccontextmanager
     async def async_run(self):
+        """Asynchronous context manager for node startup and shutdown."""
         try:
-            logger.info("Starting async node lifecycle...")
+            log.info("Starting async node lifecycle...")
             self.start()
             yield
         except KeyboardInterrupt:
-            logger.info("Keyboard interrupt!")
+            log.info("Keyboard interrupt!")
         finally:
-            logger.info("Stopping async node lifecycle...")
+            log.info("Stopping async node lifecycle...")
             self.stop()
     
     def start(self):
-        """Starts a node, call this method first.
+        """Starts a node.
         
-        Starts the processor thread (if enabled). Loads event queues into memory. Generates network graph from nodes and edges in cache. Processes any state changes of node bundle. Initiates handshake with first contact (if provided) if node doesn't have any neighbors.
+        Starts the processor thread (if enabled). Generates network 
+        graph from nodes and edges in cache. Processes any state changes 
+        of node bundle. Initiates handshake with first contact if node 
+        doesn't have any neighbors. Catches up with coordinator state.
         """
-        if self.use_kobj_processor_thread:
-            logger.info("Starting processor worker thread")
-            self.processor.worker_thread.start()
+        log.info("Starting processor worker thread")
         
+        self.kobj_worker.thread.start()
+        self.event_worker.thread.start()
         self.graph.generate()
         
         # refresh to reflect changes (if any) in config.yaml
-        self.effector.deref(self.identity.rid, refresh_cache=True)
         
-        logger.debug("Waiting for kobj queue to empty")
-        if self.use_kobj_processor_thread:
-            self.processor.kobj_queue.join()
-        else:
-            self.processor.flush_kobj_queue()
-        logger.debug("Done")
-    
-        if not self.graph.get_neighbors() and self.config.koi_net.first_contact.rid:
-            logger.debug(f"I don't have any neighbors, reaching out to first contact {self.config.koi_net.first_contact.rid!r}")
-            
-            self.actor.handshake_with(self.config.koi_net.first_contact.rid)
+        self.kobj_queue.put_kobj(bundle=Bundle.generate(
+            rid=self.identity.rid,
+            contents=self.identity.profile.model_dump()
+        ))
+        
+        log.debug("Waiting for kobj queue to empty")
+        self.kobj_queue.q.join()
         
-        for coordinator in self.actor.identify_coordinators():
-            self.actor.catch_up_with(coordinator, rid_types=[KoiNetNode])
+        coordinators = self.graph.get_neighbors(direction="in", allowed_type=KoiNetNode)
+        
+        if len(coordinators) > 0:
+            for coordinator in coordinators:
+                payload = self.request_handler.fetch_manifests(
+                    node=coordinator,
+                    rid_types=[KoiNetNode]
+                )
+                if type(payload) is ErrorResponse:
+                    continue
+                
+                for manifest in payload.manifests:
+                    self.kobj_queue.put_kobj(
+                        manifest=manifest,
+                        source=coordinator
+                    )
+                    
+        elif self.config.koi_net.first_contact.rid:
+            log.debug(f"I don't have any edges with coordinators, reaching out to first contact {self.config.koi_net.first_contact.rid!r}")
+            
+            self.handshaker.handshake_with(self.config.koi_net.first_contact.rid)
         
 
     def stop(self):
-        """Stops a node, call this method last.
+        """Stops a node.
         
-        Finishes processing knowledge object queue. Saves event queues to storage.
+        Finishes processing knowledge object queue.
         """        
-        if self.use_kobj_processor_thread:
-            logger.info(f"Waiting for kobj queue to empty ({self.processor.kobj_queue.unfinished_tasks} tasks remaining)")
-            self.processor.kobj_queue.join()
-        else:
-            self.processor.flush_kobj_queue()
+        log.info(f"Waiting for kobj queue to empty ({self.kobj_queue.q.unfinished_tasks} tasks remaining)")
+        
+        self.kobj_queue.q.put(STOP_WORKER)
+        self.event_queue.q.put(STOP_WORKER)

koi_net/logger.py

@@ -0,0 +1,176 @@
+from datetime import datetime
+import logging
+from logging.handlers import RotatingFileHandler
+import colorama
+import structlog
+import sys
+# from sentry_sdk import logger as sentry_logger
+
+
+def my_processor(_, __, event: dict):
+    # print(_, __, event)
+    event["path"] = event["module"] + "." + event["func_name"]
+    return event
+
+# def sentry_processor(_, method, event: dict):
+#     print(event)
+#     if method == "critical":
+#         sentry_logger.fatal(
+#             event["event"],
+#             attributes=event
+#         )
+#     elif method == "info":
+#         sentry_logger.info(
+#             event["event"],
+#             attributes=event
+#         )
+#     elif method == "debug":
+#         sentry_logger.debug(
+#             event["event"],
+#             attributes=event
+#         )
+#     return event
+
+console_renderer = structlog.dev.ConsoleRenderer(
+    columns=[
+        # Render the timestamp without the key name in yellow.
+        structlog.dev.Column(
+            "timestamp",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=None,
+                value_style=colorama.Style.DIM,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=lambda t: datetime.fromisoformat(t).strftime("%Y-%m-%d %H:%M:%S"),
+            ),
+        ),
+        structlog.dev.Column(
+            "level",
+            structlog.dev.LogLevelColumnFormatter(
+                level_styles={
+                    level: colorama.Style.BRIGHT + color
+                    for level, color in {
+                        "critical": colorama.Fore.RED,
+                        "exception": colorama.Fore.RED,
+                        "error": colorama.Fore.RED,
+                        "warn": colorama.Fore.YELLOW,
+                        "warning": colorama.Fore.YELLOW,
+                        "info": colorama.Fore.GREEN,
+                        "debug": colorama.Fore.GREEN,
+                        "notset": colorama.Back.RED,
+                    }.items()
+                },
+                reset_style=colorama.Style.RESET_ALL,
+                width=9
+            )
+        ),
+        # Render the event without the key name in bright magenta.
+        
+        # Default formatter for all keys not explicitly mentioned. The key is
+        # cyan, the value is green.
+        structlog.dev.Column(
+            "path",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=None,
+                value_style=colorama.Fore.MAGENTA,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=str,
+                width=30
+            ),
+        ),
+        # structlog.dev.Column(
+        #     "func_name",
+        #     structlog.dev.KeyValueColumnFormatter(
+        #         key_style=None,
+        #         value_style=colorama.Fore.MAGENTA,
+        #         reset_style=colorama.Style.RESET_ALL,
+        #         value_repr=str,
+        #         prefix="(",
+        #         postfix=")",
+        #         width=15
+        #     ),
+        # ),
+        structlog.dev.Column(
+            "event",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=None,
+                value_style=colorama.Fore.WHITE,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=str,
+                width=30
+            ),
+        ),
+        structlog.dev.Column(
+            "",
+            structlog.dev.KeyValueColumnFormatter(
+                key_style=colorama.Fore.BLUE,
+                value_style=colorama.Fore.GREEN,
+                reset_style=colorama.Style.RESET_ALL,
+                value_repr=str,
+            ),
+        )
+    ]
+)
+
+structlog.configure(
+    processors=[
+        # If log level is too low, abort pipeline and throw away log entry.
+        structlog.stdlib.filter_by_level,
+        # Add the name of the logger to event dict.
+        structlog.stdlib.add_logger_name,
+        # Add log level to event dict.
+        structlog.stdlib.add_log_level,
+        # Perform %-style formatting.
+        structlog.stdlib.PositionalArgumentsFormatter(),
+        # Add a timestamp in ISO 8601 format.
+        structlog.processors.TimeStamper(fmt="iso"),
+        # If the "stack_info" key in the event dict is true, remove it and
+        # render the current stack trace in the "stack" key.
+        structlog.processors.StackInfoRenderer(),
+        # If the "exc_info" key in the event dict is either true or a
+        # sys.exc_info() tuple, remove "exc_info" and render the exception
+        # with traceback into the "exception" key.
+        # structlog.processors.format_exc_info,
+        # If some value is in bytes, decode it to a Unicode str.
+        structlog.processors.UnicodeDecoder(),
+        # Add callsite parameters.
+        structlog.processors.CallsiteParameterAdder(
+            {
+                structlog.processors.CallsiteParameter.MODULE,
+                structlog.processors.CallsiteParameter.FUNC_NAME,
+                # structlog.processors.CallsiteParameter.LINENO,
+            }
+        ),
+        my_processor,
+        # Render the final event dict as JSON.
+        # sentry_processor,
+        console_renderer
+        # structlog.processors.JSONRenderer()
+        
+    ],
+    # `wrapper_class` is the bound logger that you get back from
+    # get_logger(). This one imitates the API of `logging.Logger`.
+    wrapper_class=structlog.stdlib.BoundLogger,
+    # `logger_factory` is used to create wrapped loggers that are used for
+    # OUTPUT. This one returns a `logging.Logger`. The final value (a JSON
+    # string) from the final processor (`JSONRenderer`) will be passed to
+    # the method of the same name as that you've called on the bound logger.
+    logger_factory=structlog.stdlib.LoggerFactory(),
+    # Effectively freeze configuration after creating the first bound
+    # logger.
+    cache_logger_on_first_use=True,
+)
+
+file_handler = RotatingFileHandler(
+    filename="app.log",
+    maxBytes=10 * 1024 * 1024,
+    backupCount=5,
+    encoding="utf-8"
+)
+
+logging.basicConfig(
+    format="%(message)s",
+    stream=sys.stdout,
+    level=logging.INFO,
+)
+
+# log = structlog.stdlib.get_logger()