python-corekit 0.1.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

corekit/decorators/exception_handling.py

@@ -1,12 +1,27 @@
 import functools
+import inspect
 import logging
 from typing import Any, Callable
 
 from corekit.utils import raise_exc, void
+from corekit.utils.coercion import safe_tuple
 
 logger = logging.getLogger(__name__)
 
 
+def _log_text(ex: Exception) -> str:
+    """
+    Prefer a backend log line when the exception provides one.
+
+    Public exceptions stringify to the user-facing copy. Logging that drops
+    the internal message.
+    """
+    for_log = getattr(ex, "for_log", None)
+    if callable(for_log):
+        return for_log()
+    return str(ex)
+
+
 def exception_handler(
     ignore: list[type[Exception]] | None = None,
     callback: Callable = raise_exc,
@@ -15,29 +30,43 @@ def exception_handler(
     """
     A decorator to handle exceptions in a class method or function.
 
+    Works on both sync and async functions. An async function keeps an async
+    wrapper, so awaiting it still works.
+
     Args:
         ignore: exception classes to suppress. These do not propagate.
         callback: called when an exception is raised that is not ignored.
         ignore_callback: called when a suppressed exception is raised.
     """
-    exceptions_to_ignore = tuple(ignore) if ignore else tuple()
+    exceptions_to_ignore = safe_tuple(ignore)
 
     def decorator(func):
+        def handle(ex: Exception, args: tuple, kwargs: dict) -> Any:
+            text = _log_text(ex)
+            if isinstance(ex, exceptions_to_ignore):
+                logger.warning(f"Suppressed exception in {func.__name__}: {text}")
+                logger.info(f"Invoking callback function: {ignore_callback.__name__}")
+                return ignore_callback(ex, *args, **kwargs)
+
+            logger.error(f"Exception in {func.__name__}: {text}", exc_info=True)
+            return callback(ex, *args, **kwargs)
+
         @functools.wraps(func)
         def wrapper(*args: Any, **kwargs: Any) -> Any:
             try:
                 return func(*args, **kwargs)
             except Exception as ex:
-                # If in ignore list, suppress and invoke the ignore_callback function
-                if isinstance(ex, exceptions_to_ignore):
-                    logger.warning(f"Suppressed exception in {func.__name__}: {ex}")
-                    logger.info(f"Invoking callback function: {ignore_callback.__name__}")
-                    return ignore_callback(ex, *args, **kwargs)
+                return handle(ex, args, kwargs)
 
-                # Log the exception and invoke the callback function, returning its return value (if any)
-                logger.error(f"Exception in {func.__name__}: {ex}", exc_info=True)
-                return callback(ex, *args, **kwargs)
+        @functools.wraps(func)
+        async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+            try:
+                return await func(*args, **kwargs)
+            except Exception as ex:
+                return handle(ex, args, kwargs)
 
+        if inspect.iscoroutinefunction(func):
+            return async_wrapper
         return wrapper
 
     return decorator

corekit/docker/watchdog.py

@@ -42,11 +42,12 @@ class Watchdog(Benchmarkable):
     background daemon, and one unreachable container should not stop it.
     """
 
-    def __init__(self, docker_host: str | None = None, enforce_label: bool = False) -> None:
+    def __init__(self, docker_host: str | None = None, enforce_label: bool = True) -> None:
         """
         :param docker_host: Docker endpoint. Defaults to the local socket.
         :param enforce_label: restrict actions to containers carrying the
-            managed label.
+            managed label. On by default; turn it off only when this watchdog
+            is meant to control every container on the host.
         """
         super().__init__()
         self.docker_host = docker_host or DEFAULT_DOCKER_HOST
@@ -96,27 +97,29 @@ class Watchdog(Benchmarkable):
             return container.labels.get(WATCHDOG_LABEL) == WATCHDOG_MANAGED
         return True
 
-    def _start_container(self, container: Container) -> None:
+    def _start_container(self, container: Container) -> bool:
         try:
             container.reload()
             if container.status != "running" and self._label_validator(container):
                 container.start()
                 self.info(f"Started container: {container.name}")
-            else:
-                self.info(f"Container {container.name!r} is already running or not managed.")
+                return True
+            self.info(f"Container {container.name!r} is already running or not managed.")
         except APIError as exc:
             self.error(f"Failed to start container {container.name}: {exc}")
+        return False
 
-    def _stop_container(self, container: Container) -> None:
+    def _stop_container(self, container: Container) -> bool:
         try:
             container.reload()
             if container.status == "running" and self._label_validator(container):
                 container.stop()
                 self.info(f"Stopped container: {container.name}")
-            else:
-                self.info(f"Container {container.name!r} is not running or not managed.")
+                return True
+            self.info(f"Container {container.name!r} is not running or not managed.")
         except APIError as exc:
             self.error(f"Failed to stop container {container.name}: {exc}")
+        return False
 
     def _by_name(self, name: str) -> Container | None:
         """
@@ -161,62 +164,78 @@ class Watchdog(Benchmarkable):
         for container in self._container_iterator(filter_func=self._get_label_filter(label, value), all=True):
             self._start_container(container)
 
-    def start_container_by_name(self, name: str) -> None:
+    def start_container_by_name(self, name: str) -> bool:
         """
-        Start one container by name.
+        Start one container by name. Returns whether it was started.
         """
         container = self._by_name(name)
-        if container is not None:
-            self._start_container(container)
+        if container is None:
+            return False
+        return self._start_container(container)
 
-    def stop_container_by_name(self, name: str) -> None:
+    def stop_container_by_name(self, name: str) -> bool:
         """
-        Stop one container by name.
+        Stop one container by name. Returns whether it was stopped.
         """
         container = self._by_name(name)
-        if container is not None:
-            self._stop_container(container)
+        if container is None:
+            return False
+        return self._stop_container(container)
 
-    def restart_container_by_name(self, name: str) -> None:
+    def restart_container_by_name(self, name: str) -> bool:
         """
-        Stop and then start one container by name.
+        Restart one container by name, honoring the managed label.
+
+        Returns whether the container was restarted. A missing or unmanaged
+        container is not touched.
         """
         container = self._by_name(name)
         if container is None:
-            return
-        self._stop_container(container)
-        self._start_container(container)
+            return False
+        try:
+            container.reload()
+            if not self._label_validator(container):
+                self.info(f"Container {container.name!r} is not managed.")
+                return False
+            container.restart()
+            self.info(f"Restarted container: {container.name}")
+            return True
+        except APIError as exc:
+            self.error(f"Failed to restart container {name}: {exc}")
+            return False
 
-    def pause_container_by_name(self, name: str) -> None:
+    def pause_container_by_name(self, name: str) -> bool:
         """
-        Pause one running container by name.
+        Pause one running container by name. Returns whether it was paused.
         """
         container = self._by_name(name)
         if container is None:
-            return
+            return False
         try:
             container.reload()
             if container.status == "running" and self._label_validator(container):
                 container.pause()
                 self.info(f"Paused container: {container.name}")
-            else:
-                self.info(f"Container {container.name!r} is not running or not managed.")
+                return True
+            self.info(f"Container {container.name!r} is not running or not managed.")
         except APIError as exc:
             self.error(f"Failed to pause container {name}: {exc}")
+        return False
 
-    def unpause_container_by_name(self, name: str) -> None:
+    def unpause_container_by_name(self, name: str) -> bool:
         """
-        Unpause one paused container by name.
+        Unpause one paused container by name. Returns whether it was unpaused.
         """
         container = self._by_name(name)
         if container is None:
-            return
+            return False
         try:
             container.reload()
             if container.status == "paused" and self._label_validator(container):
                 container.unpause()
                 self.info(f"Unpaused container: {container.name}")
-            else:
-                self.info(f"Container {container.name!r} is not paused or not managed.")
+                return True
+            self.info(f"Container {container.name!r} is not paused or not managed.")
         except APIError as exc:
             self.error(f"Failed to unpause container {name}: {exc}")
+        return False

corekit/etl/__init__.py

@@ -18,7 +18,7 @@ The run loop streams, so memory use is bounded by the batch size rather than by
 how much data the source holds.
 """
 
-from corekit.etl.connection import BaseConnection, ConnectionDetails
+from corekit.etl.connection import BaseConnection, ConnectionDetails, ETLSource
 from corekit.etl.extract.extractor import BaseETLExtractor
 from corekit.etl.extract.schemas import BaseExtractedItemModel, ExtractedItem
 from corekit.etl.load.loader import BaseETLLoader
@@ -30,6 +30,7 @@ from corekit.etl.transform.transformer import BaseETLTransformer
 
 __all__ = [
     "BaseConnection",
+    "ETLSource",
     "BaseETLExtractor",
     "BaseETLLoader",
     "BaseETLOrchestrator",

corekit/etl/connection.py

@@ -1,44 +1,46 @@
-from enum import Enum
-from typing import NamedTuple
-
-
-class ConnectionDetails(NamedTuple):
-    name: str
-    description: str
-
-
-class BaseConnection(Enum):
-    """
-    Base enum for ETL Connections. All ETL connections must inherit from this class.
-
-    Example:
-        class MyConnectionEnum(BaseConnection):
-            MY_FIRST_CONNECTION = ConnectionDetails(
-                name="my_first_connection",
-                description="My First Connection"
-            )
-            ANOTHER_CONNECTION = ConnectionDetails(
-                name="another_connection",
-                description="Another Connection"
-            )
-    """
-
-    @classmethod
-    def from_name(cls, name: str) -> "BaseConnection":
-        # TODO: Make sure this is efficient
-        for connection in cls:
-            if connection.value.name == name:
-                return connection
-        raise ValueError(f"Connection {name} not found")
-
-    def get_name(self) -> str:
-        """
-        Helper method for fetching the name of the connection
-        """
-        return self.value.name
-
-    def get_description(self) -> str:
-        """
-        Helper method for fetching the name of the connection
-        """
-        return self.value.description
+from enum import Enum
+from typing import NamedTuple
+
+
+class ConnectionDetails(NamedTuple):
+    name: str
+    description: str
+
+
+class ETLSource(Enum):
+    """
+    A named source label for an extracted item.
+
+    This is not a connection. It does not open a socket, a session, or a
+    ``Connectable``. It only names where a row came from so a transformer can
+    tell sources apart.
+
+    Example:
+        class UserSource(ETLSource):
+            USERS = ConnectionDetails(name="users", description="User records")
+    """
+
+    @classmethod
+    def from_name(cls, name: str) -> "ETLSource":
+        # TODO: Make sure this is efficient
+        for connection in cls:
+            if connection.value.name == name:
+                return connection
+        raise ValueError(f"Connection {name} not found")
+
+    def get_name(self) -> str:
+        """
+        Helper method for fetching the name of the connection
+        """
+        return self.value.name
+
+    def get_description(self) -> str:
+        """
+        Helper method for fetching the name of the connection
+        """
+        return self.value.description
+
+
+# The old name implied a second connection system. Keep it so existing imports
+# still resolve; new code should use ETLSource.
+BaseConnection = ETLSource

corekit/etl/extract/extractor.py

@@ -1,30 +1,23 @@
 from abc import ABC, abstractmethod
 from typing import Any, AsyncIterable, Callable
 
-from corekit.etl.connection import BaseConnection
 from corekit.etl.extract.schemas import ExtractedItem
-from corekit.http.client import BaseApiClient
 from corekit.observability.loggable import Loggable
 
 
 class BaseETLExtractor(Loggable, ABC):
     """
-    Base class for ETL extractors.
+    Yields items from a source.
+
+    The runner constructs this with no arguments, so a subclass that needs a
+    client or other dependency should default it or be passed in as an
+    instance. ``client`` is optional and unused by the base class.
     """
 
-    def __init__(self, client: BaseApiClient) -> None:
+    def __init__(self, client: Any | None = None) -> None:
         super().__init__()
         self.client = client
 
-    @property
-    @abstractmethod
-    def connection(self) -> BaseConnection:
-        """
-        Abstract property that must be implemented by child classes.
-        This property should return the integration object that the extractor is associated with
-        """
-        raise NotImplementedError
-
     @abstractmethod
     def _extraction_methods(self) -> list[Callable[[Any], Any]]:
         """

corekit/etl/orchestrator.py

@@ -159,12 +159,19 @@ class BaseETLOrchestrator(Loggable):
         transformer: BaseETLTransformer | None = None,
         loader: BaseETLLoader | None = None,
         batch_size: int | None = None,
+        on_item_error: str = "raise",
     ) -> None:
         """
         Build a pipeline, optionally overriding any stage with an instance.
+
+        ``on_item_error`` is ``raise`` (stop, leaving earlier batches written)
+        or ``skip`` (log the item and continue). Transform is synchronous.
         """
+        if on_item_error not in ("raise", "skip"):
+            raise ValueError("on_item_error must be 'raise' or 'skip'")
         super().__init__()
         self.batch_size = batch_size if batch_size is not None else type(self).batch_size
+        self.on_item_error = on_item_error
         self._extractor = extractor
         self._transformer = transformer
         self._loader = loader
@@ -192,10 +199,20 @@ class BaseETLOrchestrator(Loggable):
         loader = self._build(self._loader, "loader")
 
         count = 0
+        skipped = 0
         async for extracted_item in extractor.extract():
-            loader.add_item(transformer.transform(extracted_item))
+            try:
+                # transform is synchronous; do not await it.
+                transformed = transformer.transform(extracted_item)
+            except Exception:
+                if self.on_item_error == "raise":
+                    raise
+                skipped += 1
+                self.exception(f"{type(self).__name__} skipped an item")
+                continue
+            loader.add_item(transformed)
             await loader.load(min_ops=self.batch_size)
             count += 1
 
         await loader.flush()
-        self.info(f"{type(self).__name__} processed {count} items")
+        self.info(f"{type(self).__name__} processed {count} items, skipped {skipped}")

corekit/etl/schemas.py

@@ -1,6 +1,6 @@
 from pydantic import BaseModel
 
-from corekit.etl.connection import BaseConnection
+from corekit.etl.connection import ETLSource
 from corekit.schemas.types import UnknownDict
 
 
@@ -9,7 +9,7 @@ class BaseItem(BaseModel):
     Base model for all ETL items
     """
 
-    connection: BaseConnection
+    connection: ETLSource
 
 
 class RawAPIResponse(BaseItem):

corekit/etl/transform/transformer.py

@@ -18,7 +18,10 @@ class BaseETLTransformer(Loggable, ABC):
 
     def transform(self, item: ExtractedItem) -> TransformedItem:
         """
-        Primary method to run the transformer.
+        Turn one extracted item into a loadable one.
+
+        Synchronous on purpose. The orchestrator calls this directly and does
+        not await it.
         """
         data_type = item.get_data_type()
         mapping = self._extracted_item_type_to_method

corekit/events/publisher.py

@@ -11,7 +11,7 @@ constant::
     EventPublisher.for_resource("minecraft", "server", "survival")
     # -> channel "minecraft:server:survival"
 
-Requires the ``redis`` extra.
+Redis is a hard dependency of corekit; no extra install is required.
 """
 
 from typing import Any

corekit/events/reader.py

@@ -8,7 +8,9 @@ disappears -- so no consumer has to repeat it.
 Everything here uses Redis's async client. The synchronous client's
 ``get_message`` blocks the thread it is called on, which in an async server is
 the event loop, stalling every other request in the process while one client
-waits for an event that may never come.
+waits for an event that may never come. The async client's ``get_message``
+returns immediately unless it is given a timeout, so the wait has to be
+asked for there rather than wrapped around a call that already finished.
 """
 
 import asyncio
@@ -21,9 +23,6 @@ from corekit.observability.loggable import Loggable
 
 __all__ = ["RedisChannelReader"]
 
-# How long to yield to the event loop when the socket has nothing ready.
-IDLE_SLEEP_SECONDS = 0.01
-
 
 class RedisChannelReader(Loggable):
     """
@@ -88,27 +87,33 @@ class RedisChannelReader(Loggable):
         """
         Return the next decoded payload.
 
-        Returns ``None`` when nothing arrived before ``timeout``, so a caller
-        can act on the silence -- sending a keepalive, say -- rather than
-        blocking forever.
+        Returns ``None`` only when nothing arrived before ``timeout``. The wait
+        happens inside Redis ``get_message``: calling it with no timeout
+        returns immediately, and wrapping that in ``asyncio.wait_for`` never
+        blocks. ``timeout=None`` waits until a payload arrives.
         """
-        try:
-            message = await asyncio.wait_for(
-                self._pubsub.get_message(ignore_subscribe_messages=True),
-                timeout=timeout,
+        loop = asyncio.get_running_loop()
+        deadline = None if timeout is None else loop.time() + timeout
+        while True:
+            remaining = None if deadline is None else deadline - loop.time()
+            if remaining is not None and remaining <= 0:
+                return None
+
+            message = await self._pubsub.get_message(
+                ignore_subscribe_messages=True,
+                timeout=remaining,
             )
-        except asyncio.TimeoutError:
-            return None
-
-        if message is None:
-            # Nothing ready. Yield rather than spin.
-            await asyncio.sleep(IDLE_SLEEP_SECONDS)
-            return None
+            if message is None:
+                # None before the deadline is an ignored subscribe ack, not
+                # silence. Only the expired wait is a keepalive.
+                if deadline is None or loop.time() < deadline:
+                    continue
+                return None
 
-        if message.get(PubSubField.TYPE.value) not in (None, PubSubMessageType.MESSAGE.value):
-            return None
+            if message.get(PubSubField.TYPE.value) not in (None, PubSubMessageType.MESSAGE.value):
+                continue
 
-        return self._decode(message.get(PubSubField.DATA.value))
+            return self._decode(message.get(PubSubField.DATA.value))
 
     def _decode(self, raw: Any) -> Any:
         """

corekit/events/sse.py

@@ -93,7 +93,10 @@ class SSEStream(Loggable):
             while True:
                 payload = await self._reader.next_payload(timeout=self.keepalive_interval)
                 if payload is None:
-                    yield SSEFrame.comment()
+                    # None means the wait expired. A disabled keepalive never
+                    # asks for a timeout, so it must not emit a comment either.
+                    if self.keepalive_interval is not None:
+                        yield SSEFrame.comment()
                     continue
                 yield self._frame_for(payload)
 

corekit/events/websocket.py

@@ -13,6 +13,7 @@ from typing import Any
 from corekit.events.enum import MessageField
 from corekit.events.reader import RedisChannelReader
 from corekit.observability.loggable import Loggable
+from corekit.utils.coercion import safe_dict, safe_list
 
 __all__ = ["WebSocketBridge"]
 
@@ -44,7 +45,7 @@ class WebSocketBridge(Loggable):
         super().__init__()
         self.channel = channel
         self.websocket = websocket
-        self.terminal_statuses = terminal_statuses or []
+        self.terminal_statuses = safe_list(terminal_statuses)
         self.timeout = timeout
         self._reader = RedisChannelReader(channel, url=url)
 
@@ -56,16 +57,23 @@ class WebSocketBridge(Loggable):
             return False
         if payload.get(MessageField.TYPE.value) != STATUS_UPDATE_EVENT:
             return False
-        data = payload.get(MessageField.DATA.value) or {}
+        data = safe_dict(payload.get(MessageField.DATA.value))
         return data.get(MessageField.STATUS.value) in self.terminal_statuses
 
     def _expired(self, started: float) -> bool:
         """
         Whether the bridge has outlived its timeout.
         """
+        remaining = self._remaining(started)
+        return remaining is not None and remaining <= 0
+
+    def _remaining(self, started: float) -> float | None:
+        """
+        Seconds left before the bridge times out, or None if it never does.
+        """
         if self.timeout is None:
-            return False
-        return asyncio.get_event_loop().time() - started > self.timeout
+            return None
+        return self.timeout - (asyncio.get_running_loop().time() - started)
 
     async def run(self) -> None:
         """
@@ -74,21 +82,27 @@ class WebSocketBridge(Loggable):
         if not await self._reader.open():
             return
 
-        started = asyncio.get_event_loop().time()
+        started = asyncio.get_running_loop().time()
         try:
             while True:
-                payload = await self._reader.next_payload(timeout=self.timeout)
-
-                if payload is not None:
-                    await self.websocket.send_json(payload)
-                    if self._is_terminal(payload):
-                        self.info(f"Bridge on {self.channel} reached a terminal status")
-                        break
+                remaining = self._remaining(started)
+                if remaining is not None and remaining <= 0:
+                    self.warning(f"Bridge on {self.channel} timed out")
+                    break
 
-                if self._expired(started):
+                payload = await self._reader.next_payload(timeout=remaining)
+                if payload is None:
+                    # The wait expired. That is the timeout, not a reason to poll again.
+                    if remaining is None:
+                        continue
                     self.warning(f"Bridge on {self.channel} timed out")
                     break
 
+                await self.websocket.send_json(payload)
+                if self._is_terminal(payload):
+                    self.info(f"Bridge on {self.channel} reached a terminal status")
+                    break
+
         except asyncio.CancelledError:
             raise
         except Exception as exc:

corekit/exceptions/__init__.py

@@ -0,0 +1,33 @@
+"""
+Exception base classes.
+
+``CoreException`` is abstract. Raise ``InternalCoreException`` for backend-only
+failures, or ``PublicCoreException`` when there is copy that is safe to show
+a user. ``CoreHTTPException`` is the FastAPI-facing public exception.
+"""
+
+from corekit.exceptions.base import (
+    CoreException,
+    CoreHTTPException,
+    ExponentialBackoffTimeoutException,
+    InternalCoreException,
+    NonRetryableCoreHTTPException,
+    PublicCoreException,
+    RetryableCoreHTTPException,
+)
+from corekit.exceptions.enum import Retryability
+from corekit.exceptions.types import ArbitraryCoreExceptionType, CoreExceptionType, CoreHTTPExceptionType
+
+__all__ = [
+    "ArbitraryCoreExceptionType",
+    "CoreException",
+    "CoreExceptionType",
+    "CoreHTTPException",
+    "CoreHTTPExceptionType",
+    "ExponentialBackoffTimeoutException",
+    "InternalCoreException",
+    "NonRetryableCoreHTTPException",
+    "PublicCoreException",
+    "Retryability",
+    "RetryableCoreHTTPException",
+]