python-corekit 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

corekit/etl/schemas.py

@@ -1,6 +1,6 @@
 from pydantic import BaseModel
 
-from corekit.etl.connection import BaseConnection
+from corekit.etl.connection import ETLSource
 from corekit.schemas.types import UnknownDict
 
 
@@ -9,7 +9,7 @@ class BaseItem(BaseModel):
     Base model for all ETL items
     """
 
-    connection: BaseConnection
+    connection: ETLSource
 
 
 class RawAPIResponse(BaseItem):

corekit/etl/transform/transformer.py

@@ -18,7 +18,10 @@ class BaseETLTransformer(Loggable, ABC):
 
     def transform(self, item: ExtractedItem) -> TransformedItem:
         """
-        Primary method to run the transformer.
+        Turn one extracted item into a loadable one.
+
+        Synchronous on purpose. The orchestrator calls this directly and does
+        not await it.
         """
         data_type = item.get_data_type()
         mapping = self._extracted_item_type_to_method

corekit/events/publisher.py

@@ -11,7 +11,7 @@ constant::
     EventPublisher.for_resource("minecraft", "server", "survival")
     # -> channel "minecraft:server:survival"
 
-Requires the ``redis`` extra.
+Redis is a hard dependency of corekit; no extra install is required.
 """
 
 from typing import Any

corekit/events/reader.py

@@ -8,7 +8,9 @@ disappears -- so no consumer has to repeat it.
 Everything here uses Redis's async client. The synchronous client's
 ``get_message`` blocks the thread it is called on, which in an async server is
 the event loop, stalling every other request in the process while one client
-waits for an event that may never come.
+waits for an event that may never come. The async client's ``get_message``
+returns immediately unless it is given a timeout, so the wait has to be
+asked for there rather than wrapped around a call that already finished.
 """
 
 import asyncio
@@ -21,9 +23,6 @@ from corekit.observability.loggable import Loggable
 
 __all__ = ["RedisChannelReader"]
 
-# How long to yield to the event loop when the socket has nothing ready.
-IDLE_SLEEP_SECONDS = 0.01
-
 
 class RedisChannelReader(Loggable):
     """
@@ -88,27 +87,33 @@ class RedisChannelReader(Loggable):
         """
         Return the next decoded payload.
 
-        Returns ``None`` when nothing arrived before ``timeout``, so a caller
-        can act on the silence -- sending a keepalive, say -- rather than
-        blocking forever.
+        Returns ``None`` only when nothing arrived before ``timeout``. The wait
+        happens inside Redis ``get_message``: calling it with no timeout
+        returns immediately, and wrapping that in ``asyncio.wait_for`` never
+        blocks. ``timeout=None`` waits until a payload arrives.
         """
-        try:
-            message = await asyncio.wait_for(
-                self._pubsub.get_message(ignore_subscribe_messages=True),
-                timeout=timeout,
+        loop = asyncio.get_running_loop()
+        deadline = None if timeout is None else loop.time() + timeout
+        while True:
+            remaining = None if deadline is None else deadline - loop.time()
+            if remaining is not None and remaining <= 0:
+                return None
+
+            message = await self._pubsub.get_message(
+                ignore_subscribe_messages=True,
+                timeout=remaining,
             )
-        except asyncio.TimeoutError:
-            return None
-
-        if message is None:
-            # Nothing ready. Yield rather than spin.
-            await asyncio.sleep(IDLE_SLEEP_SECONDS)
-            return None
+            if message is None:
+                # None before the deadline is an ignored subscribe ack, not
+                # silence. Only the expired wait is a keepalive.
+                if deadline is None or loop.time() < deadline:
+                    continue
+                return None
 
-        if message.get(PubSubField.TYPE.value) not in (None, PubSubMessageType.MESSAGE.value):
-            return None
+            if message.get(PubSubField.TYPE.value) not in (None, PubSubMessageType.MESSAGE.value):
+                continue
 
-        return self._decode(message.get(PubSubField.DATA.value))
+            return self._decode(message.get(PubSubField.DATA.value))
 
     def _decode(self, raw: Any) -> Any:
         """

corekit/events/sse.py

@@ -93,7 +93,10 @@ class SSEStream(Loggable):
             while True:
                 payload = await self._reader.next_payload(timeout=self.keepalive_interval)
                 if payload is None:
-                    yield SSEFrame.comment()
+                    # None means the wait expired. A disabled keepalive never
+                    # asks for a timeout, so it must not emit a comment either.
+                    if self.keepalive_interval is not None:
+                        yield SSEFrame.comment()
                     continue
                 yield self._frame_for(payload)
 

corekit/events/websocket.py

@@ -64,9 +64,16 @@ class WebSocketBridge(Loggable):
         """
         Whether the bridge has outlived its timeout.
         """
+        remaining = self._remaining(started)
+        return remaining is not None and remaining <= 0
+
+    def _remaining(self, started: float) -> float | None:
+        """
+        Seconds left before the bridge times out, or None if it never does.
+        """
         if self.timeout is None:
-            return False
-        return asyncio.get_event_loop().time() - started > self.timeout
+            return None
+        return self.timeout - (asyncio.get_running_loop().time() - started)
 
     async def run(self) -> None:
         """
@@ -75,21 +82,27 @@ class WebSocketBridge(Loggable):
         if not await self._reader.open():
             return
 
-        started = asyncio.get_event_loop().time()
+        started = asyncio.get_running_loop().time()
         try:
             while True:
-                payload = await self._reader.next_payload(timeout=self.timeout)
-
-                if payload is not None:
-                    await self.websocket.send_json(payload)
-                    if self._is_terminal(payload):
-                        self.info(f"Bridge on {self.channel} reached a terminal status")
-                        break
+                remaining = self._remaining(started)
+                if remaining is not None and remaining <= 0:
+                    self.warning(f"Bridge on {self.channel} timed out")
+                    break
 
-                if self._expired(started):
+                payload = await self._reader.next_payload(timeout=remaining)
+                if payload is None:
+                    # The wait expired. That is the timeout, not a reason to poll again.
+                    if remaining is None:
+                        continue
                     self.warning(f"Bridge on {self.channel} timed out")
                     break
 
+                await self.websocket.send_json(payload)
+                if self._is_terminal(payload):
+                    self.info(f"Bridge on {self.channel} reached a terminal status")
+                    break
+
         except asyncio.CancelledError:
             raise
         except Exception as exc:

corekit/exceptions/__init__.py

@@ -1,18 +1,33 @@
 """
 Exception base classes.
 
-``CustomException`` carries a message and an optional underlying error;
-``CustomHTTPException`` is its FastAPI-facing counterpart.
+``CoreException`` is abstract. Raise ``InternalCoreException`` for backend-only
+failures, or ``PublicCoreException`` when there is copy that is safe to show
+a user. ``CoreHTTPException`` is the FastAPI-facing public exception.
 """
 
-from corekit.exceptions.base import CustomException, CustomHTTPException, ExponentialBackoffTimeoutException
-from corekit.exceptions.types import ArbitraryCustomExceptionType, CustomExceptionType, CustomHTTPExceptionType
+from corekit.exceptions.base import (
+    CoreException,
+    CoreHTTPException,
+    ExponentialBackoffTimeoutException,
+    InternalCoreException,
+    NonRetryableCoreHTTPException,
+    PublicCoreException,
+    RetryableCoreHTTPException,
+)
+from corekit.exceptions.enum import Retryability
+from corekit.exceptions.types import ArbitraryCoreExceptionType, CoreExceptionType, CoreHTTPExceptionType
 
 __all__ = [
-    "ArbitraryCustomExceptionType",
-    "CustomException",
-    "CustomExceptionType",
-    "CustomHTTPException",
-    "CustomHTTPExceptionType",
+    "ArbitraryCoreExceptionType",
+    "CoreException",
+    "CoreExceptionType",
+    "CoreHTTPException",
+    "CoreHTTPExceptionType",
     "ExponentialBackoffTimeoutException",
+    "InternalCoreException",
+    "NonRetryableCoreHTTPException",
+    "PublicCoreException",
+    "Retryability",
+    "RetryableCoreHTTPException",
 ]

corekit/exceptions/base.py

@@ -1,26 +1,104 @@
 """
 Base exception types.
+
+Every corekit exception is one of two shapes:
+
+* ``InternalCoreException`` -- backend-only. ``message``/``error`` are for
+  logs and debugging and must never reach an end user; there is no field to
+  put user-facing copy in, so there is nothing to leak by accident.
+* ``PublicCoreException`` -- also carries ``user_message``, pre-written copy
+  that *is* safe to show externally. ``message``/``error`` stay backend-only
+  even here; only ``user_message`` should ever be serialized to a client.
+
+Both carry ``retryable``, so a caller (or whatever eventually surfaces the
+error) can tell whether trying again is worth it without re-deriving that
+from a status code every time.
+
+``CoreException`` itself is abstract -- it exists to hold the shared fields,
+not to be raised. Raise one of the two subclasses above (or a subclass of
+those, like ``CoreHTTPException`` and the typed HTTP errors in ``corekit.http``).
 """
 
+from abc import ABC
 from typing import Any
 
 from fastapi import HTTPException
 
-__all__ = ["CustomException", "CustomHTTPException", "ExponentialBackoffTimeoutException"]
+from corekit.exceptions.enum import Retryability
+
+__all__ = [
+    "CoreException",
+    "CoreHTTPException",
+    "RetryableCoreHTTPException",
+    "NonRetryableCoreHTTPException",
+    "ExponentialBackoffTimeoutException",
+    "InternalCoreException",
+    "PublicCoreException",
+]
 
 
-class CustomException(Exception):
+class CoreException(Exception, ABC):
     """
-    Base for corekit exceptions, carrying optional error context.
+    Abstract base carrying the fields every corekit exception shares.
     """
 
-    def __init__(self, message: str, error: str | None = None) -> None:
+    def __init__(
+        self,
+        message: str,
+        *,
+        retryable: Retryability = Retryability.UNKNOWN,
+        error: str | None = None,
+    ) -> None:
+        if type(self) is CoreException:
+            raise TypeError("CoreException is abstract -- raise InternalCoreException or PublicCoreException")
         super().__init__(message)
         self.message = message
         self.error = error
+        self.retryable = retryable
+
+    def for_log(self) -> str:
+        """
+        Backend text for logs.
+
+        ``str(self)`` on a public exception is the user-facing copy, so a log
+        line of ``str(exc)`` drops ``message``. This always returns the
+        backend message, and the separate error detail when one was given.
+        """
+        if self.error:
+            return f"{self.message} ({self.error})"
+        return self.message
+
+
+class InternalCoreException(CoreException):
+    """
+    Backend-only exception. Never expose ``message`` or ``error`` to a user.
+    """
+
+
+class PublicCoreException(CoreException):
+    """
+    Exception with pre-written copy that is safe to show an end user.
+
+    ``message``/``error`` remain backend-only, for logs; ``user_message`` is
+    the only field on this exception meant to leave the backend.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        user_message: str,
+        *,
+        retryable: Retryability = Retryability.UNKNOWN,
+        error: str | None = None,
+    ) -> None:
+        super().__init__(message, retryable=retryable, error=error)
+        self.user_message = user_message
+
+    def __str__(self) -> str:
+        return self.user_message
 
 
-class ExponentialBackoffTimeoutException(CustomException):
+class ExponentialBackoffTimeoutException(InternalCoreException):
     """
     Raised when a retry loop exhausts its attempts without succeeding.
     """
@@ -28,18 +106,69 @@ class ExponentialBackoffTimeoutException(CustomException):
     def __init__(self, attempts: int, error: str | None = None) -> None:
         super().__init__(
             message=f"Exponential Backoff timed out after {attempts} attempts",
+            retryable=Retryability.NON_RETRYABLE,
             error=error,
         )
         self.attempts = attempts
 
 
-class CustomHTTPException(HTTPException):
+class CoreHTTPException(HTTPException, PublicCoreException):
+    """
+    FastAPI HTTPException that is also a PublicCoreException.
+
+    ``detail`` (FastAPI's client-facing field) and ``user_message`` are kept
+    in sync -- pass either one. ``message``/``error`` default to ``detail``
+    when omitted, since most call sites raising this directly have nothing
+    more specific to log.
+    """
+
+    def __init__(
+        self,
+        *,
+        status_code: int,
+        detail: str | None = None,
+        message: str | None = None,
+        user_message: str | None = None,
+        retryable: Retryability = Retryability.UNKNOWN,
+        error: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        resolved_user_message = user_message or detail
+        if not resolved_user_message:
+            raise ValueError("CoreHTTPException requires detail= or user_message=")
+
+        resolved_message = message or resolved_user_message
+        HTTPException.__init__(self, status_code=status_code, detail=resolved_user_message, **kwargs)
+        PublicCoreException.__init__(
+            self,
+            resolved_message,
+            resolved_user_message,
+            retryable=retryable,
+            error=error,
+        )
+
+    def __str__(self) -> str:
+        # HTTPException.__str__ is "404: detail" and sits ahead of
+        # PublicCoreException in the MRO. The public contract is that only
+        # user_message leaves the backend.
+        return self.user_message
+
+
+class RetryableCoreHTTPException(CoreHTTPException):
+    """
+    HTTP error where trying the same request again may succeed.
+    """
+
+    def __init__(self, **kwargs: Any) -> None:
+        kwargs.setdefault("retryable", Retryability.RETRYABLE)
+        super().__init__(**kwargs)
+
+
+class NonRetryableCoreHTTPException(CoreHTTPException):
     """
-    FastAPI HTTPException that also accepts ``message=`` as an alias for ``detail=``.
+    HTTP error where the same request will fail again until something else changes.
     """
 
     def __init__(self, **kwargs: Any) -> None:
-        message = kwargs.pop("message", None)
-        if message:
-            kwargs["detail"] = message
+        kwargs.setdefault("retryable", Retryability.NON_RETRYABLE)
         super().__init__(**kwargs)

corekit/exceptions/enum.py

@@ -0,0 +1,17 @@
+"""
+Enums used by corekit exceptions.
+"""
+
+from corekit.schemas.enum import StringEnum
+
+__all__ = ["Retryability"]
+
+
+class Retryability(StringEnum):
+    """
+    Whether retrying the operation that raised an exception can help.
+    """
+
+    RETRYABLE = "retryable"
+    NON_RETRYABLE = "non_retryable"
+    UNKNOWN = "unknown"

corekit/exceptions/types.py

@@ -2,16 +2,16 @@
 Type aliases for exception classes.
 
 These name the *class*, for signatures that accept an exception type rather than
-an instance -- ``def handle(exc: CustomExceptionType) -> None``.
+an instance -- ``def handle(exc: CoreExceptionType) -> None``.
 
 ``type(X)`` returns X's metaclass, which is ``type`` for an ordinary class and
 carries no information about X. ``type[X]`` is the subscript form these need.
 """
 
-from corekit.exceptions.base import CustomException, CustomHTTPException
+from corekit.exceptions.base import CoreException, CoreHTTPException
 
-__all__ = ["ArbitraryCustomExceptionType", "CustomExceptionType", "CustomHTTPExceptionType"]
+__all__ = ["ArbitraryCoreExceptionType", "CoreExceptionType", "CoreHTTPExceptionType"]
 
-CustomExceptionType = type[CustomException]
-CustomHTTPExceptionType = type[CustomHTTPException]
-ArbitraryCustomExceptionType = CustomExceptionType | CustomHTTPExceptionType
+CoreExceptionType = type[CoreException]
+CoreHTTPExceptionType = type[CoreHTTPException]
+ArbitraryCoreExceptionType = CoreExceptionType | CoreHTTPExceptionType

corekit/files/__init__.py

@@ -8,10 +8,9 @@ File reading and writing, with format handled by the manager.
 ``_serialize`` and ``_deserialize`` for its format.
 """
 
-from corekit.files.base import FileContent, FileManager
-from corekit.files.enum import FileError, FileMode
+from corekit.files.base import FileContent, FileError, FileManager
+from corekit.files.enum import FileMode
 from corekit.files.json import JsonFileManager
-from corekit.files.pickle import PickleFileManager
 from corekit.files.toml import TomlFileManager
 
 __all__ = [
@@ -20,6 +19,5 @@ __all__ = [
     "FileManager",
     "FileMode",
     "JsonFileManager",
-    "PickleFileManager",
     "TomlFileManager",
 ]

corekit/files/base.py

@@ -1,10 +1,23 @@
 from typing import Any, BinaryIO, Iterator, TextIO, Union
 
-from corekit.files.enum import FileError, FileMode
+from corekit.exceptions import InternalCoreException, Retryability
+from corekit.files.enum import FileMode
 
 FileContent: type[str | bytes] = Union[str, bytes]
 
 
+class FileError(InternalCoreException):
+    """
+    Raised when a file manager is used in the wrong open/closed state.
+    """
+
+    NOT_OPEN = "File is not open"
+    OPEN = "File is open"
+
+    def __init__(self, message: str, *, error: str | None = None) -> None:
+        super().__init__(message, retryable=Retryability.NON_RETRYABLE, error=error)
+
+
 class FileManager:
     def __init__(
         self,
@@ -36,7 +49,7 @@ class FileManager:
 
     def _verify_file_status(self, should_be_open: bool) -> None:
         if self.is_open != should_be_open:
-            raise ValueError(FileError.NOT_OPEN if should_be_open else FileError.OPEN)  # FIXME: raise custom exception
+            raise FileError(FileError.NOT_OPEN if should_be_open else FileError.OPEN)
 
     # ===== File Opening/Closing Methods =====
     def _open(self) -> None:

corekit/files/enum.py

@@ -23,8 +23,3 @@ class FileMode(StringEnum):
         if self in {FileMode.READ, FileMode.READ_BINARY}:
             return False
         return True
-
-
-class FileError(StringEnum):
-    NOT_OPEN = "File is not open"
-    OPEN = "File is open"

corekit/files/json.py

@@ -1,7 +1,7 @@
 import json as _json
-from typing import Any
+from typing import Any, Iterator
 
-from corekit.files.base import FileContent, FileManager
+from corekit.files.base import FileContent, FileError, FileManager
 
 
 class JsonFileManager(FileManager):
@@ -10,3 +10,17 @@ class JsonFileManager(FileManager):
 
     def _deserialize(self, content: FileContent) -> Any:
         return _json.loads(content)
+
+    def read(self, size: int = -1) -> Any:
+        """
+        Read and parse the whole document. Partial reads are refused.
+        """
+        if size != -1:
+            raise FileError("JsonFileManager refuses partial reads; JSON must be parsed as a whole document.")
+        return super().read(size)
+
+    def readline(self) -> Any:
+        raise FileError("JsonFileManager cannot readline; JSON is not line-oriented.")
+
+    def stream(self) -> Iterator[Any]:
+        raise FileError("JsonFileManager cannot stream; JSON must be parsed as a whole document.")

corekit/http/__init__.py

@@ -1,13 +1,51 @@
 """
 HTTP client building blocks.
 
-``BaseApiClient`` wraps httpx with retries; ``ExponentialBackoff`` is the retry
-policy behind it and is useful on its own for any operation that should back off
-rather than hammer.
+``BaseHttpClient`` is the transport; ``BaseApiClient`` is the strict API
+wrapper. ``ExponentialBackoff`` is the retry policy behind the client and is
+useful on its own for any operation that should back off rather than hammer.
 """
 
-from corekit.http.client import BaseApiClient, URLMismatchError
+from corekit.http.api import BaseApiClient
+from corekit.http.client import BaseHttpClient
+from corekit.http.exceptions import (
+    BadGatewayException,
+    BadRequestException,
+    ConflictErrorException,
+    ForbiddenException,
+    GatewayTimeoutException,
+    InternalServerErrorException,
+    NotFoundException,
+    ServiceUnavailableException,
+    TooManyRequestsException,
+    UnauthorizedException,
+    UnprocessableEntityException,
+    UnsupportedMediaTypeException,
+    UnsupportedMethodError,
+    URLMismatchError,
+)
 from corekit.http.exponential_backoff import ExponentialBackoff
 from corekit.http.response import BaseApiResponse
+from corekit.http.status import HTTPStatusCode
 
-__all__ = ["BaseApiClient", "BaseApiResponse", "ExponentialBackoff", "URLMismatchError"]
+__all__ = [
+    "BadGatewayException",
+    "BadRequestException",
+    "BaseApiClient",
+    "BaseApiResponse",
+    "BaseHttpClient",
+    "ConflictErrorException",
+    "ExponentialBackoff",
+    "ForbiddenException",
+    "GatewayTimeoutException",
+    "HTTPStatusCode",
+    "InternalServerErrorException",
+    "NotFoundException",
+    "ServiceUnavailableException",
+    "TooManyRequestsException",
+    "UnauthorizedException",
+    "UnprocessableEntityException",
+    "UnsupportedMediaTypeException",
+    "UnsupportedMethodError",
+    "URLMismatchError",
+]

corekit/http/api.py

@@ -0,0 +1,24 @@
+"""
+API clients: a ``BaseHttpClient`` that always talks to one origin.
+"""
+
+from typing import Any
+
+from corekit.http.client import BaseHttpClient
+
+__all__ = ["BaseApiClient"]
+
+
+class BaseApiClient(BaseHttpClient):
+    """
+    Base for API clients. Override ``base_url`` and, usually, ``headers``.
+
+    Strict by default, so an absolute URL that does not share ``base_url`` is
+    a bug rather than a request to somewhere else. ``base_url`` must be
+    available during ``__init__`` — a property on the subclass is the usual way.
+    """
+
+    def __init__(self, *args: Any, strict: bool = True, **kwargs: Any) -> None:
+        super().__init__(*args, strict=strict, **kwargs)
+        if not self.base_url:
+            raise ValueError("BaseApiClient requires a non-empty base_url")