python-corekit 0.1.0__py3-none-any.whl

corekit/etl/transform/schemas.py

@@ -0,0 +1,15 @@
+from pydantic import BaseModel
+
+from corekit.etl.schemas import BaseItem
+
+
+class BaseTransformedItemModel(BaseModel):
+    """
+    Parent model for transformed data. All transformed data models should inherit from this.
+    """
+
+    pass
+
+
+class TransformedItem(BaseItem):
+    transformed_data: BaseTransformedItemModel

corekit/etl/transform/transformer.py

@@ -0,0 +1,28 @@
+from abc import ABC, abstractmethod
+from functools import cached_property
+from typing import Any, Callable
+
+from corekit.etl.extract.schemas import ExtractedItem
+from corekit.etl.transform.schemas import TransformedItem
+from corekit.observability.loggable import Loggable
+
+
+class BaseETLTransformer(Loggable, ABC):
+    def __init__(self) -> None:
+        super().__init__()
+
+    @cached_property
+    @abstractmethod
+    def _extracted_item_type_to_method(self) -> dict[type, Callable[[Any], Any]]:
+        raise NotImplementedError
+
+    def transform(self, item: ExtractedItem) -> TransformedItem:
+        """
+        Primary method to run the transformer.
+        """
+        data_type = item.get_data_type()
+        mapping = self._extracted_item_type_to_method
+        if data_type not in mapping:
+            raise ValueError(f"No transformation method found for data type: {data_type}")
+
+        return mapping[data_type](item)

corekit/events/__init__.py

@@ -0,0 +1,38 @@
+"""
+Real-time events over Redis Pub/Sub.
+
+One side publishes::
+
+    EventPublisher("backups:events").publish("finished", {"size": "4.2GB"})
+
+The other streams to a browser::
+
+    @router.get("/events")
+    async def events(channel: str) -> SSEResponse:
+        return SSEResponse(SSEStream(channel))
+
+Publishers and subscribers meet on a channel name; ``for_resource`` builds the
+conventional ``service:type:id`` form so both ends agree without sharing a
+constant.
+"""
+
+from corekit.events.enum import MessageField, PubSubField, PubSubMessageType, StreamEvent
+from corekit.events.frames import SSEFrame
+from corekit.events.models import BaseEvent
+from corekit.events.publisher import EventPublisher
+from corekit.events.reader import RedisChannelReader
+from corekit.events.sse import SSEStream
+from corekit.events.websocket import WebSocketBridge
+
+__all__ = [
+    "BaseEvent",
+    "EventPublisher",
+    "MessageField",
+    "PubSubField",
+    "PubSubMessageType",
+    "RedisChannelReader",
+    "SSEFrame",
+    "SSEStream",
+    "StreamEvent",
+    "WebSocketBridge",
+]

corekit/events/enum.py

@@ -0,0 +1,58 @@
+"""
+Wire-level names used by the event stream.
+
+These are the strings that cross the network, so they are declared once rather
+than repeated at each site that writes or reads one.
+
+They are enums rather than bare constants because each set is closed: an event
+is one of four kinds, a Redis message is one of three. A typo in a member name
+is an AttributeError, where a typo in a string literal is a message nobody
+receives.
+"""
+
+from corekit.schemas.enum import StringEnum
+
+__all__ = ["MessageField", "PubSubField", "PubSubMessageType", "StreamEvent"]
+
+
+class StreamEvent(StringEnum):
+    """
+    Event names corekit itself emits, as distinct from application events.
+    """
+
+    CONNECTED = "connected"
+    INITIAL_STATE = "initial_state"
+    ERROR = "error"
+    MESSAGE = "message"
+
+
+class MessageField(StringEnum):
+    """
+    Keys in an event's JSON body.
+    """
+
+    TYPE = "type"
+    DATA = "data"
+    STATUS = "status"
+    ERROR = "error"
+    CHANNEL = "channel"
+
+
+class PubSubField(StringEnum):
+    """
+    Keys in a message as Redis delivers it.
+    """
+
+    TYPE = "type"
+    DATA = "data"
+
+
+class PubSubMessageType(StringEnum):
+    """
+    Redis message kinds. Only MESSAGE carries a payload; the rest are
+    subscription bookkeeping.
+    """
+
+    MESSAGE = "message"
+    SUBSCRIBE = "subscribe"
+    UNSUBSCRIBE = "unsubscribe"

corekit/events/frames.py

@@ -0,0 +1,51 @@
+"""
+The Server-Sent Events wire format.
+
+A frame is ``event: <name>``, then ``data: <json>``, then a blank line. The
+blank line is what terminates it; without one a browser buffers indefinitely
+waiting for more.
+"""
+
+import json
+from typing import Any
+
+from corekit.events.enum import MessageField, StreamEvent
+
+__all__ = ["SSEFrame"]
+
+
+class SSEFrame:
+    """
+    Renders the Server-Sent Events wire format.
+
+    A frame is ``event: <name>`` then ``data: <json>`` then a blank line. The
+    blank line terminates the frame; without it a browser buffers indefinitely
+    waiting for more.
+    """
+
+    TERMINATOR = "\n\n"
+
+    @classmethod
+    def event(cls, event_type: str, data: Any) -> str:
+        """
+        A named event carrying a JSON payload.
+        """
+        name = event_type.value if isinstance(event_type, StreamEvent) else event_type
+        return f"event: {name}\ndata: {json.dumps(data)}{cls.TERMINATOR}"
+
+    @classmethod
+    def error(cls, message: str) -> str:
+        """
+        An error the client should surface rather than retry silently.
+        """
+        return cls.event(StreamEvent.ERROR, {MessageField.ERROR.value: message})
+
+    @classmethod
+    def comment(cls, text: str = "keepalive") -> str:
+        """
+        A comment frame.
+
+        Browsers ignore comments, so they keep an idle connection from being
+        closed by an intermediate proxy without the client seeing anything.
+        """
+        return f": {text}{cls.TERMINATOR}"

corekit/events/models.py

@@ -0,0 +1,23 @@
+"""
+Event payloads.
+"""
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from corekit.utils.time import time_now
+
+__all__ = ["BaseEvent"]
+
+
+class BaseEvent(BaseModel):
+    """
+    An event: a type, a payload, and when it happened.
+
+    Subclass it to add fields; the wire format stays JSON either way.
+    """
+
+    type: str
+    data: dict[str, Any] = Field(default_factory=dict)
+    timestamp: str = Field(default_factory=lambda: time_now().isoformat())

corekit/events/publisher.py

@@ -0,0 +1,75 @@
+"""
+Publishing events over Redis Pub/Sub.
+
+    publisher = EventPublisher("backups:events")
+    publisher.publish("finished", {"size": "4.2GB"})
+
+Channels are just strings. ``for_resource`` builds the conventional
+``service:type:id`` form so publishers and subscribers agree without a shared
+constant::
+
+    EventPublisher.for_resource("minecraft", "server", "survival")
+    # -> channel "minecraft:server:survival"
+
+Requires the ``redis`` extra.
+"""
+
+from typing import Any
+
+from corekit.connections import connect
+from corekit.connections.redis import RedisConnection
+from corekit.events.models import BaseEvent
+from corekit.observability.benchmarkable import Benchmarkable
+
+__all__ = ["EventPublisher"]
+
+
+class EventPublisher(Benchmarkable):
+    """
+    Publishes events to one Redis channel.
+
+    Publishing never raises: an event that cannot be delivered should not take
+    down the operation that produced it. ``publish`` returns whether it worked.
+    """
+
+    def __init__(self, channel: str) -> None:
+        super().__init__()
+        self.channel = channel
+
+    @classmethod
+    def for_resource(cls, service: str, resource_type: str, resource_id: str) -> "EventPublisher":
+        """
+        Build a publisher for the conventional ``service:type:id`` channel.
+        """
+        return cls(f"{service}:{resource_type}:{resource_id}")
+
+    def publish(self, event_type: str, data: dict[str, Any], **kwargs: Any) -> bool:
+        """
+        Publish an event, returning whether it was delivered.
+        """
+        return self.publish_event(BaseEvent(type=event_type, data=data, **kwargs))
+
+    @connect(RedisConnection)
+    def publish_event(self, conn: RedisConnection, event: BaseEvent) -> bool:
+        """
+        Publish an already-built event.
+        """
+        try:
+            subscribers = conn.client.publish(self.channel, event.model_dump_json())
+            self.debug(f"Published {event.type!r} to {self.channel} ({subscribers} subscriber(s))")
+            return True
+        except Exception as exc:
+            self.error(f"Failed to publish to {self.channel}: {exc}")
+            return False
+
+    @connect(RedisConnection)
+    def subscriber_count(self, conn: RedisConnection) -> int:
+        """
+        How many subscribers this channel currently has.
+        """
+        try:
+            result = conn.client.pubsub_numsub(self.channel)
+            return result[0][1] if result else 0
+        except Exception as exc:
+            self.error(f"Failed to count subscribers for {self.channel}: {exc}")
+            return 0

corekit/events/reader.py

@@ -0,0 +1,132 @@
+"""
+Reading a Redis Pub/Sub channel.
+
+Holds what every streaming shape needs -- opening an async connection,
+subscribing, decoding payloads, and unsubscribing even when the consumer
+disappears -- so no consumer has to repeat it.
+
+Everything here uses Redis's async client. The synchronous client's
+``get_message`` blocks the thread it is called on, which in an async server is
+the event loop, stalling every other request in the process while one client
+waits for an event that may never come.
+"""
+
+import asyncio
+import json
+from typing import Any
+
+from corekit.connections.redis import RedisConnection
+from corekit.events.enum import PubSubField, PubSubMessageType
+from corekit.observability.loggable import Loggable
+
+__all__ = ["RedisChannelReader"]
+
+# How long to yield to the event loop when the socket has nothing ready.
+IDLE_SLEEP_SECONDS = 0.01
+
+
+class RedisChannelReader(Loggable):
+    """
+    Subscribes to a Redis channel and yields decoded messages.
+
+    Holds the parts that both streaming shapes need -- opening an async
+    connection, subscribing, decoding payloads, and unsubscribing even when the
+    consumer disappears -- so neither has to repeat them.
+    """
+
+    def __init__(self, channel: str, url: str | None = None) -> None:
+        super().__init__()
+        self.channel = channel
+        self._url = url
+        self._connection: RedisConnection | None = None
+        self._pubsub: Any = None
+
+    @property
+    def is_open(self) -> bool:
+        """
+        Whether the subscription is currently established.
+        """
+        return self._pubsub is not None
+
+    async def open(self) -> bool:
+        """
+        Connect and subscribe, reporting whether it worked.
+
+        Returns False rather than raising when Redis is unreachable: a stream
+        that cannot start should tell its client, not crash the request.
+        """
+        connection = RedisConnection(url=self._url, safe=True)
+        await connection.async_connect()
+
+        if not connection.is_async_connected:
+            self.error(f"Redis unavailable; cannot subscribe to {self.channel}")
+            return False
+
+        self._connection = connection
+        self._pubsub = connection.async_client.pubsub()
+        await self._pubsub.subscribe(self.channel)
+        self.info(f"Subscribed to {self.channel}")
+        return True
+
+    async def close(self) -> None:
+        """
+        Unsubscribe and disconnect. Safe to call more than once.
+        """
+        if self._pubsub is not None:
+            try:
+                await self._pubsub.unsubscribe(self.channel)
+                await self._pubsub.aclose()
+            except Exception as exc:
+                self.debug(f"Error closing subscription to {self.channel}: {exc}")
+            self._pubsub = None
+
+        if self._connection is not None:
+            await self._connection.async_disconnect()
+            self._connection = None
+
+    async def next_payload(self, timeout: float | None) -> Any:
+        """
+        Return the next decoded payload.
+
+        Returns ``None`` when nothing arrived before ``timeout``, so a caller
+        can act on the silence -- sending a keepalive, say -- rather than
+        blocking forever.
+        """
+        try:
+            message = await asyncio.wait_for(
+                self._pubsub.get_message(ignore_subscribe_messages=True),
+                timeout=timeout,
+            )
+        except asyncio.TimeoutError:
+            return None
+
+        if message is None:
+            # Nothing ready. Yield rather than spin.
+            await asyncio.sleep(IDLE_SLEEP_SECONDS)
+            return None
+
+        if message.get(PubSubField.TYPE.value) not in (None, PubSubMessageType.MESSAGE.value):
+            return None
+
+        return self._decode(message.get(PubSubField.DATA.value))
+
+    def _decode(self, raw: Any) -> Any:
+        """
+        Decode a payload, returning the raw text when it is not JSON.
+
+        A shared channel may have writers that are not EventPublisher, and one
+        of those should not look like a fault.
+        """
+        if isinstance(raw, bytes):
+            raw = raw.decode("utf-8", errors="replace")
+        try:
+            return json.loads(raw)
+        except (TypeError, ValueError):
+            return raw
+
+    async def __aenter__(self) -> "RedisChannelReader":
+        await self.open()
+        return self
+
+    async def __aexit__(self, *exc_info: Any) -> None:
+        await self.close()

corekit/events/sse.py

@@ -0,0 +1,109 @@
+"""
+Streaming events to a browser over Server-Sent Events.
+
+SSE is the simpler half of real-time: a plain HTTP response the browser
+reconnects on its own, with no protocol upgrade.
+
+    @router.get("/events")
+    async def events(channel: str) -> SSEResponse:
+        return SSEResponse(SSEStream(channel))
+
+The browser side is three lines::
+
+    const source = new EventSource("/events?channel=backups:events");
+    source.addEventListener("finished", e => console.log(JSON.parse(e.data)));
+"""
+
+import asyncio
+from typing import Any, AsyncIterator
+
+from corekit.events.enum import MessageField, StreamEvent
+from corekit.events.frames import SSEFrame
+from corekit.events.reader import RedisChannelReader
+from corekit.observability.loggable import Loggable
+
+__all__ = ["SSEStream"]
+
+DEFAULT_KEEPALIVE_SECONDS = 30.0
+
+
+class SSEStream(Loggable):
+    """
+    An async iterator of SSE frames for one Redis channel.
+
+        return SSEResponse(SSEStream(channel, initial_state={"running": 2}))
+
+    Emits a ``connected`` frame on subscribe, an optional ``initial_state`` so a
+    client arriving late renders immediately, then one frame per published
+    event, with a comment frame whenever the channel is quiet.
+    """
+
+    def __init__(
+        self,
+        channel: str,
+        initial_state: dict[str, Any] | None = None,
+        keepalive_interval: float | None = DEFAULT_KEEPALIVE_SECONDS,
+        url: str | None = None,
+    ) -> None:
+        """
+        :param channel: the Redis channel to subscribe to.
+        :param initial_state: sent once, immediately after connecting.
+        :param keepalive_interval: seconds between comment frames; None disables them.
+        :param url: Redis URL, defaulting to the configured one.
+        """
+        super().__init__()
+        self.channel = channel
+        self.initial_state = initial_state
+        self.keepalive_interval = keepalive_interval
+        self._reader = RedisChannelReader(channel, url=url)
+
+    def _opening_frames(self) -> list[str]:
+        """
+        The frames sent before any published event.
+        """
+        frames = [SSEFrame.event(StreamEvent.CONNECTED, {MessageField.CHANNEL.value: self.channel})]
+        if self.initial_state is not None:
+            frames.append(SSEFrame.event(StreamEvent.INITIAL_STATE, self.initial_state))
+        return frames
+
+    @staticmethod
+    def _frame_for(payload: Any) -> str:
+        """
+        Render one decoded payload as a frame.
+
+        A dict carrying a ``type`` is an event and keeps its name; anything else
+        is passed through under a generic name rather than being dropped.
+        """
+        if isinstance(payload, dict) and MessageField.TYPE.value in payload:
+            return SSEFrame.event(payload[MessageField.TYPE.value], payload)
+        return SSEFrame.event(StreamEvent.MESSAGE, payload)
+
+    async def __aiter__(self) -> AsyncIterator[str]:
+        """
+        Yield frames until the client disconnects.
+        """
+        if not await self._reader.open():
+            yield SSEFrame.error("Event stream unavailable")
+            return
+
+        try:
+            for frame in self._opening_frames():
+                yield frame
+
+            while True:
+                payload = await self._reader.next_payload(timeout=self.keepalive_interval)
+                if payload is None:
+                    yield SSEFrame.comment()
+                    continue
+                yield self._frame_for(payload)
+
+        except asyncio.CancelledError:
+            self.info(f"Stream for {self.channel} cancelled")
+            raise
+        except GeneratorExit:
+            self.info(f"Client disconnected from {self.channel}")
+        except Exception as exc:
+            self.error(f"Error streaming {self.channel}: {exc}")
+            yield SSEFrame.error(str(exc))
+        finally:
+            await self._reader.close()

corekit/events/websocket.py

@@ -0,0 +1,97 @@
+"""
+Relaying events to a WebSocket.
+
+    await WebSocketBridge(channel, websocket, terminal_statuses=["done"]).run()
+
+Use this rather than SSE when the client also needs to send messages back; for
+one-way updates SSE is less machinery and reconnects on its own.
+"""
+
+import asyncio
+from typing import Any
+
+from corekit.events.enum import MessageField
+from corekit.events.reader import RedisChannelReader
+from corekit.observability.loggable import Loggable
+
+__all__ = ["WebSocketBridge"]
+
+DEFAULT_BRIDGE_TIMEOUT_SECONDS = 300.0
+
+# Event type carrying a lifecycle status, checked against terminal_statuses.
+STATUS_UPDATE_EVENT = "status_update"
+
+
+class WebSocketBridge(Loggable):
+    """
+    Relays everything published to a channel to an open WebSocket.
+
+        await WebSocketBridge(channel, websocket, terminal_statuses=["done"]).run()
+
+    Stops on a terminal status so a client watching a job disconnects when the
+    job ends, and on a timeout so an abandoned socket cannot hold a subscription
+    open forever.
+    """
+
+    def __init__(
+        self,
+        channel: str,
+        websocket: Any,
+        terminal_statuses: list[str] | None = None,
+        timeout: float | None = DEFAULT_BRIDGE_TIMEOUT_SECONDS,
+        url: str | None = None,
+    ) -> None:
+        super().__init__()
+        self.channel = channel
+        self.websocket = websocket
+        self.terminal_statuses = terminal_statuses or []
+        self.timeout = timeout
+        self._reader = RedisChannelReader(channel, url=url)
+
+    def _is_terminal(self, payload: Any) -> bool:
+        """
+        Whether this payload means the thing being watched has finished.
+        """
+        if not self.terminal_statuses or not isinstance(payload, dict):
+            return False
+        if payload.get(MessageField.TYPE.value) != STATUS_UPDATE_EVENT:
+            return False
+        data = payload.get(MessageField.DATA.value) or {}
+        return data.get(MessageField.STATUS.value) in self.terminal_statuses
+
+    def _expired(self, started: float) -> bool:
+        """
+        Whether the bridge has outlived its timeout.
+        """
+        if self.timeout is None:
+            return False
+        return asyncio.get_event_loop().time() - started > self.timeout
+
+    async def run(self) -> None:
+        """
+        Relay until a terminal status, the timeout, or an error.
+        """
+        if not await self._reader.open():
+            return
+
+        started = asyncio.get_event_loop().time()
+        try:
+            while True:
+                payload = await self._reader.next_payload(timeout=self.timeout)
+
+                if payload is not None:
+                    await self.websocket.send_json(payload)
+                    if self._is_terminal(payload):
+                        self.info(f"Bridge on {self.channel} reached a terminal status")
+                        break
+
+                if self._expired(started):
+                    self.warning(f"Bridge on {self.channel} timed out")
+                    break
+
+        except asyncio.CancelledError:
+            raise
+        except Exception as exc:
+            self.error(f"Error bridging {self.channel}: {exc}")
+        finally:
+            await self._reader.close()

corekit/exceptions/base.py

@@ -0,0 +1,45 @@
+"""
+Base exception types.
+"""
+
+from typing import Any
+
+from fastapi import HTTPException
+
+__all__ = ["CustomException", "CustomHTTPException", "ExponentialBackoffTimeoutException"]
+
+
+class CustomException(Exception):
+    """
+    Base for corekit exceptions, carrying optional error context.
+    """
+
+    def __init__(self, message: str, error: str | None = None) -> None:
+        super().__init__(message)
+        self.message = message
+        self.error = error
+
+
+class ExponentialBackoffTimeoutException(CustomException):
+    """
+    Raised when a retry loop exhausts its attempts without succeeding.
+    """
+
+    def __init__(self, attempts: int, error: str | None = None) -> None:
+        super().__init__(
+            message=f"Exponential Backoff timed out after {attempts} attempts",
+            error=error,
+        )
+        self.attempts = attempts
+
+
+class CustomHTTPException(HTTPException):
+    """
+    FastAPI HTTPException that also accepts ``message=`` as an alias for ``detail=``.
+    """
+
+    def __init__(self, **kwargs: Any) -> None:
+        message = kwargs.pop("message", None)
+        if message:
+            kwargs["detail"] = message
+        super().__init__(**kwargs)