python-corekit 0.1.0__py3-none-any.whl

corekit/docker/watchdog.py

@@ -0,0 +1,222 @@
+"""
+Docker container lifecycle control.
+
+    watchdog = Watchdog(enforce_label=True)
+    watchdog.start_container_by_name("minecraft")
+    watchdog.find_and_stop(label="app", value="staging")
+
+``enforce_label`` is a blast-radius limiter: with it on, the watchdog will only
+start, stop, pause or unpause containers carrying the managed label, so a
+mistaken name cannot take down something unrelated. Leave it on unless the
+watchdog is genuinely meant to control every container on the host.
+
+"""
+
+from typing import Any, Callable, Iterator
+
+from docker import DockerClient
+from docker.errors import APIError, NotFound
+from docker.models.containers import Container
+
+from corekit.observability.benchmarkable import Benchmarkable
+
+__all__ = ["Watchdog"]
+
+DEFAULT_DOCKER_HOST = "unix:///var/run/docker.sock"
+WATCHDOG_LABEL = "watchdog"
+WATCHDOG_MANAGED = "true"
+
+
+def _accept_all(*args: Any, **kwargs: Any) -> bool:
+    """
+    Filter that accepts every container.
+    """
+    return True
+
+
+class Watchdog(Benchmarkable):
+    """
+    Starts, stops, pauses and lists Docker containers.
+
+    Docker API failures are logged rather than raised: a watchdog is usually a
+    background daemon, and one unreachable container should not stop it.
+    """
+
+    def __init__(self, docker_host: str | None = None, enforce_label: bool = False) -> None:
+        """
+        :param docker_host: Docker endpoint. Defaults to the local socket.
+        :param enforce_label: restrict actions to containers carrying the
+            managed label.
+        """
+        super().__init__()
+        self.docker_host = docker_host or DEFAULT_DOCKER_HOST
+        self.enforce_label = enforce_label
+        self._client_instance: DockerClient | None = None
+
+    @property
+    def _client(self) -> DockerClient:
+        """
+        The Docker client, connected on first use.
+
+        Constructing a Watchdog does not reach the daemon, so one can be built
+        and inspected on a machine where Docker is not running.
+        """
+        if self._client_instance is None:
+            self._client_instance = DockerClient(base_url=self.docker_host)
+        return self._client_instance
+
+    @staticmethod
+    def _get_label_filter(
+        label: str = WATCHDOG_LABEL,
+        value: str = WATCHDOG_MANAGED,
+    ) -> Callable[[Container], bool]:
+        """
+        Build a filter matching containers whose label equals a value.
+        """
+        return lambda container: container.labels.get(label) == value
+
+    def _container_iterator(
+        self, filter_func: Callable[[Container], bool] = _accept_all, **kwargs: Any
+    ) -> Iterator[Container]:
+        """
+        Yield containers matching a filter, logging API errors.
+        """
+        try:
+            for container in self._client.containers.list(**kwargs):
+                if filter_func(container):
+                    yield container
+        except APIError as exc:
+            self.error(f"Docker API error during iteration: {exc}")
+
+    def _label_validator(self, container: Container) -> bool:
+        """
+        Whether this watchdog is permitted to act on a container.
+        """
+        if self.enforce_label:
+            return container.labels.get(WATCHDOG_LABEL) == WATCHDOG_MANAGED
+        return True
+
+    def _start_container(self, container: Container) -> None:
+        try:
+            container.reload()
+            if container.status != "running" and self._label_validator(container):
+                container.start()
+                self.info(f"Started container: {container.name}")
+            else:
+                self.info(f"Container {container.name!r} is already running or not managed.")
+        except APIError as exc:
+            self.error(f"Failed to start container {container.name}: {exc}")
+
+    def _stop_container(self, container: Container) -> None:
+        try:
+            container.reload()
+            if container.status == "running" and self._label_validator(container):
+                container.stop()
+                self.info(f"Stopped container: {container.name}")
+            else:
+                self.info(f"Container {container.name!r} is not running or not managed.")
+        except APIError as exc:
+            self.error(f"Failed to stop container {container.name}: {exc}")
+
+    def _by_name(self, name: str) -> Container | None:
+        """
+        Look a container up by name, logging if it is missing.
+        """
+
+        try:
+            return self._client.containers.get(name)
+        except NotFound:
+            self.error(f"Container {name!r} not found.")
+        except APIError as exc:
+            self.error(f"Failed to fetch container {name}: {exc}")
+        return None
+
+    def list_containers(self, **kwargs: Any) -> list[Container]:
+        """
+        Every container, or an empty list if Docker is unreachable.
+        """
+        try:
+            return self._client.containers.list(**kwargs)
+        except APIError as exc:
+            self.error(f"Failed to list containers: {exc}")
+            return []
+
+    def list_containers_by_label(self, label: str, value: str) -> list[Container]:
+        """
+        Containers whose label equals a value.
+        """
+        return list(self._container_iterator(filter_func=self._get_label_filter(label, value), all=True))
+
+    def find_and_stop(self, label: str, value: str) -> None:
+        """
+        Stop every container matching a label.
+        """
+        for container in self._container_iterator(filter_func=self._get_label_filter(label, value), all=True):
+            self._stop_container(container)
+
+    def find_and_start(self, label: str, value: str) -> None:
+        """
+        Start every container matching a label.
+        """
+        for container in self._container_iterator(filter_func=self._get_label_filter(label, value), all=True):
+            self._start_container(container)
+
+    def start_container_by_name(self, name: str) -> None:
+        """
+        Start one container by name.
+        """
+        container = self._by_name(name)
+        if container is not None:
+            self._start_container(container)
+
+    def stop_container_by_name(self, name: str) -> None:
+        """
+        Stop one container by name.
+        """
+        container = self._by_name(name)
+        if container is not None:
+            self._stop_container(container)
+
+    def restart_container_by_name(self, name: str) -> None:
+        """
+        Stop and then start one container by name.
+        """
+        container = self._by_name(name)
+        if container is None:
+            return
+        self._stop_container(container)
+        self._start_container(container)
+
+    def pause_container_by_name(self, name: str) -> None:
+        """
+        Pause one running container by name.
+        """
+        container = self._by_name(name)
+        if container is None:
+            return
+        try:
+            container.reload()
+            if container.status == "running" and self._label_validator(container):
+                container.pause()
+                self.info(f"Paused container: {container.name}")
+            else:
+                self.info(f"Container {container.name!r} is not running or not managed.")
+        except APIError as exc:
+            self.error(f"Failed to pause container {name}: {exc}")
+
+    def unpause_container_by_name(self, name: str) -> None:
+        """
+        Unpause one paused container by name.
+        """
+        container = self._by_name(name)
+        if container is None:
+            return
+        try:
+            container.reload()
+            if container.status == "paused" and self._label_validator(container):
+                container.unpause()
+                self.info(f"Unpaused container: {container.name}")
+            else:
+                self.info(f"Container {container.name!r} is not paused or not managed.")
+        except APIError as exc:
+            self.error(f"Failed to unpause container {name}: {exc}")

corekit/etl/__init__.py

@@ -0,0 +1,44 @@
+"""
+ETL pipelines.
+
+Implement an extractor, a transformer and a loader, then declare them on an
+orchestrator::
+
+    class UserPipeline(BaseETLOrchestrator):
+        '''
+        Loads users from the API into the warehouse.
+        '''
+        extractor = UserExtractor
+        transformer = UserTransformer
+        loader = UserLoader
+
+    await UserPipeline().run()
+
+The run loop streams, so memory use is bounded by the batch size rather than by
+how much data the source holds.
+"""
+
+from corekit.etl.connection import BaseConnection, ConnectionDetails
+from corekit.etl.extract.extractor import BaseETLExtractor
+from corekit.etl.extract.schemas import BaseExtractedItemModel, ExtractedItem
+from corekit.etl.load.loader import BaseETLLoader
+from corekit.etl.load.schemas import LoadableBatch
+from corekit.etl.orchestrator import BaseETLOrchestrator
+from corekit.etl.schemas import BaseItem
+from corekit.etl.transform.schemas import BaseTransformedItemModel, TransformedItem
+from corekit.etl.transform.transformer import BaseETLTransformer
+
+__all__ = [
+    "BaseConnection",
+    "BaseETLExtractor",
+    "BaseETLLoader",
+    "BaseETLOrchestrator",
+    "BaseETLTransformer",
+    "BaseExtractedItemModel",
+    "BaseItem",
+    "BaseTransformedItemModel",
+    "ConnectionDetails",
+    "ExtractedItem",
+    "LoadableBatch",
+    "TransformedItem",
+]

corekit/etl/connection.py

@@ -0,0 +1,44 @@
+from enum import Enum
+from typing import NamedTuple
+
+
+class ConnectionDetails(NamedTuple):
+    name: str
+    description: str
+
+
+class BaseConnection(Enum):
+    """
+    Base enum for ETL Connections. All ETL connections must inherit from this class.
+
+    Example:
+        class MyConnectionEnum(BaseConnection):
+            MY_FIRST_CONNECTION = ConnectionDetails(
+                name="my_first_connection",
+                description="My First Connection"
+            )
+            ANOTHER_CONNECTION = ConnectionDetails(
+                name="another_connection",
+                description="Another Connection"
+            )
+    """
+
+    @classmethod
+    def from_name(cls, name: str) -> "BaseConnection":
+        # TODO: Make sure this is efficient
+        for connection in cls:
+            if connection.value.name == name:
+                return connection
+        raise ValueError(f"Connection {name} not found")
+
+    def get_name(self) -> str:
+        """
+        Helper method for fetching the name of the connection
+        """
+        return self.value.name
+
+    def get_description(self) -> str:
+        """
+        Helper method for fetching the name of the connection
+        """
+        return self.value.description

corekit/etl/extract/extractor.py

@@ -0,0 +1,48 @@
+from abc import ABC, abstractmethod
+from typing import Any, AsyncIterable, Callable
+
+from corekit.etl.connection import BaseConnection
+from corekit.etl.extract.schemas import ExtractedItem
+from corekit.http.client import BaseApiClient
+from corekit.observability.loggable import Loggable
+
+
+class BaseETLExtractor(Loggable, ABC):
+    """
+    Base class for ETL extractors.
+    """
+
+    def __init__(self, client: BaseApiClient) -> None:
+        super().__init__()
+        self.client = client
+
+    @property
+    @abstractmethod
+    def connection(self) -> BaseConnection:
+        """
+        Abstract property that must be implemented by child classes.
+        This property should return the integration object that the extractor is associated with
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def _extraction_methods(self) -> list[Callable[[Any], Any]]:
+        """
+        Abstract method that must be implemented by child classes.
+        This method should return an ordered list of extraction methods to call
+        """
+        raise NotImplementedError
+
+    def extraction_methods(self) -> list[Callable[[Any], Any]]:
+        """
+        Parent class method used to return an ordered list of extraction methods to call
+        """
+        return self._extraction_methods()
+
+    async def extract(self, *args: Any, **kwargs: Any) -> AsyncIterable[ExtractedItem]:
+        """
+        Primary method to run the extractor.
+        """
+        for extraction_method in self.extraction_methods():
+            async for item in extraction_method(*args, **kwargs):
+                yield item

corekit/etl/extract/schemas.py

@@ -0,0 +1,18 @@
+from pydantic import BaseModel
+
+from corekit.etl.schemas import BaseItem
+
+
+class BaseExtractedItemModel(BaseModel):
+    """
+    Parent model for extracted data. All extracted data models should inherit from this.
+    """
+
+    pass
+
+
+class ExtractedItem(BaseItem):
+    extracted_data: BaseExtractedItemModel
+
+    def get_data_type(self) -> type:
+        return type(self.extracted_data)

corekit/etl/load/loader.py

@@ -0,0 +1,53 @@
+from abc import ABC, abstractmethod
+
+from corekit.etl.load.schemas import LoadableBatch
+from corekit.etl.transform.schemas import TransformedItem
+from corekit.observability.loggable import Loggable
+
+
+class BaseETLLoader(Loggable, ABC):
+    def __init__(self) -> None:
+        super().__init__()
+        self.batch: LoadableBatch = LoadableBatch()
+
+    def _should_load(self, min_ops: int) -> bool:
+        """
+        Whether the batch is worth writing out.
+
+        An empty batch never is: flush() passes min_ops=0 to mean "write
+        whatever is left", and without this an exhausted or empty source would
+        still cost a write.
+        """
+        return bool(len(self.batch)) and len(self.batch) >= min_ops
+
+    def add_item(self, item: TransformedItem) -> None:
+        """
+        Adds an item to the batch
+        """
+        self.batch.add_item(item)
+
+    async def flush(self) -> None:
+        """
+        Load whatever is in the batch, however small.
+        """
+        await self.load(min_ops=0)
+
+    async def load(self, min_ops: int) -> None:
+        """
+        Load the batch if it has reached ``min_ops`` items.
+
+        The threshold is checked here so that implementations of ``_load`` only
+        have to handle writing a batch out.
+        """
+        if not self._should_load(min_ops):
+            return
+
+        await self._load(self.batch)
+        self.batch.clear()
+
+    @abstractmethod
+    async def _load(self, batch: LoadableBatch) -> None:
+        """
+        Write a batch to its destination.
+        """
+        raise NotImplementedError

corekit/etl/load/schemas.py

@@ -0,0 +1,33 @@
+from typing import Iterator
+
+from pydantic import BaseModel
+
+from corekit.etl.transform.schemas import TransformedItem
+from corekit.schemas.pydantic.fields import DefaultListField
+
+
+class LoadableBatch(BaseModel):
+    """
+    Parent model for batches of loadable data. All loadable data batch models should inherit from this.
+    """
+
+    items: list[TransformedItem] = DefaultListField()
+
+    def __iter__(self) -> Iterator[TransformedItem]:
+        yield from self.items
+
+    def __len__(self) -> int:
+        return len(self.items)
+
+    def add_item(self, item: TransformedItem) -> None:
+        self.items.append(item)
+
+    def clear(self) -> None:
+        self.items.clear()
+
+    def stream(self) -> Iterator[TransformedItem]:
+        """
+        Stream the items in the batch and clear the batch.
+        """
+        yield from self
+        self.clear()

corekit/etl/orchestrator.py

@@ -0,0 +1,201 @@
+"""
+ETL pipeline orchestration.
+
+A pipeline declares its three stages as class attributes and the orchestrator
+assembles them::
+
+    class UserPipeline(BaseETLOrchestrator):
+        '''
+        Loads users from the API into the warehouse.
+        '''
+        extractor = UserExtractor
+        transformer = UserTransformer
+        loader = UserLoader
+
+    await UserPipeline().run()
+
+The bracket form is shorthand for the same thing::
+
+    UserPipeline = BaseETLOrchestrator[UserExtractor, UserTransformer, UserLoader]
+
+Stages are validated when the class is defined, so a wrong or missing stage is
+reported at import rather than part-way through a run. Instances can still be
+passed explicitly, which is how tests substitute doubles.
+
+The run loop streams: items are extracted one at a time and loaded in batches,
+so memory use is bounded by the batch size rather than the size of the source.
+"""
+
+from dataclasses import dataclass
+from typing import Any, ClassVar
+
+from corekit.etl.extract.extractor import BaseETLExtractor
+from corekit.etl.load.loader import BaseETLLoader
+from corekit.etl.transform.transformer import BaseETLTransformer
+from corekit.observability.loggable import Loggable
+from corekit.registry import SmartRegistry
+
+__all__ = ["BaseETLOrchestrator"]
+
+DEFAULT_BATCH_SIZE = 100
+
+
+@dataclass(frozen=True)
+class PipelineStage:
+    """
+    One stage of a pipeline: the attribute naming it, and what it must be.
+
+    Holds the validation for that stage as well as its description, so the
+    orchestrator asks each stage to check itself rather than unpacking a tuple
+    and reimplementing the rules inline.
+    """
+
+    attribute: str
+    base_class: type
+
+    @property
+    def label(self) -> str:
+        """
+        The stage's name as it reads in a message: "Extractor", "Loader".
+        """
+        return self.base_class.__name__.removeprefix("BaseETL")
+
+    def declared_on(self, cls: type) -> Any:
+        """
+        The stage as declared directly on a class, ignoring inheritance.
+        """
+        return cls.__dict__.get(self.attribute)
+
+    def validate(self, cls: type) -> None:
+        """
+        Confirm a pipeline declares this stage correctly.
+        """
+        stage = getattr(cls, self.attribute, None)
+        if stage is None:
+            raise TypeError(f"{cls.__name__} is missing its {self.attribute}. Set {self.attribute} = Your{self.label}.")
+        if not (isinstance(stage, type) and issubclass(stage, self.base_class)):
+            raise TypeError(
+                f"{cls.__name__}.{self.attribute} must be a {self.base_class.__name__} subclass, "
+                f"got {stage!r}. Assign the class itself, not an instance."
+            )
+
+
+class BaseETLOrchestrator(Loggable):
+    """
+    Runs an extract, transform and load pipeline.
+
+    Subclasses set ``extractor``, ``transformer`` and ``loader`` to the classes
+    for each stage. Every concrete pipeline registers itself by name.
+    """
+
+    extractor: ClassVar[type[BaseETLExtractor] | None] = None
+    transformer: ClassVar[type[BaseETLTransformer] | None] = None
+    loader: ClassVar[type[BaseETLLoader] | None] = None
+    batch_size: ClassVar[int] = DEFAULT_BATCH_SIZE
+
+    __registry__: SmartRegistry = SmartRegistry()
+
+    # Set on the intermediate class produced by the bracket form, which is not
+    # itself a usable pipeline and so must skip validation and registration.
+    __binding_only__: ClassVar[bool] = False
+
+    #: The stages every pipeline declares, in the order they run.
+    STAGES: ClassVar[list[PipelineStage]] = [
+        PipelineStage(attribute="extractor", base_class=BaseETLExtractor),
+        PipelineStage(attribute="transformer", base_class=BaseETLTransformer),
+        PipelineStage(attribute="loader", base_class=BaseETLLoader),
+    ]
+
+    def __init_subclass__(cls, **kwargs: Any) -> None:
+        """
+        Validate and register a pipeline at class-definition time.
+        """
+        super().__init_subclass__(**kwargs)
+
+        if cls.__dict__.get("__binding_only__"):
+            return
+
+        # A subclass that declares no stages at all is an intermediate base,
+        # not a pipeline; only complete ones are validated and registered.
+        if not any(stage.declared_on(cls) for stage in cls.STAGES):
+            return
+
+        for stage in cls.STAGES:
+            stage.validate(cls)
+
+        BaseETLOrchestrator.__registry__[cls.__name__] = cls
+
+    def __class_getitem__(cls, item: tuple[type, ...]) -> Any:
+        """
+        Bind the three stages, returning a pipeline class.
+
+        Sugar for declaring them as attributes; both forms produce the same class.
+        """
+        if not isinstance(item, tuple) or len(item) != 3:
+            raise TypeError(
+                f"{cls.__name__}[...] takes exactly three stages: {cls.__name__}[Extractor, Transformer, Loader]"
+            )
+
+        bound = {stage.attribute: value for stage, value in zip(cls.STAGES, item, strict=True)}
+        return type(f"{cls.__name__}_{item[0].__name__}", (cls,), bound)
+
+    @classmethod
+    def get_pipeline_types(cls) -> SmartRegistry:
+        """
+        Return the registry of known pipelines.
+        """
+        return cls.__registry__
+
+    @classmethod
+    def get_pipeline_by_name(cls, name: str) -> type["BaseETLOrchestrator"] | None:
+        """
+        Look up a pipeline class by name, using the registry's normalization.
+        """
+        return cls.__registry__.get(name)
+
+    def __init__(
+        self,
+        extractor: BaseETLExtractor | None = None,
+        transformer: BaseETLTransformer | None = None,
+        loader: BaseETLLoader | None = None,
+        batch_size: int | None = None,
+    ) -> None:
+        """
+        Build a pipeline, optionally overriding any stage with an instance.
+        """
+        super().__init__()
+        self.batch_size = batch_size if batch_size is not None else type(self).batch_size
+        self._extractor = extractor
+        self._transformer = transformer
+        self._loader = loader
+
+    def _build(self, override: Any, name: str) -> Any:
+        """
+        Return an explicitly supplied stage, or construct the declared class.
+        """
+        if override is not None:
+            return override
+        stage_class = getattr(type(self), name, None)
+        if stage_class is None:
+            raise TypeError(f"{type(self).__name__} has no {name}. Declare one, or pass {name}=...")
+        return stage_class()
+
+    async def run(self) -> None:
+        """
+        Extract, transform and load until the source is exhausted.
+
+        Items are processed one at a time and loaded whenever the batch reaches
+        ``batch_size``; whatever remains is flushed at the end.
+        """
+        extractor = self._build(self._extractor, "extractor")
+        transformer = self._build(self._transformer, "transformer")
+        loader = self._build(self._loader, "loader")
+
+        count = 0
+        async for extracted_item in extractor.extract():
+            loader.add_item(transformer.transform(extracted_item))
+            await loader.load(min_ops=self.batch_size)
+            count += 1
+
+        await loader.flush()
+        self.info(f"{type(self).__name__} processed {count} items")

corekit/etl/schemas.py

@@ -0,0 +1,22 @@
+from pydantic import BaseModel
+
+from corekit.etl.connection import BaseConnection
+from corekit.schemas.types import UnknownDict
+
+
+class BaseItem(BaseModel):
+    """
+    Base model for all ETL items
+    """
+
+    connection: BaseConnection
+
+
+class RawAPIResponse(BaseItem):
+    """
+    Base model for raw API responses
+    """
+
+    data: UnknownDict
+    response_time_ms: int
+    status_code: int