python-corekit 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

corekit/data/stats.py

@@ -1,157 +1,159 @@
-import statistics
-from collections import Counter
-from dataclasses import dataclass
-from dataclasses import field as dataclass_field
-from typing import Any, Iterator
-
-
-@dataclass(slots=True)
-class FieldStats:
-    """
-    Common shape shared by every field's statistics
-    """
-
-    field: Any
-    count: int
-    missing: int
-
-
-@dataclass(slots=True)
-class NumericFieldStats(FieldStats):
-    """
-    Stats for a field where every non-missing value is int/float (not bool)
-    """
-
-    mean: float
-    min: Any
-    max: Any
-    stdev: float
-
-
-@dataclass(slots=True)
-class CategoricalFieldStats(FieldStats):
-    """
-    Stats for any field that isn't purely numeric
-    """
-
-    unique: int
-    top: Any
-    freq: int
-
-
-@dataclass(slots=True)
-class ValueCounts:
-    """
-    How often each distinct value of a field occurs, most frequent first
-    """
-
-    field: Any
-    counts: dict[Any, int]
-
-    def most_common(self, n: int | None = None) -> list[tuple[Any, int]]:
-        items = list(self.counts.items())
-        return items[:n] if n is not None else items
-
-    def top(self) -> tuple[Any, int] | None:
-        return next(iter(self.counts.items()), None)
-
-    def __iter__(self) -> Iterator[tuple[Any, int]]:
-        return iter(self.counts.items())
-
-    def __len__(self) -> int:
-        return len(self.counts)
-
-    def __getitem__(self, value: Any) -> int:
-        return self.counts[value]
-
-
-@dataclass(slots=True)
-class FieldDescription:
-    field: Any
-    values: list[Any] = dataclass_field(default_factory=list)
-    non_missing: list[Any] = dataclass_field(default_factory=list)
-    num_values: int = dataclass_field(default=0)
-    num_non_missing: int = dataclass_field(default=0)
-
-    @classmethod
-    def from_data(cls, data: list[Any], field: Any) -> "FieldDescription":
-        instance = cls(field)
-        for record in data:
-            instance.add(record[field])
-        return instance
-
-    @property
-    def missing(self) -> int:
-        return self.num_values - self.num_non_missing
-
-    @property
-    def is_numeric(self) -> bool:
-        return all(isinstance(v, (int, float)) and not isinstance(v, bool) for v in self.non_missing)
-
-    def to_field_stats(self) -> FieldStats:
-        if self.is_numeric:
-            return NumericFieldStats(
-                field=self.field,
-                count=self.num_non_missing,
-                missing=self.missing,
-                mean=statistics.fmean(self.non_missing),
-                min=min(self.non_missing),
-                max=max(self.non_missing),
-                stdev=statistics.stdev(self.non_missing) if self.num_non_missing > 1 else 0.0,
-            )
-
-        counts = Counter(self.non_missing)
-        top_value, top_count = counts.most_common(1)[0] if counts else (None, 0)
-        return CategoricalFieldStats(
-            field=self.field,
-            count=self.num_non_missing,
-            missing=self.missing,
-            unique=len(counts),
-            top=top_value,
-            freq=top_count,
-        )
-
-    def add(self, value: Any) -> None:
-        self.values.append(value)
-        self.num_values += 1
-        if value is not None:
-            self.non_missing.append(value)
-            self.num_non_missing += 1
-
-
-class DatasetStats:
-    """
-    Read-only field statistics for a Dataset's current records.
-
-    Computed fresh on every call from the dataset's public interface (never
-    cached), so there's nothing here that can go stale after a mutation --
-    the same reasoning that ruled out persistent secondary indexes applies:
-    recomputing is cheap and always correct, caching is not free and can lie.
-    """
-
-    def __init__(self, data: list[Any], schema: tuple[Any, ...] | None = None) -> None:
-        self._data = data
-        self._schema = schema or ()
-
-    def value_counts(self, field: Any) -> ValueCounts:
-        """
-        How often each distinct value of `field` occurs, most frequent first
-        """
-        counts = dict(Counter(rec[field] for rec in self._data).most_common())
-        return ValueCounts(field=field, counts=counts)
-
-    def describe(self, field: Any | None = None) -> dict[Any, FieldStats]:
-        """
-        Summary statistics per field, pandas-.describe()-style.
-
-        Numeric fields (int/float, excluding bool) get a NumericFieldStats
-        (mean/min/max/stdev). Everything else gets a CategoricalFieldStats
-        (unique/top/freq). Pass `field` to describe just one; omit to
-        describe every field in the schema.
-        """
-        fields = (field,) if field is not None else self._schema
-        stats = {}
-        for f in fields:
-            field_description = FieldDescription.from_data(self._data, f)
-            stats[f] = field_description.to_field_stats()
-        return stats
+import statistics
+from collections import Counter
+from dataclasses import dataclass
+from dataclasses import field as dataclass_field
+from typing import Any, Iterator
+
+from corekit.utils.coercion import safe_tuple
+
+
+@dataclass(slots=True)
+class FieldStats:
+    """
+    Common shape shared by every field's statistics
+    """
+
+    field: Any
+    count: int
+    missing: int
+
+
+@dataclass(slots=True)
+class NumericFieldStats(FieldStats):
+    """
+    Stats for a field where every non-missing value is int/float (not bool)
+    """
+
+    mean: float
+    min: Any
+    max: Any
+    stdev: float
+
+
+@dataclass(slots=True)
+class CategoricalFieldStats(FieldStats):
+    """
+    Stats for any field that isn't purely numeric
+    """
+
+    unique: int
+    top: Any
+    freq: int
+
+
+@dataclass(slots=True)
+class ValueCounts:
+    """
+    How often each distinct value of a field occurs, most frequent first
+    """
+
+    field: Any
+    counts: dict[Any, int]
+
+    def most_common(self, n: int | None = None) -> list[tuple[Any, int]]:
+        items = list(self.counts.items())
+        return items[:n] if n is not None else items
+
+    def top(self) -> tuple[Any, int] | None:
+        return next(iter(self.counts.items()), None)
+
+    def __iter__(self) -> Iterator[tuple[Any, int]]:
+        return iter(self.counts.items())
+
+    def __len__(self) -> int:
+        return len(self.counts)
+
+    def __getitem__(self, value: Any) -> int:
+        return self.counts[value]
+
+
+@dataclass(slots=True)
+class FieldDescription:
+    field: Any
+    values: list[Any] = dataclass_field(default_factory=list)
+    non_missing: list[Any] = dataclass_field(default_factory=list)
+    num_values: int = dataclass_field(default=0)
+    num_non_missing: int = dataclass_field(default=0)
+
+    @classmethod
+    def from_data(cls, data: list[Any], field: Any) -> "FieldDescription":
+        instance = cls(field)
+        for record in data:
+            instance.add(record[field])
+        return instance
+
+    @property
+    def missing(self) -> int:
+        return self.num_values - self.num_non_missing
+
+    @property
+    def is_numeric(self) -> bool:
+        return all(isinstance(v, (int, float)) and not isinstance(v, bool) for v in self.non_missing)
+
+    def to_field_stats(self) -> FieldStats:
+        if self.is_numeric:
+            return NumericFieldStats(
+                field=self.field,
+                count=self.num_non_missing,
+                missing=self.missing,
+                mean=statistics.fmean(self.non_missing),
+                min=min(self.non_missing),
+                max=max(self.non_missing),
+                stdev=statistics.stdev(self.non_missing) if self.num_non_missing > 1 else 0.0,
+            )
+
+        counts = Counter(self.non_missing)
+        top_value, top_count = counts.most_common(1)[0] if counts else (None, 0)
+        return CategoricalFieldStats(
+            field=self.field,
+            count=self.num_non_missing,
+            missing=self.missing,
+            unique=len(counts),
+            top=top_value,
+            freq=top_count,
+        )
+
+    def add(self, value: Any) -> None:
+        self.values.append(value)
+        self.num_values += 1
+        if value is not None:
+            self.non_missing.append(value)
+            self.num_non_missing += 1
+
+
+class DatasetStats:
+    """
+    Read-only field statistics for a Dataset's current records.
+
+    Computed fresh on every call from the dataset's public interface (never
+    cached), so there's nothing here that can go stale after a mutation --
+    the same reasoning that ruled out persistent secondary indexes applies:
+    recomputing is cheap and always correct, caching is not free and can lie.
+    """
+
+    def __init__(self, data: list[Any], schema: tuple[Any, ...] | None = None) -> None:
+        self._data = data
+        self._schema = safe_tuple(schema)
+
+    def value_counts(self, field: Any) -> ValueCounts:
+        """
+        How often each distinct value of `field` occurs, most frequent first
+        """
+        counts = dict(Counter(rec[field] for rec in self._data).most_common())
+        return ValueCounts(field=field, counts=counts)
+
+    def describe(self, field: Any | None = None) -> dict[Any, FieldStats]:
+        """
+        Summary statistics per field, pandas-.describe()-style.
+
+        Numeric fields (int/float, excluding bool) get a NumericFieldStats
+        (mean/min/max/stdev). Everything else gets a CategoricalFieldStats
+        (unique/top/freq). Pass `field` to describe just one; omit to
+        describe every field in the schema.
+        """
+        fields = (field,) if field is not None else self._schema
+        stats = {}
+        for f in fields:
+            field_description = FieldDescription.from_data(self._data, f)
+            stats[f] = field_description.to_field_stats()
+        return stats

corekit/decorators/__init__.py

@@ -1,2 +1,2 @@
-from .exception_handling import exception_handler
-from .warnings import deprecated
+from .exception_handling import exception_handler
+from .warnings import deprecated

corekit/decorators/exception_handling.py

@@ -3,6 +3,7 @@ import logging
 from typing import Any, Callable
 
 from corekit.utils import raise_exc, void
+from corekit.utils.coercion import safe_tuple
 
 logger = logging.getLogger(__name__)
 
@@ -20,7 +21,7 @@ def exception_handler(
         callback: called when an exception is raised that is not ignored.
         ignore_callback: called when a suppressed exception is raised.
     """
-    exceptions_to_ignore = tuple(ignore) if ignore else tuple()
+    exceptions_to_ignore = safe_tuple(ignore)
 
     def decorator(func):
         @functools.wraps(func)

corekit/etl/connection.py

@@ -1,44 +1,44 @@
-from enum import Enum
-from typing import NamedTuple
-
-
-class ConnectionDetails(NamedTuple):
-    name: str
-    description: str
-
-
-class BaseConnection(Enum):
-    """
-    Base enum for ETL Connections. All ETL connections must inherit from this class.
-
-    Example:
-        class MyConnectionEnum(BaseConnection):
-            MY_FIRST_CONNECTION = ConnectionDetails(
-                name="my_first_connection",
-                description="My First Connection"
-            )
-            ANOTHER_CONNECTION = ConnectionDetails(
-                name="another_connection",
-                description="Another Connection"
-            )
-    """
-
-    @classmethod
-    def from_name(cls, name: str) -> "BaseConnection":
-        # TODO: Make sure this is efficient
-        for connection in cls:
-            if connection.value.name == name:
-                return connection
-        raise ValueError(f"Connection {name} not found")
-
-    def get_name(self) -> str:
-        """
-        Helper method for fetching the name of the connection
-        """
-        return self.value.name
-
-    def get_description(self) -> str:
-        """
-        Helper method for fetching the name of the connection
-        """
-        return self.value.description
+from enum import Enum
+from typing import NamedTuple
+
+
+class ConnectionDetails(NamedTuple):
+    name: str
+    description: str
+
+
+class BaseConnection(Enum):
+    """
+    Base enum for ETL Connections. All ETL connections must inherit from this class.
+
+    Example:
+        class MyConnectionEnum(BaseConnection):
+            MY_FIRST_CONNECTION = ConnectionDetails(
+                name="my_first_connection",
+                description="My First Connection"
+            )
+            ANOTHER_CONNECTION = ConnectionDetails(
+                name="another_connection",
+                description="Another Connection"
+            )
+    """
+
+    @classmethod
+    def from_name(cls, name: str) -> "BaseConnection":
+        # TODO: Make sure this is efficient
+        for connection in cls:
+            if connection.value.name == name:
+                return connection
+        raise ValueError(f"Connection {name} not found")
+
+    def get_name(self) -> str:
+        """
+        Helper method for fetching the name of the connection
+        """
+        return self.value.name
+
+    def get_description(self) -> str:
+        """
+        Helper method for fetching the name of the connection
+        """
+        return self.value.description

corekit/events/websocket.py

@@ -13,6 +13,7 @@ from typing import Any
 from corekit.events.enum import MessageField
 from corekit.events.reader import RedisChannelReader
 from corekit.observability.loggable import Loggable
+from corekit.utils.coercion import safe_dict, safe_list
 
 __all__ = ["WebSocketBridge"]
 
@@ -44,7 +45,7 @@ class WebSocketBridge(Loggable):
         super().__init__()
         self.channel = channel
         self.websocket = websocket
-        self.terminal_statuses = terminal_statuses or []
+        self.terminal_statuses = safe_list(terminal_statuses)
         self.timeout = timeout
         self._reader = RedisChannelReader(channel, url=url)
 
@@ -56,7 +57,7 @@ class WebSocketBridge(Loggable):
             return False
         if payload.get(MessageField.TYPE.value) != STATUS_UPDATE_EVENT:
             return False
-        data = payload.get(MessageField.DATA.value) or {}
+        data = safe_dict(payload.get(MessageField.DATA.value))
         return data.get(MessageField.STATUS.value) in self.terminal_statuses
 
     def _expired(self, started: float) -> bool:

corekit/exceptions/__init__.py

@@ -0,0 +1,18 @@
+"""
+Exception base classes.
+
+``CustomException`` carries a message and an optional underlying error;
+``CustomHTTPException`` is its FastAPI-facing counterpart.
+"""
+
+from corekit.exceptions.base import CustomException, CustomHTTPException, ExponentialBackoffTimeoutException
+from corekit.exceptions.types import ArbitraryCustomExceptionType, CustomExceptionType, CustomHTTPExceptionType
+
+__all__ = [
+    "ArbitraryCustomExceptionType",
+    "CustomException",
+    "CustomExceptionType",
+    "CustomHTTPException",
+    "CustomHTTPExceptionType",
+    "ExponentialBackoffTimeoutException",
+]

corekit/http/__init__.py

@@ -0,0 +1,13 @@
+"""
+HTTP client building blocks.
+
+``BaseApiClient`` wraps httpx with retries; ``ExponentialBackoff`` is the retry
+policy behind it and is useful on its own for any operation that should back off
+rather than hammer.
+"""
+
+from corekit.http.client import BaseApiClient, URLMismatchError
+from corekit.http.exponential_backoff import ExponentialBackoff
+from corekit.http.response import BaseApiResponse
+
+__all__ = ["BaseApiClient", "BaseApiResponse", "ExponentialBackoff", "URLMismatchError"]

corekit/jobs/__init__.py

@@ -0,0 +1,26 @@
+"""
+Background tasks, independent of any queue.
+
+Define a task by subclassing ``Task`` or ``ScheduledTask``; it registers itself
+by name when the class is defined. A worker calls ``run_task(name, payload)``,
+which builds the task from its own registry and runs it.
+
+Nothing here imports a queue library. Bind these to RQ, Celery or anything else
+with a small adapter, and keep the tasks themselves portable.
+
+Arguments cross the queue as JSON, never as a serialized object: pickle and dill
+execute code while loading, so a queue holding objects turns write access into
+code execution in a worker.
+"""
+
+from corekit.jobs.registry import TaskNotRegisteredError, task_registry
+from corekit.jobs.runner import run_task
+from corekit.jobs.task import ScheduledTask, Task
+
+__all__ = [
+    "ScheduledTask",
+    "Task",
+    "TaskNotRegisteredError",
+    "run_task",
+    "task_registry",
+]

corekit/jobs/registry.py

@@ -0,0 +1,87 @@
+"""
+The task registry.
+
+A worker rebuilds a task from a name, so the name has to resolve to a class in
+the worker's *own* code. That is the whole security property: the queue chooses
+which registered task runs, never what code exists.
+
+``SmartRegistry`` normalizes keys, so ``"NightlyBackup"``, ``"nightly_backup"``
+and ``"nightly backup"`` all address the same task.
+"""
+
+from typing import TYPE_CHECKING, Any
+
+from corekit.registry import SmartRegistry
+
+if TYPE_CHECKING:
+    from corekit.jobs.task import Task
+
+__all__ = ["TaskNotRegisteredError", "task_registry"]
+
+
+class TaskNotRegisteredError(LookupError):
+    """
+    Raised when a queued name matches no registered task.
+    """
+
+
+class _TaskRegistry:
+    """
+    Every concrete ``Task`` subclass, keyed by name.
+
+    Registration happens when a subclass is defined, so importing the module
+    that declares a task is what makes it runnable. A worker must therefore
+    import its tasks before it starts consuming the queue.
+    """
+
+    def __init__(self) -> None:
+        self._registry = SmartRegistry()
+
+    def register(self, task_class: type["Task"]) -> None:
+        """
+        Register a task class under its name.
+        """
+        self._registry[task_class.task_name or task_class.__name__] = task_class
+
+    def get(self, name: str) -> type["Task"]:
+        """
+        Look up a task class by name.
+
+        Raises:
+            TaskNotRegisteredError: If nothing is registered under that name.
+                Usually means the module declaring the task was never imported
+                by this worker.
+        """
+        task_class = self._registry.get(name)
+        if task_class is None:
+            known = ", ".join(sorted(self._registry.keys())) or "none"
+            raise TaskNotRegisteredError(
+                f"No task registered as '{name}'. Import the module that defines it before "
+                f"running the worker. Registered: {known}."
+            )
+        return task_class
+
+    def build(self, name: str) -> "Task":
+        """
+        Construct the task registered under ``name``.
+        """
+        return self.get(name)()
+
+    def __contains__(self, name: str) -> bool:
+        return name in self._registry
+
+    def __len__(self) -> int:
+        return len(self._registry)
+
+    def __iter__(self) -> Any:
+        return iter(self._registry)
+
+    @property
+    def names(self) -> list[str]:
+        """
+        The registered task names, normalized and sorted.
+        """
+        return sorted(self._registry.keys())
+
+
+task_registry = _TaskRegistry()