python-corekit 0.1.0__py3-none-any.whl

corekit/connections/sql/migration/registry.py

@@ -0,0 +1,166 @@
+"""
+Discovering, verifying and running migrations.
+
+On startup the registry checks that every previously-applied migration still
+matches the checksum it was applied under, then runs whatever is pending in
+version order.
+
+The checksum is taken over each operation's ``canonical()`` -- a structural
+description of what the operation does -- rather than over its rendered SQL.
+Reformatting the generated SQL therefore leaves history valid, while changing
+what a migration actually does invalidates it. A mismatch stops startup rather
+than being repaired silently, because a migration that has already run cannot
+be un-run by editing its source.
+"""
+
+import importlib
+import inspect
+import pkgutil
+from types import ModuleType
+
+from corekit.connections.sql.connection import SQLConnection
+from corekit.connections.sql.migration.base import Migration
+from corekit.connections.sql.migration.operations import DataOperation
+from corekit.connections.sql.migration.table import SchemaMigration
+from corekit.observability.benchmarkable import Benchmarkable
+
+__all__ = ["MigrationChecksumError", "MigrationRegistry"]
+
+
+class MigrationChecksumError(RuntimeError):
+    """
+    Raised when an applied migration no longer matches its recorded checksum.
+    """
+
+
+class MigrationRegistry(Benchmarkable):
+    """
+    An ordered, validated set of migrations.
+    """
+
+    def __init__(self, *migrations: Migration) -> None:
+        super().__init__()
+        self._migrations = self._validate(list(migrations))
+
+    @property
+    def migrations(self) -> list[Migration]:
+        """
+        The migrations, in version order.
+        """
+        return list(self._migrations)
+
+    @classmethod
+    def discover(cls, package: ModuleType) -> "MigrationRegistry":
+        """
+        Build a registry from every Migration subclass defined in a package.
+
+        Adding a file to that package is the whole registration step.
+        """
+        migrations: list[Migration] = []
+        for module_info in pkgutil.iter_modules(package.__path__, prefix=f"{package.__name__}."):
+            module = importlib.import_module(module_info.name)
+            for _, obj in inspect.getmembers(module, inspect.isclass):
+                if obj is Migration or not issubclass(obj, Migration) or obj.__module__ != module.__name__:
+                    continue
+                migrations.append(obj())
+        return cls(*migrations)
+
+    @staticmethod
+    def _validate(migrations: list[Migration]) -> list[Migration]:
+        """
+        Reject duplicates and incomplete definitions, and sort by version.
+        """
+        seen_versions: set[int] = set()
+        for migration in migrations:
+            # version and name are annotated on the base class but not
+            # assigned, so an undeclared one raises AttributeError rather than
+            # returning the default. Repr is avoided for the same reason.
+            version = getattr(migration, "version", None)
+            if version is None or getattr(migration, "name", None) is None:
+                raise ValueError(f"Migration {type(migration).__name__} is missing version or name.")
+            if version in seen_versions:
+                raise ValueError(f"Duplicate migration version: {migration.version}")
+            seen_versions.add(version)
+        return sorted(migrations, key=lambda m: m.version)
+
+    def _fetch_applied(self, conn: SQLConnection) -> dict[int, str]:
+        """
+        Return ``{version: checksum}`` for everything already applied.
+        """
+        return {row.version: row.checksum for row in conn.fetch_all(SchemaMigration)}
+
+    def _verify_checksums(self, applied: dict[int, str]) -> None:
+        """
+        Confirm no applied migration has been edited since it ran.
+        """
+        for migration in self._migrations:
+            stored = applied.get(migration.version)
+            if stored is None or stored == migration.checksum:
+                continue
+            raise MigrationChecksumError(
+                f"\n\nMIGRATION CHECKSUM MISMATCH - REFUSING TO START\n"
+                f"  Migration : v{migration.version} ({migration.name})\n"
+                f"  Stored    : {stored}\n"
+                f"  Current   : {migration.checksum}\n"
+                f"This migration was modified after it was applied to the database.\n"
+                f"Restore it to its original content, or add a new migration instead.\n"
+            )
+
+    def pending(self, conn: SQLConnection) -> list[Migration]:
+        """
+        The migrations not yet applied to this database.
+        """
+        applied = self._fetch_applied(conn)
+        return [m for m in self._migrations if m.version not in applied]
+
+    def _run_migration(self, conn: SQLConnection, migration: Migration) -> None:
+        """
+        Apply one migration and record it.
+        """
+        self.info(f"Applying migration v{migration.version}: {migration.name}")
+        for operation in migration.operations():
+            self.info(f"  {operation.canonical()}")
+            if isinstance(operation, DataOperation):
+                operation.execute(conn)
+            else:
+                conn.exec_ddl(operation.to_sql())
+
+        conn.insert(
+            SchemaMigration(
+                version=migration.version,
+                name=migration.name,
+                checksum=migration.checksum,
+            )
+        )
+        self.info(f"Migration v{migration.version} applied.")
+
+    def run(self, conn: SQLConnection) -> None:
+        """
+        Verify history, then apply anything pending.
+
+        A failing migration stops the run: later migrations are written against
+        the schema earlier ones produce, so continuing past a failure would
+        apply them to a schema they were never written for.
+
+        Note that DDL is committed per statement, so a migration that fails
+        part-way leaves its earlier operations applied but the migration itself
+        unrecorded. Prefer several small migrations over one large one, and
+        check the schema before re-running after a failure.
+        """
+        applied = self._fetch_applied(conn)
+        self._verify_checksums(applied)
+
+        pending = [m for m in self._migrations if m.version not in applied]
+        if not pending:
+            self.info("Database schema is up to date.")
+            return
+
+        self.info(f"{len(pending)} migration(s) pending.")
+        for migration in pending:
+            try:
+                self._run_migration(conn, migration)
+            except MigrationChecksumError:
+                raise
+            except Exception as exc:
+                self.error(f"Migration v{migration.version} ({migration.name}) failed: {exc}")
+                raise

corekit/connections/sql/migration/table.py

@@ -0,0 +1,27 @@
+"""
+The table recording which migrations have been applied.
+"""
+
+from datetime import datetime
+
+from sqlmodel import Field, SQLModel
+
+from corekit.utils.time import time_now
+
+__all__ = ["SchemaMigration"]
+
+
+class SchemaMigration(SQLModel, table=True):
+    """
+    One row per applied migration, with the checksum it was applied under.
+
+    Inherits SQLModel rather than NamedTable: this table is keyed by version,
+    and has no string id for NamedTable's registry to address it by.
+    """
+
+    __tablename__ = "schema_migration"
+
+    version: int = Field(primary_key=True)
+    name: str
+    checksum: str
+    applied_at: datetime = Field(default_factory=time_now)

corekit/connections/sql/query.py

@@ -0,0 +1,68 @@
+"""
+A chainable query builder over SQLModel's select().
+"""
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+from sqlmodel import desc as _desc
+from sqlmodel import select
+
+from corekit.connections.sql.table import NamedTable
+
+__all__ = ["Query"]
+
+
+class Query(BaseModel):
+    """
+    Accumulates filters, ordering and a limit, then builds a select statement.
+
+        Query(table=User).where(User.age >= 18).order_by(User.name).limit(10)
+    """
+
+    table: type[NamedTable]
+    queries: list[Any] = Field(default_factory=list)
+    order_by_field: Any = None
+    order_by_desc: bool = False
+    limit_count: int = -1
+
+    def where(self, query: Any) -> "Query":
+        """
+        Add a filter clause.
+        """
+        self.queries.append(query)
+        return self
+
+    def order_by(self, field: Any, desc: bool = False) -> "Query":
+        """
+        Order results by a field.
+        """
+        self.order_by_field = field
+        self.order_by_desc = desc
+        return self
+
+    def limit(self, limit: int) -> "Query":
+        """
+        Cap the number of rows returned. A non-positive value means no limit.
+        """
+        self.limit_count = limit
+        return self
+
+    def build(self) -> Any:
+        """
+        Produce the select statement.
+        """
+        statement = select(self.table)
+        for query in self.queries:
+            statement = statement.where(query)
+
+        if self.order_by_field is not None:
+            order_by_field = self.order_by_field
+            if self.order_by_desc:
+                order_by_field = _desc(order_by_field)
+            statement = statement.order_by(order_by_field)
+
+        if self.limit_count > 0:
+            statement = statement.limit(self.limit_count)
+
+        return statement

corekit/connections/sql/table.py

@@ -0,0 +1,96 @@
+"""
+Base table type.
+"""
+
+import json
+import logging
+from typing import Any
+
+from sqlmodel import SQLModel
+
+from corekit.registry import SmartRegistry
+
+__all__ = ["NamedTable"]
+
+logger = logging.getLogger(__name__)
+
+
+class NamedTable(SQLModel):
+    """
+    Base for tables that are addressable by a string id.
+
+    Subclasses register themselves by name, so a table class can be resolved
+    from a string -- useful for imports, exports and admin tooling that work
+    with table names rather than imported classes.
+    """
+
+    __registry__: SmartRegistry = SmartRegistry()
+
+    def __init_subclass__(cls, **kwargs: Any) -> None:
+        """
+        Register every subclass under its class name.
+        """
+        super().__init_subclass__(**kwargs)
+        NamedTable.__registry__[cls.__name__] = cls
+
+    @classmethod
+    def get_table_types(cls) -> SmartRegistry:
+        """
+        Return the registry of known table classes.
+        """
+        return cls.__registry__
+
+    @classmethod
+    def get_table_by_name(cls, name: str) -> type["NamedTable"] | None:
+        """
+        Look up a table class by name, using the registry's normalization.
+        """
+        return cls.__registry__.get(name)
+
+    @property
+    def table_name(self) -> str:
+        """
+        The name of the table this record belongs to.
+        """
+        return type(self).__name__
+
+    @classmethod
+    def get_display_name(cls) -> str:
+        """
+        A human-readable name for the table.
+        """
+        return getattr(cls, "__display_name__", None) or cls.__name__
+
+    @classmethod
+    def create_default(cls, identifier: str, **kwargs: Any) -> "NamedTable":
+        """
+        Build an instance carrying only its identifier and any supplied fields.
+        """
+        kwargs["id"] = kwargs.get("id") or identifier
+        logger.info(f"Creating default {cls.__name__} with id {kwargs['id']!r}")
+        return cls(**kwargs)
+
+    @staticmethod
+    def _serialize(value: Any) -> str:
+        return json.dumps(value)
+
+    @staticmethod
+    def _deserialize(value: str) -> Any:
+        try:
+            return json.loads(value)
+        except (TypeError, ValueError):
+            logger.error(f"Unable to deserialize value: {value!r}")
+            raise
+
+    def serialize(self, values: dict[str, Any], key: str) -> dict[str, Any]:
+        """
+        Replace ``values[key]`` with its JSON encoding.
+        """
+        values[key] = self._serialize(values.get(key))
+        return values
+
+    def deserialize(self, key: str) -> Any:
+        """
+        Decode the JSON stored in the named field.
+        """
+        return self._deserialize(getattr(self, key))

corekit/constants.py

@@ -0,0 +1,45 @@
+# Minutes
+ONE_MINUTE = 60
+TWO_MINUTES = ONE_MINUTE * 2
+THREE_MINUTES = ONE_MINUTE * 3
+FIVE_MINUTES = ONE_MINUTE * 5
+TEN_MINUTES = ONE_MINUTE * 10
+FIFTEEN_MINUTES = ONE_MINUTE * 15
+THIRTY_MINUTES = ONE_MINUTE * 30
+FORTY_FIVE_MINUTES = ONE_MINUTE * 45
+
+# Hours
+ONE_HOUR = ONE_MINUTE * 60
+TWO_HOURS = ONE_HOUR * 2
+THREE_HOURS = ONE_HOUR * 3
+FIVE_HOURS = ONE_HOUR * 5
+SIX_HOURS = ONE_HOUR * 6
+NINE_HOURS = ONE_HOUR * 9
+TWELVE_HOURS = ONE_HOUR * 12
+FIFTEEN_HOURS = ONE_HOUR * 15
+EIGHTEEN_HOURS = ONE_HOUR * 18
+
+# Days
+ONE_DAY = ONE_HOUR * 24
+TWO_DAYS = ONE_DAY * 2
+THREE_DAYS = ONE_DAY * 3
+FOUR_DAYS = ONE_DAY * 4
+FIVE_DAYS = ONE_DAY * 5
+SIX_DAYS = ONE_DAY * 6
+TEN_DAYS = ONE_DAY * 10
+FIFTEEN_DAYS = ONE_DAY * 15
+THIRTY_DAYS = ONE_DAY * 30
+FORTY_FIVE_DAYS = ONE_DAY * 45
+SIXTY_FIVE_DAYS = ONE_DAY * 60
+NINETY_FIVE_DAYS = ONE_DAY * 90
+ONE_HUNDRED_TWENTY_DAYS = ONE_DAY * 120
+ONE_HUNDRED_EIGHTY_DAYS = ONE_DAY * 180
+
+# Weeks
+ONE_WEEK = ONE_DAY * 7
+TWO_WEEKS = ONE_WEEK * 2
+THREE_WEEKS = ONE_WEEK * 3
+FOUR_WEEKS = ONE_WEEK * 4
+
+# Years
+ONE_YEAR = ONE_DAY * 365

corekit/crypto/__init__.py

@@ -0,0 +1 @@
+from .hasher import Hasher

corekit/crypto/constants.py

@@ -0,0 +1,7 @@
+HASH_JOINER = ":"
+EMPTY_HASH = ""
+
+# NOTE: a hardcoded CRYPTO_SALT previously lived here and is still present in this
+# repository's git history. It must be considered compromised and must never be
+# reintroduced. The salt is now supplied through configuration -- see
+# `corekit.config.CorekitSettings.crypto_salt` and `corekit.crypto.hasher.Hasher`.

corekit/crypto/enum.py

@@ -0,0 +1,11 @@
+from corekit.schemas.enum import StringEnum
+
+
+class SaltMethod(StringEnum):
+    PRE = "pre"
+    POST = "post"
+    BOTH = "BOTH"
+
+    @classmethod
+    def get_default(cls) -> "SaltMethod":
+        return cls.BOTH

corekit/crypto/hasher.py

@@ -0,0 +1,89 @@
+import hashlib
+from typing import Any
+
+from corekit.config import get_settings
+from corekit.crypto.constants import EMPTY_HASH, HASH_JOINER
+from corekit.crypto.enum import SaltMethod
+from corekit.observability.loggable import Loggable
+
+
+class MissingSaltError(RuntimeError):
+    """
+    Raised when hashing is attempted without a configured salt.
+    """
+
+
+class Hasher(Loggable):
+    """
+    Salted SHA-256 hashing.
+
+    The salt has no default. Supply it explicitly, or configure it via
+    ``COREKIT_CRYPTO_SALT`` / ``crypto_salt`` in a corekit config file. Resolution
+    is deferred until first use so that importing this module never fails.
+    """
+
+    def __init__(
+        self, joiner: str = HASH_JOINER, salt: str | None = None, salt_method: SaltMethod = SaltMethod.get_default()
+    ) -> None:
+        super().__init__()
+        self._joiner = joiner
+        self._explicit_salt = salt
+        self._salt_method = salt_method
+
+    @property
+    def _salt(self) -> str:
+        """
+        Resolve the salt, preferring the explicit argument over configuration.
+        """
+        salt = self._explicit_salt
+        if salt is None:
+            salt = get_settings().crypto.salt
+        if not salt:
+            raise MissingSaltError(
+                "No crypto salt configured. Pass salt=... to Hasher, set "
+                "COREKIT_CRYPTO_SALT, or add crypto_salt to your corekit config. "
+                'Generate one with: python -c "import secrets; print(secrets.token_hex(32))"'
+            )
+        return salt
+
+    def salt(self, *args: Any) -> str:
+        """
+        Given any number of inputs, combine and salt the input in preparation for hashing.
+        """
+        if len(args) < 1:
+            self.warning("No input to hash. Returning empty hash")
+            return EMPTY_HASH
+
+        salted: str = ""
+        # Add the prefix for all methods other than POST
+        if self._salt_method != SaltMethod.POST:
+            salted += f"{self._salt}{self._joiner}"
+
+        salted += self._joiner.join(str(arg) for arg in args)
+
+        # Add the suffix for all methods other than PRE
+        if self._salt_method != SaltMethod.PRE:
+            salted += f"{self._joiner}{self._salt}"
+
+        return salted
+
+    def hash(self, *args: Any) -> str:
+        """
+        Salt the inputs and return their SHA-256 digest.
+
+        With no input this returns EMPTY_HASH rather than the digest of the
+        empty string, so "nothing was hashed" stays distinguishable from a real
+        digest -- which is what salt() already reports and what EMPTY_HASH is
+        for.
+        """
+        to_hash = self.salt(*args)
+        if to_hash == EMPTY_HASH:
+            return EMPTY_HASH
+        return hashlib.sha256(to_hash.encode(self.encoding)).hexdigest()
+
+    @property
+    def encoding(self) -> str:
+        """
+        The encoding used to turn salted text into bytes.
+        """
+        return "utf-8"

corekit/data/__init__.py

@@ -0,0 +1,81 @@
+"""
+In-memory datasets with composable filtering.
+
+``Dataset`` holds ``__slots__``-based records sharing one schema, with O(1)
+lookup by id, in-place editing, schema evolution and clean pickling. Filtering
+uses expressions rather than callables::
+
+    from corekit.data import Dataset, Field
+
+    people = Dataset(id_key="name", schema=["name", "age"])
+    people.add({"name": "Ada", "age": 36})
+
+    adults = people.filter(Field("age") >= 18)
+    people.get_record("Ada").age
+
+``Field`` is an alias of ``FieldExpression``: ``Field("a") > Field("b")``
+compares two fields, while ``Field("a") > 3`` compares against a value.
+
+Depends only on the standard library.
+"""
+
+from corekit.data.dataset import Dataset
+from corekit.data.expressions import (
+    And,
+    Comparison,
+    Contains,
+    Equals,
+    Expression,
+    FieldExpression,
+    GreaterThan,
+    GreaterThanOrEquals,
+    IsIn,
+    LessThan,
+    LessThanOrEquals,
+    Not,
+    NotEquals,
+    Or,
+    ValueExpression,
+)
+from corekit.data.record import BaseRecord, is_valid_key, resolve_default
+from corekit.data.stats import (
+    CategoricalFieldStats,
+    DatasetStats,
+    FieldDescription,
+    FieldStats,
+    NumericFieldStats,
+    ValueCounts,
+)
+
+# The expression used to name a field. Field("age") reads better than
+# FieldExpression("age") at a call site.
+Field = FieldExpression
+
+__all__ = [
+    "And",
+    "BaseRecord",
+    "CategoricalFieldStats",
+    "Comparison",
+    "Contains",
+    "Dataset",
+    "DatasetStats",
+    "Equals",
+    "Expression",
+    "Field",
+    "FieldDescription",
+    "FieldExpression",
+    "FieldStats",
+    "GreaterThan",
+    "GreaterThanOrEquals",
+    "IsIn",
+    "LessThan",
+    "LessThanOrEquals",
+    "Not",
+    "NotEquals",
+    "NumericFieldStats",
+    "Or",
+    "ValueCounts",
+    "ValueExpression",
+    "is_valid_key",
+    "resolve_default",
+]