python-corekit 0.1.0__py3-none-any.whl

corekit/connections/redis/connection.py

@@ -0,0 +1,239 @@
+"""
+Redis connections.
+
+    with RedisConnection() as cache:
+        cache.set("user:1", {"name": "Ada"}, ex=3600)
+        cache.get("user:1")            # -> {"name": "Ada"}
+
+Values are JSON-encoded on the way in and decoded on the way out, so ordinary
+Python data round-trips without the caller doing anything. JSON rather than
+pickle: cached data is frequently written by one process and read by another,
+and an unpickle of someone else's bytes executes code.
+
+Connection details come from the constructor, then configuration, then a local
+default. Nothing is read at import time.
+"""
+
+import json
+import logging
+from typing import Any
+
+import redis
+from redis import asyncio as aioredis
+
+from corekit.config import get_settings
+from corekit.connections import Connectable
+
+__all__ = ["RedisConnection", "RedisNotConnectedError"]
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_URL = "redis://localhost:6379/0"
+
+
+class RedisNotConnectedError(RuntimeError):
+    """
+    Raised when a client is used before its connection is opened.
+    """
+
+
+def _encode(value: Any) -> str:
+    """
+    Encode a value for storage.
+    """
+    return json.dumps(value)
+
+
+def _decode(raw: Any) -> Any:
+    """
+    Decode a stored value, returning it unchanged if it is not JSON.
+
+    Keys written by other tools are common in a shared Redis, so a value that
+    does not parse is returned as-is rather than raising.
+    """
+    if raw is None:
+        return None
+    if isinstance(raw, bytes):
+        raw = raw.decode("utf-8", errors="replace")
+    try:
+        return json.loads(raw)
+    except (TypeError, ValueError):
+        return raw
+
+
+class RedisConnection(Connectable):
+    """
+    A Connectable wrapper over a Redis client.
+
+    The sync client is shared per URL for the life of the process, because a
+    Redis client is a connection pool and building one per caller defeats it.
+    ``_disconnect`` is therefore a no-op: one caller finishing must not close a
+    pool others are still using. Async clients are per-instance, since they are
+    bound to a running event loop.
+    """
+
+    _shared_clients: dict[str, Any] = {}
+
+    def __init__(self, url: str | None = None, safe: bool = False, **kwargs: Any) -> None:
+        """
+        :param url: a redis:// URL. Falls back to configuration, then localhost.
+        :param safe: log connection failures instead of raising.
+        """
+        super().__init__(**kwargs)
+        self._url = url or get_settings().redis.url or DEFAULT_URL
+        self._safe = safe
+        self._async_client: Any = None
+
+    @property
+    def url(self) -> str:
+        """
+        The URL this connection was built with.
+        """
+        return self._url
+
+    @property
+    def is_connected(self) -> bool:
+        return RedisConnection._shared_clients.get(self._url) is not None
+
+    @property
+    def is_async_connected(self) -> bool:
+        return self._async_client is not None
+
+    @property
+    def client(self) -> Any:
+        """
+        The sync client, raising if the connection was never opened.
+        """
+        existing = RedisConnection._shared_clients.get(self._url)
+        if existing is None:
+            raise RedisNotConnectedError("RedisConnection is not connected. Call connect() or use a with block.")
+        return existing
+
+    @property
+    def async_client(self) -> Any:
+        """
+        The async client, raising if the async connection was never opened.
+        """
+        if self._async_client is None:
+            raise RedisNotConnectedError("RedisConnection async client is not connected.")
+        return self._async_client
+
+    def _build(self, is_async: bool) -> Any:
+        """
+        Build a client, honouring ``safe`` when the connection cannot be made.
+        """
+        client_cls = aioredis.Redis if is_async else redis.Redis
+        try:
+            return client_cls.from_url(self._url)
+        except Exception as exc:
+            if self._safe:
+                self.warning(f"Could not connect to Redis at {self._url}: {exc}")
+                return None
+            raise
+
+    def _connect(self) -> None:
+        if RedisConnection._shared_clients.get(self._url) is None:
+            self.info(f"Creating shared Redis client for {self._url}")
+            RedisConnection._shared_clients[self._url] = self._build(is_async=False)
+
+    def _disconnect(self) -> None:
+        """
+        Deliberately a no-op: the sync client is a process-wide pool.
+        """
+
+    async def _async_connect(self) -> None:
+        self._async_client = self._build(is_async=True)
+
+    async def _async_disconnect(self) -> None:
+        if self._async_client is not None:
+            try:
+                await self._async_client.aclose()
+            except Exception as exc:
+                self.debug(f"Error closing async Redis client: {exc}")
+            self._async_client = None
+
+    @classmethod
+    def close_shared_clients(cls) -> None:
+        """
+        Close every shared sync client. For process shutdown and tests.
+
+        Logs through the module logger rather than Loggable, since there is no
+        instance here to carry one.
+        """
+        for url, client in list(cls._shared_clients.items()):
+            try:
+                client.close()
+            except Exception as exc:
+                logger.debug("Error closing Redis client for %s: %s", url, exc)
+            del cls._shared_clients[url]
+
+    # ------------------------------------------------------------------
+    # Sync API
+    # ------------------------------------------------------------------
+
+    def get(self, key: str) -> Any:
+        return _decode(self.client.get(key))
+
+    def set(self, key: str, value: Any, **kwargs: Any) -> None:
+        self.client.set(key, _encode(value), **kwargs)
+
+    def delete(self, *keys: str) -> None:
+        self.client.delete(*keys)
+
+    def ping(self) -> bool:
+        return bool(self.client.ping())
+
+    def exists(self, key: str) -> bool:
+        return bool(self.client.exists(key))
+
+    def expire(self, key: str, seconds: int) -> None:
+        self.client.expire(key, seconds)
+
+    def ttl(self, key: str) -> int:
+        return self.client.ttl(key)
+
+    def incr(self, key: str, amount: int = 1) -> int:
+        return self.client.incr(key, amount)
+
+    def decr(self, key: str, amount: int = 1) -> int:
+        return self.client.decr(key, amount)
+
+    def keys(self, pattern: str = "*") -> list[Any]:
+        return self.client.keys(pattern)
+
+    def hget(self, key: str, field: str) -> Any:
+        return _decode(self.client.hget(key, field))
+
+    def hset(self, key: str, field: str, value: Any) -> None:
+        self.client.hset(key, field, _encode(value))
+
+    def hdel(self, key: str, *fields: str) -> None:
+        self.client.hdel(key, *fields)
+
+    def publish(self, channel: str, message: Any) -> int:
+        return self.client.publish(channel, _encode(message))
+
+    def pubsub(self) -> Any:
+        return self.client.pubsub()
+
+    # ------------------------------------------------------------------
+    # Async API
+    # ------------------------------------------------------------------
+
+    async def aget(self, key: str) -> Any:
+        return _decode(await self.async_client.get(key))
+
+    async def aset(self, key: str, value: Any, **kwargs: Any) -> None:
+        await self.async_client.set(key, _encode(value), **kwargs)
+
+    async def asetex(self, key: str, seconds: int, value: Any) -> None:
+        await self.async_client.setex(key, seconds, _encode(value))
+
+    async def adelete(self, *keys: str) -> None:
+        await self.async_client.delete(*keys)
+
+    async def akeys(self, pattern: str = "*") -> list[Any]:
+        return await self.async_client.keys(pattern)
+
+    async def apublish(self, channel: str, message: Any) -> int:
+        return await self.async_client.publish(channel, _encode(message))

corekit/connections/registry.py

@@ -0,0 +1,80 @@
+"""
+Per-thread connection storage.
+
+A thread keeps at most one open connection of each type, so a whole call tree
+shares one session rather than opening a connection per function.
+
+    registry.get(SQLConnection)   # this thread's connection, or None
+
+    registry.seed()               # open one of every connection type
+    registry.teardown()           # close them again
+
+Connection classes are the keys. There is no list of known types to maintain:
+``Connectable`` registers its subclasses as they are defined, so ``seed`` and
+``teardown`` cover whatever the application has imported.
+"""
+
+import threading
+
+from corekit.concurrency import ThreadLocalRegistry
+from corekit.connections.connectable import Connectable
+
+__all__ = ["ConnectionRegistry", "registry"]
+
+
+class ConnectionRegistry(ThreadLocalRegistry):
+    """
+    Per-thread storage that opens and closes the connections it holds.
+
+    Adds lifecycle to ``ThreadLocalRegistry``: the base class stores whatever it
+    is given, while this one knows its values are ``Connectable`` and can
+    therefore open a full set for a thread and close them again afterwards.
+    """
+
+    @staticmethod
+    def known_types() -> tuple[type[Connectable], ...]:
+        """
+        Every concrete Connectable defined so far.
+
+        A class that has not been imported has no entry, which is correct:
+        nothing can be holding a connection of a type that does not yet exist.
+        """
+        return tuple(Connectable.get_connection_types().values())
+
+    def seed(self, *connection_classes: type[Connectable], safe: bool = True) -> None:
+        """
+        Open and store a connection of each type, defaulting to all known types.
+
+        Failures are logged rather than raised unless ``safe`` is False, so one
+        unreachable service does not stop a worker thread from starting.
+        """
+        for connection_cls in connection_classes or self.known_types():
+            if self.get(connection_cls) is not None:
+                continue
+            try:
+                connection = connection_cls()
+                connection.connect()
+                self.set(connection_cls, connection)
+                self.debug(f"Seeded {connection_cls.__name__} on thread {threading.current_thread().name}")
+            except Exception as exc:
+                if not safe:
+                    raise
+                self.warning(f"Could not seed {connection_cls.__name__}: {exc}")
+
+    def teardown(self, *connection_classes: type[Connectable]) -> None:
+        """
+        Disconnect and forget each connection, defaulting to all known types.
+        """
+        for connection_cls in connection_classes or self.known_types():
+            connection = self.get(connection_cls)
+            if connection is None:
+                continue
+            try:
+                connection.disconnect()
+            except Exception as exc:
+                self.warning(f"Error disconnecting {connection_cls.__name__}: {exc}")
+            self.clear(connection_cls)
+
+
+# One object; each thread sees its own connections.
+registry: ConnectionRegistry = ConnectionRegistry()

corekit/connections/sql/__init__.py

@@ -0,0 +1,10 @@
+"""
+SQL building blocks: a Connectable session, a query builder and a base table.
+"""
+
+from corekit.connections.sql.connection import PaginatedResult, SQLConnection
+from corekit.connections.sql.fields import JSONBField, PydanticJSON
+from corekit.connections.sql.query import Query
+from corekit.connections.sql.table import NamedTable
+
+__all__ = ["JSONBField", "NamedTable", "PaginatedResult", "PydanticJSON", "Query", "SQLConnection"]

corekit/connections/sql/connection.py

@@ -0,0 +1,342 @@
+"""
+SQL connections.
+
+``SQLConnection`` is a ``Connectable`` over a SQLModel session. The database URL
+is resolved in this order: the constructor argument, ``COREKIT_DATABASE_URL`` or
+``database_url`` in configuration, and finally an on-disk SQLite file. Nothing is
+read at import time, so importing this module never fails for want of a URL.
+
+    with SQLConnection.sqlite("app.db") as db:
+        db.insert(User(id="ada"))
+        user = db.fetch_one_by_id(User, "ada")
+
+Engines are cached per URL for the life of the process; sessions are per
+connection instance.
+"""
+
+import logging
+import uuid
+from pathlib import Path
+from typing import Any, Generic, Iterator, NamedTuple, TypeVar
+
+from sqlalchemy import Engine, func, text
+from sqlmodel import Session, SQLModel, create_engine, select
+
+from corekit.config import get_settings
+from corekit.connections import Connectable
+from corekit.connections.sql.query import Query
+from corekit.connections.sql.table import NamedTable
+
+__all__ = ["PaginatedResult", "SQLConnection"]
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+DEFAULT_SQLITE_DIR = Path.home() / ".corekit" / "databases"
+POOL_SIZE = 10
+MAX_OVERFLOW = 20
+POOL_RECYCLE = 3600
+
+
+class PaginatedResult(NamedTuple, Generic[T]):
+    """
+    A page of rows, plus the total number of rows matching the query.
+    """
+
+    rows: list[T]
+    total: int
+
+
+# Engines are expensive and hold a connection pool, so one is kept per URL and
+# shared by every connection using it. Sessions remain per-instance.
+_engines: dict[str, Engine] = {}
+
+
+def _get_engine(url: str) -> Engine:
+    """
+    Return the shared engine for a URL, building it on first use.
+    """
+    engine = _engines.get(url)
+    if engine is None:
+        logger.info(f"Creating SQLAlchemy engine for {url.split('://')[0]}")
+        if url.startswith("sqlite"):
+            # SQLite has no server-side pool, and its default single-thread
+            # check trips whenever a session is used off the creating thread.
+            engine = create_engine(url, connect_args={"check_same_thread": False})
+        else:
+            engine = create_engine(
+                url,
+                pool_size=POOL_SIZE,
+                max_overflow=MAX_OVERFLOW,
+                pool_recycle=POOL_RECYCLE,
+                pool_pre_ping=True,
+            )
+        _engines[url] = engine
+    return engine
+
+
+class SQLConnection(Connectable):
+    """
+    A session-scoped SQL connection.
+
+    Implements the Connectable lifecycle, so it works with ``with`` and
+    ``async with`` and can be nested by connection-injecting decorators.
+    """
+
+    def __init__(self, connection_url: str | None = None, **kwargs: Any) -> None:
+        """
+        Build a connection. Without a URL, configuration is consulted, then a
+        local SQLite file is used.
+        """
+        super().__init__(**kwargs)
+        self._connection_url = connection_url or self._default_url()
+        self._session: Session | None = None
+
+    @staticmethod
+    def _default_url() -> str:
+        """
+        Resolve a URL from configuration, falling back to local SQLite.
+        """
+        configured = get_settings().database.url
+        if configured:
+            return configured
+        DEFAULT_SQLITE_DIR.mkdir(parents=True, exist_ok=True)
+        return f"sqlite:///{DEFAULT_SQLITE_DIR / 'corekit.db'}"
+
+    @classmethod
+    def sqlite(cls, filename: str, directory: Path | str | None = None, **kwargs: Any) -> "SQLConnection":
+        """
+        Build a connection to a SQLite file, creating its directory if needed.
+
+        Zero configuration: no server, no environment variables.
+        """
+        target_dir = Path(directory) if directory is not None else DEFAULT_SQLITE_DIR
+        target_dir = target_dir.expanduser()
+        target_dir.mkdir(parents=True, exist_ok=True)
+        return cls(f"sqlite:///{target_dir / filename}", **kwargs)
+
+    @classmethod
+    def in_memory(cls, shared_name: str | None = None, **kwargs: Any) -> "SQLConnection":
+        """
+        Build a connection to an ephemeral in-memory database, for tests.
+
+        Each call gets its own database by default. Engines are cached per URL,
+        so a plain ``sqlite://`` would silently hand every caller the same one
+        and leak rows between tests. Pass ``shared_name`` when two connections
+        genuinely need to see the same in-memory data.
+        """
+        name = shared_name or f"corekit_{uuid.uuid4().hex}"
+        return cls(f"sqlite:///file:{name}?mode=memory&cache=shared&uri=true", **kwargs)
+
+    @property
+    def connection_url(self) -> str:
+        """
+        The URL this connection was built with.
+        """
+        return self._connection_url
+
+    @property
+    def session(self) -> Session:
+        """
+        The active session, started on first use.
+        """
+        self._start_session()
+        assert self._session is not None
+        return self._session
+
+    @staticmethod
+    def get_all_tables() -> Any:
+        """
+        Every table registered in SQLModel's metadata.
+        """
+        return SQLModel.metadata.tables.items()
+
+    @property
+    def is_connected(self) -> bool:
+        return self._session is not None
+
+    @property
+    def session_initialized(self) -> bool:
+        return self._session is not None
+
+    # =============
+    # Lifecycle
+    # =============
+
+    def _connect(self) -> None:
+        self._session = Session(_get_engine(self._connection_url))
+
+    def _disconnect(self) -> None:
+        if self._session is not None:
+            self._session.close()
+            self._session = None
+
+    async def _async_connect(self) -> None:
+        """
+        SQLModel sessions are synchronous; this exists to satisfy Connectable.
+        """
+        self._connect()
+
+    async def _async_disconnect(self) -> None:
+        """
+        SQLModel sessions are synchronous; this exists to satisfy Connectable.
+        """
+        self._disconnect()
+
+    def _start_session(self) -> None:
+        """
+        Open a session if one is not already active.
+        """
+        if not self.session_initialized:
+            self._connect()
+
+    def _exec(self, statement: Any) -> Any:
+        self._start_session()
+        return self.session.exec(statement)
+
+    # =============
+    # Schema
+    # =============
+
+    def initialize_database(self) -> None:
+        """
+        Create any tables present in SQLModel metadata but missing from the database.
+        """
+        SQLModel.metadata.create_all(_get_engine(self._connection_url))
+
+    def exec_ddl(self, sql: str) -> None:
+        """
+        Run a raw DDL statement. Intended for migrations, not application code.
+        """
+        self._start_session()
+        self.session.exec(text(sql))
+        self.session.commit()
+
+    # =============
+    # Writes
+    # =============
+
+    def insert(self, *items: Any) -> None:
+        """
+        Insert new rows.
+        """
+        self._start_session()
+        self.session.add_all(items)
+        self.session.commit()
+
+    def upsert(self, *items: Any) -> None:
+        """
+        Insert or update rows. Slower than insert, but tolerates existing ids.
+        """
+        self._start_session()
+        for item in items:
+            self.session.merge(item)
+        self.session.commit()
+
+    def delete(self, table: type[NamedTable], identifier: str) -> None:
+        """
+        Delete a row by id.
+        """
+        item = self.fetch_one_by_id(table, identifier)
+        if item is None:
+            return
+        self.session.delete(item)
+        self.session.commit()
+
+    def rollback(self) -> None:
+        """
+        Discard uncommitted changes.
+        """
+        if self._session is not None:
+            self._session.rollback()
+
+    def refresh(self, item: Any) -> None:
+        """
+        Reload an instance from the database.
+        """
+        self.session.refresh(item)
+
+    # =============
+    # Reads
+    # =============
+
+    def exec(self, statement: Any) -> Any:
+        """
+        Execute an arbitrary statement. Prefer a more specific method.
+        """
+        return self._exec(statement)
+
+    def select(self, statement: Any) -> Any:
+        """
+        Run a select statement.
+        """
+        return self._exec(statement)
+
+    def first(self, statement: Any) -> Any:
+        """
+        The first row of a select statement, or None.
+        """
+        result = self.select(statement)
+        return None if result is None else result.first()
+
+    def fetch_all(self, table: type[NamedTable]) -> Iterator[NamedTable]:
+        """
+        Every row in a table.
+        """
+        return self._exec(select(table))
+
+    def fetch_one_by_id(self, table: type[NamedTable], identifier: str) -> NamedTable | None:
+        """
+        A single row by id.
+        """
+        return self.first(select(table).where(table.id == identifier))
+
+    def fetch_one_by_condition(self, table: type[NamedTable], key: Any, value: Any) -> NamedTable | None:
+        """
+        The first row where a column equals a value.
+        """
+        return self.first(select(table).where(key == value))
+
+    def exists(self, table: type[NamedTable], identifier: str) -> bool:
+        """
+        Whether a row with this id exists.
+        """
+        return self.fetch_one_by_id(table, identifier) is not None
+
+    def fetch_or_create_one(self, table: type[NamedTable], identifier: str) -> NamedTable:
+        """
+        Fetch a row by id, inserting a default one if it is missing.
+        """
+        item = self.fetch_one_by_id(table, identifier)
+        if item is None:
+            item = table.create_default(identifier)
+            self.insert(item)
+        return item
+
+    def fetch_many_by_query(self, query: Query) -> list[NamedTable]:
+        """
+        Every row matching a Query.
+        """
+        return list(self._exec(query.build()))
+
+    def fetch_one_by_query(self, query: Query) -> NamedTable | None:
+        """
+        The first row matching a Query.
+        """
+        return self.first(query.build())
+
+    def count(self, table: type[NamedTable]) -> int:
+        """
+        The number of rows in a table.
+        """
+        self._start_session()
+        return self.session.exec(select(func.count()).select_from(table)).one()
+
+    def paginate(self, table: type[NamedTable], offset: int = 0, limit: int = 50) -> PaginatedResult:
+        """
+        A page of rows, together with the total row count.
+        """
+        total = self.count(table)
+        rows = list(self._exec(select(table).offset(offset).limit(limit)))
+        return PaginatedResult(rows=rows, total=total)

corekit/connections/sql/fields/__init__.py

@@ -0,0 +1,7 @@
+"""
+Custom SQL column types.
+"""
+
+from corekit.connections.sql.fields.jsonb import JSONBField, PydanticJSON
+
+__all__ = ["JSONBField", "PydanticJSON"]