ekbatan 0.1.0__py3-none-any.whl

ekbatan/__init__.py

@@ -0,0 +1,262 @@
+"""Ekbatan: a persistence framework for event-driven systems.
+
+A service that publishes domain events usually writes twice - once to its database, once to
+its broker - and the two disagree whenever the second write fails. The transactional outbox is
+the established answer, normally bolted on as a separate library or assembled by hand.
+
+This folds it into the persistence layer, with three pieces:
+
+- a **model** emits an event when it changes;
+- an **action** stages changes on a **plan** instead of writing them;
+- the **executor** opens one transaction, writes the rows and their events, and commits.
+
+State and events land together or neither does, and no line of application code mentions an
+outbox.
+"""
+
+from .accounting import StatementCount, count_statements
+from .action import (
+    DEFAULT_RETRIES,
+    Action,
+    ActionExecutor,
+    ActionExecutorBuilder,
+    ActionPlan,
+    ActionRegistry,
+    ActionRegistryBuilder,
+    ChangePersister,
+    EventEntity,
+    EventEntityBuilder,
+    EventPersister,
+    ExecutionConfiguration,
+    ExecutionConfigurationBuilder,
+    RetryConfig,
+    SingleTableJsonEventPersister,
+    current_plan,
+)
+from .clock import Clock, SystemClock, VirtualClock
+from .config import (
+    DataSourceConfig,
+    DataSourceConfigBuilder,
+    ShardGroupConfig,
+    ShardGroupConfigBuilder,
+    ShardingConfig,
+    ShardingConfigBuilder,
+    ShardMemberConfig,
+    ShardMemberConfigBuilder,
+)
+from .distributedjobs import (
+    DistributedJob,
+    ExecutionContext,
+    JobRegistry,
+    JobRegistryBuilder,
+    JobsConfig,
+    Schedule,
+    job_registry,
+)
+from .domain import (
+    Entity,
+    EntityBuilder,
+    GenericState,
+    Model,
+    ModelBuilder,
+    ModelEvent,
+    Persistable,
+)
+from .errors import (
+    BlockingCallError,
+    ConnectivityError,
+    CrossShardError,
+    EkbatanError,
+    EntityNotFoundError,
+    ForkedProcessError,
+    LockAcquisitionError,
+    StaleRecordError,
+    TransactionContextError,
+    TransactionThreadError,
+    UnsupportedError,
+    ValidationError,
+)
+from .events import (
+    EventDispatch,
+    EventEnvelope,
+    EventEnvelopeBuilder,
+    EventFanout,
+    EventFanoutJob,
+    EventHandler,
+    EventHandlerRegistry,
+    EventHandlerRegistryBuilder,
+    EventHandlingJob,
+    EventNotificationState,
+    HandlingConfig,
+    HandlingConfigBuilder,
+    LocalEventHandlerConfig,
+    LocalEventHandlerConfigBuilder,
+)
+from .fetching import Db, FetchResult
+from .ids import Id, ShardedId, TypedValue, unwrap
+from .locks import (
+    KeyedLockProvider,
+    Lease,
+    MariaDBKeyedLockProvider,
+    MySQLKeyedLockProvider,
+    PostgresKeyedLockProvider,
+    SqlKeyedLockProviderBuilder,
+)
+from .migrations import (
+    MigrationError,
+    YoyoMigrator,
+    YoyoMigratorBuilder,
+    emit_ddl,
+    migrate,
+    migrate_registry,
+    validate_scripts,
+)
+from .persistence import (
+    DatabaseRegistry,
+    DatabaseRegistryBuilder,
+    DataSource,
+    Transaction,
+    TransactionManager,
+)
+from .repository import (
+    AbstractRepository,
+    EntityRepository,
+    ModelRepository,
+    Repository,
+    RepositoryRegistry,
+    RepositoryRegistryBuilder,
+)
+from .schema import notifications_table, outbox_table, scheduled_tasks_table
+from .sharding import (
+    DEFAULT_SHARD,
+    EmbeddedBitsShardingStrategy,
+    NoShardingStrategy,
+    ShardIdentifier,
+    ShardingStrategy,
+    UnknownShardPolicy,
+    read_shard,
+    sharded_uuid,
+    uuid_instant,
+)
+from .testing import ActionAssert, ActionSpec, StagedWork, run_action
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "DEFAULT_RETRIES",
+    "DEFAULT_SHARD",
+    "AbstractRepository",
+    "Action",
+    "ActionAssert",
+    "ActionExecutor",
+    "ActionExecutorBuilder",
+    "ActionPlan",
+    "ActionRegistry",
+    "ActionRegistryBuilder",
+    "ActionSpec",
+    "BlockingCallError",
+    "ChangePersister",
+    "Clock",
+    "ConnectivityError",
+    "CrossShardError",
+    "DataSource",
+    "DataSourceConfig",
+    "DataSourceConfigBuilder",
+    "DatabaseRegistry",
+    "DatabaseRegistryBuilder",
+    "Db",
+    "DistributedJob",
+    "EkbatanError",
+    "EmbeddedBitsShardingStrategy",
+    "Entity",
+    "EntityBuilder",
+    "EntityNotFoundError",
+    "EntityRepository",
+    "EventDispatch",
+    "EventEntity",
+    "EventEntityBuilder",
+    "EventEnvelope",
+    "EventEnvelopeBuilder",
+    "EventFanout",
+    "EventFanoutJob",
+    "EventHandler",
+    "EventHandlerRegistry",
+    "EventHandlerRegistryBuilder",
+    "EventHandlingJob",
+    "EventNotificationState",
+    "EventPersister",
+    "ExecutionConfiguration",
+    "ExecutionConfigurationBuilder",
+    "ExecutionContext",
+    "FetchResult",
+    "ForkedProcessError",
+    "GenericState",
+    "HandlingConfig",
+    "HandlingConfigBuilder",
+    "Id",
+    "JobRegistry",
+    "JobRegistryBuilder",
+    "JobsConfig",
+    "KeyedLockProvider",
+    "Lease",
+    "LocalEventHandlerConfig",
+    "LocalEventHandlerConfigBuilder",
+    "LockAcquisitionError",
+    "MariaDBKeyedLockProvider",
+    "MigrationError",
+    "Model",
+    "ModelBuilder",
+    "ModelEvent",
+    "ModelRepository",
+    "MySQLKeyedLockProvider",
+    "NoShardingStrategy",
+    "Persistable",
+    "PostgresKeyedLockProvider",
+    "Repository",
+    "RepositoryRegistry",
+    "RepositoryRegistryBuilder",
+    "RetryConfig",
+    "Schedule",
+    "ShardGroupConfig",
+    "ShardGroupConfigBuilder",
+    "ShardIdentifier",
+    "ShardMemberConfig",
+    "ShardMemberConfigBuilder",
+    "ShardedId",
+    "ShardingConfig",
+    "ShardingConfigBuilder",
+    "ShardingStrategy",
+    "SingleTableJsonEventPersister",
+    "SqlKeyedLockProviderBuilder",
+    "StagedWork",
+    "StaleRecordError",
+    "StatementCount",
+    "SystemClock",
+    "Transaction",
+    "TransactionContextError",
+    "TransactionManager",
+    "TransactionThreadError",
+    "TypedValue",
+    "UnknownShardPolicy",
+    "UnsupportedError",
+    "ValidationError",
+    "VirtualClock",
+    "YoyoMigrator",
+    "YoyoMigratorBuilder",
+    "__version__",
+    "count_statements",
+    "current_plan",
+    "emit_ddl",
+    "job_registry",
+    "migrate",
+    "migrate_registry",
+    "notifications_table",
+    "outbox_table",
+    "read_shard",
+    "run_action",
+    "scheduled_tasks_table",
+    "sharded_uuid",
+    "unwrap",
+    "uuid_instant",
+    "validate_scripts",
+]

ekbatan/_builder.py

@@ -0,0 +1,41 @@
+"""The shared half of the builders for the framework's plain data classes.
+
+Each builder keeps its own typed setters - that is what a checker reads - and hands the value to
+:meth:`FieldBuilder._set`. What they share is ``build()``: every required field named, in one
+refusal, before the constructor is reached. Left to the dataclass, a missing field is a
+``TypeError`` about a keyword argument, which names the constructor rather than the setter that
+was never called.
+"""
+
+from __future__ import annotations
+
+from typing import Any, ClassVar, Self, cast
+
+from .errors import ValidationError
+
+
+class FieldBuilder[T]:
+    """Collects fields by name, then builds ``_target`` from them."""
+
+    __slots__ = ("_given",)
+
+    #: The class ``build()`` constructs.
+    _target: ClassVar[type[Any]]
+    #: Fields ``build()`` refuses to go without, in the order they are named when missing.
+    _required: ClassVar[tuple[str, ...]] = ()
+    #: Values used for a field whose setter was never called, where the dataclass has none.
+    _defaults: ClassVar[dict[str, Any]] = {}
+
+    def __init__(self) -> None:
+        self._given: dict[str, Any] = {}
+
+    def _set(self, name: str, value: Any) -> Self:
+        self._given[name] = value
+        return self
+
+    def build(self) -> T:
+        missing = [name for name in self._required if self._given.get(name) is None]
+        if missing:
+            verb = "is" if len(missing) == 1 else "are"
+            raise ValidationError(f"{', '.join(missing)} {verb} required")
+        return cast("T", self._target(**{**self._defaults, **self._given}))

ekbatan/_credentials.py

@@ -0,0 +1,162 @@
+"""Refuses a url that carries a user name or a secret - Java's ``JdbcUrlCredentials``.
+
+A url is logged and repeated - drivers repeat it in their errors, SQLAlchemy's echo prints it,
+people paste it into tickets - so it carries neither: the login user goes to ``username``, the
+login password to ``password``, and every other secret to ``data_source_properties``.
+
+The url is read with SQLAlchemy's own parser, which is what turns it into the driver's
+arguments: a user and a password before the ``@``, and the settings of the query string. Java
+reads the url's raw text instead, because JDBC urls come in a dozen forms; a SQLAlchemy url comes
+in one, and reading it the way SQLAlchemy reads it means a setting is refused exactly when it
+would reach the driver.
+
+Which setting names hold a user name or a secret is decided once, for every setting of every
+driver this port connects through - psycopg (and the libpq under it), PyMySQL and sqlite3, and
+``ekbatan.aio``'s asyncpg, aiomysql and aiosqlite - in the ledger ``tests/driver_settings.tsv``.
+The names here are its refused part, and a test fails when the two disagree. Names are matched
+ignoring case and spaces, as Java matches them; a setting with no value carries nothing.
+"""
+
+from __future__ import annotations
+
+import enum
+import re
+from typing import Any
+from urllib.parse import unquote
+
+from .errors import ValidationError
+
+__all__ = ["Kind", "kind_of", "require_none_in"]
+
+
+class Kind(enum.Enum):
+    """What a refused setting holds, which decides where its value goes instead."""
+
+    #: The user this configuration logs in as.
+    LOGIN_USER = "login-user"
+    #: This configuration's login password.
+    LOGIN_PASSWORD = "login-password"
+    #: Any other user name. None of this port's drivers has one yet; kept so the kinds match Java's.
+    USER_NAME = "user-name"
+    #: Any other secret: a key's password, a client secret, a key that logs in like a password.
+    SECRET = "secret"
+
+
+#: The refused setting names of the ledger, lower-cased.
+SETTINGS: dict[str, Kind] = {
+    # every driver that logs in
+    "user": Kind.LOGIN_USER,
+    "password": Kind.LOGIN_PASSWORD,
+    # PyMySQL's old name for the password, read when `password` is empty
+    "passwd": Kind.LOGIN_PASSWORD,
+    # libpq, under psycopg: the client key's password, OAuth, and two SCRAM keys that log in
+    # like a password although libpq does not mark them as secret
+    "sslpassword": Kind.SECRET,
+    "oauth_client_secret": Kind.SECRET,
+    "scram_client_key": Kind.SECRET,
+    "scram_server_key": Kind.SECRET,
+    # PyMySQL: the client key's password
+    "ssl_key_password": Kind.SECRET,
+}
+
+_SPACES = re.compile(r"\s+")
+
+_PROPERTIES = "data_source_properties"
+
+_WHY = (
+    ": a url is logged and repeated where user names and secrets must never be. "
+    "The url is not repeated here."
+)
+
+
+def kind_of(name: str) -> Kind | None:
+    """What the setting of this name holds, if it is refused. Spaces inside the name and its
+    case are ignored."""
+    return SETTINGS.get(_SPACES.sub("", name).lower())
+
+
+def require_none_in(url: str) -> None:
+    """Raise when the url carries a user name or a secret.
+
+    The message names every one - each setting once - and where each goes instead, and never
+    repeats the url or a value.
+    """
+    from sqlalchemy.engine import make_url
+
+    try:
+        parsed = make_url(url)
+        user, password = bool(parsed.username), bool(parsed.password)
+        query: list[tuple[str, Any]] = list(parsed.query.items())
+    except Exception:
+        # SQLAlchemy cannot read it - two hosts written `h1:3306,h2:3306`, say - so read its
+        # text the way Java reads every url: the part before an `@`, and the query's names.
+        user, password, query = _read_text(url)
+
+    refusals: list[tuple[str, str]] = []
+    if user and password:
+        refusals.append(
+            (
+                "a user name and a password before an '@'",
+                "give them to username(...) and password(...)",
+            )
+        )
+    elif user:
+        refusals.append(("a user name before an '@'", "give it to username(...)"))
+    elif password:
+        refusals.append(("a password before an '@'", "give it to password(...)"))
+
+    named: set[str] = set()
+    for name, value in query:
+        if not _has_value(value):
+            continue
+        kind = kind_of(name)
+        if kind is None or name.lower() in named:
+            continue
+        named.add(name.lower())
+        refusals.append(_refusal(kind, name))
+
+    if len(refusals) == 1:
+        what, move = refusals[0]
+        raise ValidationError(f"the url carries {what}. {move[0].upper()}{move[1:]} instead{_WHY}")
+    if refusals:
+        listed = "; ".join(f"{what} - {move}" for what, move in refusals)
+        raise ValidationError(f"the url carries {listed}{_WHY}")
+
+
+def _read_text(url: str) -> tuple[bool, bool, list[tuple[str, Any]]]:
+    """A url SQLAlchemy cannot parse, read from its text: a user and a password before an ``@`` in
+    the host part, and the names of the query's settings. Values are never kept."""
+    _, _, rest = url.partition("://")
+    host_end = min(
+        (i for i in (rest.find("/"), rest.find("?"), rest.find("#")) if i >= 0), default=len(rest)
+    )
+    host_part = rest[:host_end]
+    user = password = False
+    if "@" in host_part:
+        info = host_part.rpartition("@")[0]
+        name, colon, secret = info.partition(":")
+        user, password = bool(name.strip()), bool(colon and secret.strip())
+    query: list[tuple[str, Any]] = []
+    _, question, after = rest.partition("?")
+    if question:
+        for piece in after.split("#", 1)[0].split("&"):
+            name, equals, value = piece.partition("=")
+            if name.strip():
+                query.append((unquote(name).strip(), value if equals else ""))
+    return user, password, query
+
+
+def _has_value(value: Any) -> bool:
+    """A setting written with no value carries nothing; a repeated one is a tuple of values."""
+    values = value if isinstance(value, tuple) else (value,)
+    return any(str(each).strip() for each in values)
+
+
+def _refusal(kind: Kind, setting: str) -> tuple[str, str]:
+    if kind is Kind.LOGIN_USER:
+        return f"the login user (its '{setting}' setting)", "give it to username(...)"
+    if kind is Kind.LOGIN_PASSWORD:
+        return f"the login password (its '{setting}' setting)", "give it to password(...)"
+    if kind is Kind.USER_NAME:
+        return f"a user name (its '{setting}' setting)", f"put it under {_PROPERTIES}"
+    return f"a secret (its '{setting}' setting)", f"put it under {_PROPERTIES}"

ekbatan/_guards.py

@@ -0,0 +1,64 @@
+"""The two refusals: an object used in a process it was copied into, and the sync API on an
+event loop's thread.
+
+Both refuse and explain; neither fixes anything. A copied pool is not reset, a blocking call is
+not moved to a thread - doing either behind the caller's back would hide a mistake in how the
+application starts or where it calls from, and the fix is one line in the application.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import threading
+
+from .errors import BlockingCallError, ForkedProcessError
+
+__all__ = ["refuse_if_forked", "refuse_on_an_event_loop"]
+
+
+def refuse_if_forked(created_in: int, what: str) -> None:
+    """Refuse ``what`` in any process but ``created_in``, the one that built it.
+
+    ``fork()`` copies an object's pools and threads' bookkeeping into the child, but not the
+    threads, and the connections it copies are the parent's sockets: two processes talking on
+    one connection corrupt each other's conversations.
+    """
+    if os.getpid() != created_in:
+        raise ForkedProcessError(
+            f"{what} was created in another process and copied into this one by fork() - "
+            f"gunicorn --preload, uwsgi, Celery prefork, RQ and multiprocessing all fork. Its "
+            f"connections and threads belong to the process that created it. Create ekbatan's "
+            f"objects - data sources, the database registry, the job registry, lock providers - "
+            f"in your app's startup code (the FastAPI lifespan, or the top of the worker's "
+            f"main()), not at import time."
+        )
+
+
+def refuse_on_an_event_loop(what: str) -> None:
+    """Refuse ``what`` - a sync call that waits on a database - on an asyncio event loop's
+    thread, where it would stop every other task on the loop until it returns.
+
+    asyncio not imported means no loop can be running, so then this costs a dictionary lookup.
+    A worker thread (``asyncio.to_thread``, ``run_in_executor``, a FastAPI ``def`` endpoint)
+    has no running loop. Under gevent a worker "thread" is a greenlet on the loop's own OS
+    thread, so asyncio reports the loop as running there too; the loop records the thread - the
+    greenlet, under gevent - that runs it, and only that one is refused. There a blocking call
+    yields, as every call does under gevent.
+    """
+    asyncio = sys.modules.get("asyncio")
+    if asyncio is None:
+        return
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        return
+    runner = getattr(loop, "_thread_id", None)
+    if runner is not None and runner != threading.get_ident():
+        return
+    raise BlockingCallError(
+        f"{what} was called on an asyncio event loop's thread (inside `async def`). The sync "
+        f"API waits on the database, and here it would stop the event loop and every request "
+        f"on it. In `async def` code use ekbatan.aio; or make the endpoint a plain `def`; or "
+        f"call it with `await asyncio.to_thread(...)`."
+    )