sqlite-ursula 0.11.3.dev0__py3-none-any.whl

sqlite_ursula/_doctor.py

@@ -0,0 +1,37 @@
+"""``python -m sqlite_ursula``: whether this Python can host the extension, and if not, what to use
+instead. It loads the extension as ``open()`` would, and prints what it found. The exit code is 0
+when this Python can host it and 1 when it cannot.
+"""
+
+from __future__ import annotations
+
+import platform
+import sqlite3
+import sys
+import sysconfig
+from typing import TextIO
+
+from . import __version__
+from . import _host
+from .errors import UrsulaSetupError
+
+
+def main(out: TextIO = sys.stdout) -> int:
+    free_threaded = bool(sysconfig.get_config_var("Py_GIL_DISABLED"))
+    build = f"{platform.python_implementation()}{', free-threaded' if free_threaded else ''}"
+    print(f"sqlite-ursula {__version__}", file=out)
+    print(f"python: {platform.python_version()} ({build}) at {sys.executable}", file=out)
+    print(f"sqlite: {sqlite3.sqlite_version}", file=out)
+    print(f"extension loading: {'yes' if _host.loads_extensions() else 'no'}", file=out)
+    try:
+        lib = _host.library()
+        print(f"extension: {lib.path} ({lib.source})", file=out)
+        _host.connection(lib).close()
+    except UrsulaSetupError as error:
+        print(f"result: {error.kind}: this Python cannot host sqlite-ursula", file=out)
+        print(f"why: {error}", file=out)
+        if error.fix is not None:
+            print(f"fix: {error.fix}", file=out)
+        return 1
+    print("result: ok: this Python can host sqlite-ursula", file=out)
+    return 0

sqlite_ursula/_events.py

@@ -0,0 +1,477 @@
+"""The events thread (design: Pump): this process's view of the extension's events.
+
+Calls into the extension never wait on the network: ``ursula_open`` and ``ursula_synced`` are
+made with timeout 0 and either answer at once or hand back a request or a ticket, which an event
+later resolves. One daemon thread, ``sqlite-ursula-events``, started at the first open or
+``synced()``, waits for those events on a connection of its own and applies them under the
+control lock:
+
+- ``synced {path, through}`` resolves every ticket of the path up to ``through``. The extension
+  keeps only the latest watermark of a binding in its queue, so any number of waiters costs one
+  event.
+- ``sync_failed {ticket}`` fails that ticket alone.
+- ``reset {path}`` fails the path's tickets, then runs the ``on_reset`` callbacks of its
+  connections on a thread of their own, so a callback may call into the package.
+- ``gap`` (events fell out of the queue) asks for every pending ticket again under a new ticket,
+  wakes every open, and reads resets from status.
+- ``opened {request}`` wakes the opens waiting for it, which call ``ursula_open`` again to
+  collect the result.
+
+A waiter is a ``threading.Event`` for ``synced()``, ``gate`` and ``open()``, and a
+:class:`LoopEvent` for ``asynced()``, ``agate`` and ``aopen()``, which the thread wakes through its
+event loop's ``call_soon_threadsafe``. Nothing runs on the loop's default executor. One wait
+covers the tickets of every database it was given, with one waiter that each ticket's outcome
+wakes: it ends once every ticket resolved, or at the first that failed, and drops the rest.
+
+The thread takes its cursor before any request or ticket is handed out, and requests and tickets
+are registered under the lock in the same step that asks for them, so no event about one applies
+before it is registered. A thread that fails is replaced while tickets wait or connections are
+open, and the new one starts with a gap, since events may have passed between the two. After three
+failures in a row, waits fail ``internal``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+import os
+import threading
+import time
+import weakref
+from collections.abc import Sequence
+from typing import TYPE_CHECKING
+from typing import Any
+
+from . import _args
+from . import _control
+from .errors import UrsulaResetError
+from .errors import UrsulaSetupError
+from .errors import again
+from .errors import error_of
+
+if TYPE_CHECKING:
+    from ._connection import UrsulaConnection
+
+log = logging.getLogger("sqlite_ursula")
+
+#: After this many failures of the thread in a row, it stays down and its waits fail.
+MAX_FAILURES = 3
+#: How long one ``ursula_events`` call waits for an event.
+POLL_MS = 1000
+
+
+class LoopEvent:
+    """What a coroutine waits on where a thread waits on a ``threading.Event``: :meth:`set` from any
+    thread wakes it on its event loop. Made on the loop, by the coroutine that waits."""
+
+    __slots__ = ("_future", "_loop")
+
+    def __init__(self) -> None:
+        self._loop = asyncio.get_running_loop()
+        self._future: asyncio.Future[None] = self._loop.create_future()
+
+    def set(self) -> None:
+        # A loop that closed raises RuntimeError: nothing waits on it any more.
+        with contextlib.suppress(RuntimeError):
+            self._loop.call_soon_threadsafe(self._settle)
+
+    def clear(self) -> None:
+        """On the loop: the next :meth:`set` wakes the waiter again. A wake already on its way may
+        still come, which only makes the waiter look again."""
+        if self._future.done():
+            self._future = self._loop.create_future()
+
+    def _settle(self) -> None:
+        # A waiter that gave up (its timeout, a cancellation) cancelled the future.
+        if not self._future.done():
+            self._future.set_result(None)
+
+    async def wait(self, timeout: float | None) -> bool:
+        """Whether it was set within ``timeout`` seconds (``None``: no limit). Cancelling the
+        coroutine ends the wait."""
+        try:
+            async with asyncio.timeout(timeout):
+                await self._future
+        except TimeoutError:
+            return False
+        return True
+
+
+Waiter = threading.Event | LoopEvent
+
+
+class Wait:
+    """One ``synced()``, ``asynced()`` or gate call waiting for the tickets of its databases. Its
+    state changes under the lock, and each change wakes ``waiter``."""
+
+    __slots__ = ("failed", "left", "waiter")
+
+    def __init__(self, waiter: Waiter) -> None:
+        self.waiter = waiter
+        #: The tickets not resolved yet.
+        self.left = 0
+        #: The path and the error of the first ticket that failed.
+        self.failed: tuple[str, BaseException] | None = None
+
+    def done(self) -> bool:
+        return self.failed is not None or self.left == 0
+
+
+class Ticket:
+    """The ticket of one database of a :class:`Wait`."""
+
+    __slots__ = ("path", "wait")
+
+    def __init__(self, path: str, wait: Wait) -> None:
+        self.path = path
+        self.wait = wait
+
+    def resolve(self) -> None:
+        self.wait.left -= 1
+        self.wait.waiter.set()
+
+    def reject(self, error: BaseException) -> None:
+        if self.wait.failed is None:
+            self.wait.failed = (self.path, error)
+        self.wait.waiter.set()
+
+
+#: The pending tickets of the process, by ticket: kept across threads.
+_tickets: dict[int, Ticket] = {}
+#: The open bound connections by bound path, whose callbacks a reset runs. Held weakly: a
+#: connection the application dropped is collected.
+_targets: dict[str, weakref.WeakSet[UrsulaConnection]] = {}
+_current: Events | None = None
+#: Threads that failed in a row, before any delivered events.
+_failures = 0
+
+
+def reset_error(named: dict[str, Any]) -> UrsulaResetError:
+    """The error of the reset the extension named (a ``reset`` event, ``status()["reset"]``)."""
+    return UrsulaResetError(str(named["message"]), str(named["kind"]), named["action"])
+
+
+def track(target: UrsulaConnection) -> None:
+    """Runs ``target``'s callbacks on the resets of its path."""
+    with _control.lock():
+        _targets.setdefault(target.path, weakref.WeakSet()).add(target)
+
+
+def untrack(target: UrsulaConnection) -> None:
+    with _control.lock():
+        found = _targets.get(target.path)
+        if found is not None:
+            found.discard(target)
+            if not found:
+                del _targets[target.path]
+
+
+def _live(path: str | None = None) -> list[UrsulaConnection]:
+    """The tracked connections of ``path``, or of every path."""
+    if path is None:
+        return [t for found in _targets.values() for t in found]
+    return list(_targets.get(path, ()))
+
+
+def _fire(hit: list[tuple[UrsulaConnection, UrsulaResetError]]) -> None:
+    """Runs the callbacks of the connections ``hit`` on a thread of their own."""
+    if not hit:
+        return
+
+    def run() -> None:
+        for target, error in hit:
+            target._reset_by(error)
+
+    threading.Thread(target=run, name="sqlite-ursula-reset", daemon=True).start()
+
+
+def _reissue() -> None:
+    """Asks for every pending ticket again (``ursula_synced``, timeout 0), under a new ticket.
+    Called with the lock held."""
+    pending = list(_tickets.values())
+    _tickets.clear()
+    for t in pending:
+        try:
+            again = _control.call("SELECT ursula_synced(?, 0)", t.path)
+        except Exception as error:
+            t.reject(error)
+            continue
+        if again["synced"]:
+            t.resolve()
+        else:
+            _tickets[int(again["ticket"])] = t
+
+
+class Events:
+    """One events thread, from the cursor at its start."""
+
+    def __init__(self) -> None:
+        # From the cursor at this moment: every request and ticket handed out after it is
+        # answered by an event this thread sees, and no event of an earlier binding is taken for a
+        # later one's.
+        self._cursor = int(_control.call("SELECT ursula_events(0, 0)")["cursor"])
+        self._opens: dict[int, set[Waiter]] = {}
+        self._failed = False
+        self.conn: Any = None
+        self._thread = threading.Thread(target=self._run, name="sqlite-ursula-events", daemon=True)
+        self._thread.start()
+
+    def wait_open(self, request: int, waiter: Waiter) -> None:
+        """Wakes ``waiter`` for open ``request`` at its ``opened`` event, a gap, or the thread's
+        end. Called with the lock held, in the step that got the request."""
+        self._opens.setdefault(request, set()).add(waiter)
+
+    def forget_open(self, request: int, waiter: Waiter) -> None:
+        with _control.lock():
+            found = self._opens.get(request)
+            if found is not None:
+                found.discard(waiter)
+                if not found:
+                    del self._opens[request]
+
+    def _wake(self, request: int) -> None:
+        for waiter in self._opens.pop(request, ()):
+            waiter.set()
+
+    def _run(self) -> None:
+        global _failures
+        try:
+            self.conn = _control.connection()
+            cursor = self._cursor
+            while True:
+                row = self.conn.execute("SELECT ursula_events(?, ?)", (cursor, POLL_MS)).fetchone()
+                batch = _control.parsed(row[0])
+                cursor = int(batch["cursor"])
+                if batch["events"]:
+                    with _control.lock():
+                        if self._failed:
+                            return
+                        _failures = 0
+                        self.handle(batch["events"])
+        except Exception:
+            self._fail()
+
+    def handle(self, events: list[dict[str, Any]]) -> None:
+        """A batch of events, in order. Called with the lock held."""
+        for event in events:
+            kind = event.get("event")
+            if kind == "opened":
+                self._wake(int(event["request"]))
+            elif kind == "synced":
+                through = int(event["through"])
+                for ticket, t in list(_tickets.items()):
+                    if t.path == event["path"] and ticket <= through:
+                        del _tickets[ticket]
+                        t.resolve()
+            elif kind == "sync_failed":
+                t = _tickets.pop(int(event["ticket"]), None)
+                if t is not None:
+                    t.reject(error_of(event, str(event["message"]), None))
+            elif kind == "reset":
+                self._reset(event.get("path"), event)
+            elif kind == "gap":
+                self._gap()
+
+    def _reset(self, path: str | None, named: dict[str, Any]) -> None:
+        """A reset of ``path`` (``None``: a binding the process no longer has): its pending
+        ``synced()`` calls fail, then its callbacks run."""
+        hit: list[tuple[UrsulaConnection, UrsulaResetError]] = []
+        if path is None:
+            for target in _live():
+                error = target._current_reset()
+                if error is not None:
+                    hit.append((target, error))
+            paths = {target.path for target, _ in hit}
+        else:
+            hit = [(target, reset_error(named)) for target in _live(path)]
+            paths = {path}
+        for ticket, t in list(_tickets.items()):
+            if t.path in paths:
+                del _tickets[ticket]
+                t.reject(reset_error(named))
+        _fire(hit)
+
+    def _gap(self) -> None:
+        """Events fell out of the queue before the thread read them. Every pending ticket is asked
+        for again under a new ticket, which no watermark of the batch covers, so a ticket whose
+        failure the gap dropped never passes for resolved. Opens ask again, and resets are read
+        from status."""
+        for request in list(self._opens):
+            self._wake(request)
+        _reissue()
+        hit = []
+        for target in _live():
+            error = target._current_reset()
+            if error is not None:
+                hit.append((target, error))
+        _fire(hit)
+
+    def _fail(self) -> None:
+        """The thread is gone: pending opens ask again, and while tickets wait or connections are
+        open a new thread starts with a gap, since events may have passed between the two. After
+        :data:`MAX_FAILURES` failures in a row the tickets fail instead, and so does every later
+        call."""
+        global _current, _failures
+        with _control.lock():
+            if self._failed:
+                return
+            self._failed = True
+            _failures += 1
+            if _current is self:
+                _current = None
+            for request in list(self._opens):
+                self._wake(request)
+            if not _tickets and not _live():
+                return
+            try:
+                replacement = events()
+            except Exception as error:
+                pending = list(_tickets.values())
+                _tickets.clear()
+                for t in pending:
+                    t.reject(error)
+                return
+            replacement._gap()
+
+
+def events() -> Events:
+    """The process's events thread, started at the first call that needs it."""
+    global _current
+    with _control.lock():
+        if _current is None:
+            if _failures >= MAX_FAILURES:
+                raise UrsulaSetupError(
+                    f"internal: the sqlite-ursula events thread failed {_failures} times in a "
+                    "row: check the extension, then restart the process",
+                    "internal",
+                    "reopen",
+                )
+            _current = Events()
+        return _current
+
+
+def _ask(dbs: Sequence[UrsulaConnection], waiter: type[Waiter]) -> Wait | None:
+    """Asks for a ticket of every database in ``dbs`` (``ursula_synced``, timeout 0), in one step
+    under the lock: ``None`` when every one is synced already, else the :class:`Wait` of the
+    tickets, registered in the same step, with a new ``waiter``."""
+    for db in dbs:
+        db._usable()
+    with _control.lock():
+        # Started before the calls, so its events cursor precedes the tickets.
+        events()
+        wait = Wait(waiter())
+        try:
+            for db in dbs:
+                try:
+                    answer = _control.call("SELECT ursula_synced(?, 0)", db.path)
+                except UrsulaResetError as error:
+                    db._remember(error)
+                    raise
+                if not answer["synced"]:
+                    _tickets[int(answer["ticket"])] = Ticket(db.path, wait)
+                    wait.left += 1
+        except BaseException:
+            _drop(wait)
+            raise
+    return wait if wait.left else None
+
+
+def _drop(wait: Wait) -> None:
+    """The call of ``wait`` ended: no event resolves its tickets any more."""
+    with _control.lock():
+        for ticket, pending in list(_tickets.items()):
+            if pending.wait is wait:
+                del _tickets[ticket]
+
+
+def _settled(wait: Wait) -> bool:
+    """Whether ``wait`` is over, else its waiter cleared for the next change: under the lock, so no
+    change falls between the two."""
+    with _control.lock():
+        if wait.done():
+            return True
+        wait.waiter.clear()
+        return False
+
+
+def _left(deadline: float | None) -> float | None:
+    return None if deadline is None else max(0.0, deadline - time.monotonic())
+
+
+def _outcome(wait: Wait, dbs: Sequence[UrsulaConnection]) -> None:
+    """Raises the first failure of ``wait``, if any, and the connections of its path remember a
+    reset."""
+    if wait.failed is None:
+        return
+    path, error = wait.failed
+    if isinstance(error, UrsulaResetError):
+        for db in dbs:
+            if db.path == path:
+                db._remember(error)
+    raise again(error)
+
+
+def _timed_out(call: str, timeout: float | None) -> TimeoutError:
+    return TimeoutError(
+        f"{call} gave up after {timeout} s: the commits keep replicating, so call it again to wait "
+        "for them"
+    )
+
+
+def synced(dbs: Sequence[UrsulaConnection], timeout: float | None) -> None:
+    """Waits until the stream of each database in ``dbs`` holds every commit the process logged on
+    it before the call, for at most ``timeout`` seconds (``None``: no limit). Raises
+    ``TimeoutError`` on the timeout, and the first failure, such as a reset, at once."""
+    limit = _args.limit(timeout)
+    wait = _ask(dbs, threading.Event)
+    if wait is None:
+        return
+    assert isinstance(wait.waiter, threading.Event)
+    deadline = None if limit is None else time.monotonic() + limit
+    try:
+        while not _settled(wait):
+            left = _left(deadline)
+            if left == 0.0:
+                raise _timed_out("synced()", timeout)
+            wait.waiter.wait(left)
+    finally:
+        _drop(wait)
+    _outcome(wait, dbs)
+
+
+async def asynced(dbs: Sequence[UrsulaConnection], timeout: float | None) -> None:
+    """:func:`synced` for a coroutine: it waits on the event loop, and cancelling it ends the wait
+    while the commits keep replicating."""
+    limit = _args.limit(timeout)
+    wait = _ask(dbs, LoopEvent)
+    if wait is None:
+        return
+    assert isinstance(wait.waiter, LoopEvent)
+    deadline = None if limit is None else time.monotonic() + limit
+    try:
+        while not _settled(wait):
+            left = _left(deadline)
+            if left == 0.0:
+                raise _timed_out("asynced()", timeout)
+            await wait.waiter.wait(left)
+    finally:
+        _drop(wait)
+    _outcome(wait, dbs)
+
+
+def _after_fork_in_child() -> None:
+    global _current, _failures
+    if _current is not None:
+        _control._inherited.append(_current)
+    # The parent's connections stay referenced, so collecting one never closes a bound file in
+    # the child.
+    _control._inherited.extend(_live())
+    _current = None
+    _failures = 0
+    _tickets.clear()
+    _targets.clear()
+
+
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(after_in_child=_after_fork_in_child)

sqlite_ursula/_gate.py

@@ -0,0 +1,136 @@
+"""``gate`` and ``agate``: the output gate as a context manager (design: Gates).
+
+The block runs, and leaving it waits for a ``synced()`` call made after the block ended, on every
+database given, before the result or the block's error goes on. ``synced()`` is the gate: this
+module only decides when to wait and which errors not to hold, and adds no call of the extension
+but ``classify`` for an error a framework's own connection raised.
+
+- An :class:`UrsulaError` the block raises passes at once, as the block raised it, also when a
+  library wrapped it (as ``__cause__``, as SQLAlchemy's ``.orig``, or in an ``ExceptionGroup``):
+  it carries no application state. A refused write committed nothing, and a reset is what the wait
+  would report. So does a SQLite error, also wrapped, that the extension classifies
+  (``ursula_classify``) on one of the databases, which a framework's own ``sqlite3`` connection
+  raised unclassified. A bare SQLite error that one database's state names a refused write or a
+  reset raises as that :class:`UrsulaError`, which is a ``sqlite3.OperationalError`` too. With
+  several databases a statement's I/O error does not say whose it is, so it passes unchanged.
+- Every other ``Exception`` waits as a result does, then passes. A ``BaseException`` that is not an
+  ``Exception`` (Ctrl-C, a cancellation, ``SystemExit``) passes at once.
+- The wait covers every database at once, and the first failure, such as a reset, ends it and
+  replaces the outcome.
+- ``timeout`` bounds the wait, not the block: past it, ``TimeoutError``, while the commits keep
+  replicating.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import sqlite3
+from collections.abc import AsyncIterator
+from collections.abc import Iterator
+
+from . import _control
+from . import _events
+from ._args import invalid
+from ._args import limit
+from ._connection import UrsulaConnection
+from ._connection import bound
+from .errors import UrsulaError
+from .errors import UrsulaResetError
+from .errors import UrsulaWriteRefusedError
+
+
+def _databases(dbs: tuple[object, ...]) -> tuple[UrsulaConnection, ...]:
+    if not dbs:
+        raise invalid("gate needs at least one database")
+    return tuple(bound(db, "gate") for db in dbs)
+
+
+def _passing(error: Exception, dbs: tuple[UrsulaConnection, ...]) -> BaseException | None:
+    """What raises at once instead of waiting, or ``None`` when ``error`` waits: ``error`` itself
+    when its chain holds an :class:`UrsulaError` or a SQLite error the extension classifies, or
+    the refused write or reset that a bare SQLite error is on the one database given."""
+    classified = _control.classify_wrapped(error, [db.path for db in dbs])
+    if not isinstance(classified, UrsulaError):
+        return None
+    if (
+        len(dbs) == 1
+        and classified.__cause__ is error
+        and isinstance(error, sqlite3.Error)
+        and isinstance(classified, (UrsulaWriteRefusedError, UrsulaResetError))
+    ):
+        return classified
+    return error
+
+
+def _gave_up(timeout: float | None, dbs: tuple[UrsulaConnection, ...]) -> TimeoutError:
+    names = ", ".join(db.name for db in dbs)
+    return TimeoutError(
+        f"the gate gave up after {timeout:g} s waiting for {names}: the commits keep replicating"
+    )
+
+
+@contextlib.contextmanager
+def gate(*dbs: UrsulaConnection, timeout: float | None = None) -> Iterator[None]:
+    """Runs the block, then lets its outcome go on only once a ``synced()`` call made after the
+    block ended returned on every database in ``dbs``, so nothing outside the process acts on a
+    result, or an error, that rests on an unacknowledged commit or on ownership this process is
+    about to lose. A block that only read waits too: it may have read another connection's pending
+    commit::
+
+        with sqlite_ursula.gate(db):
+            reply = handle(request)
+        send(reply)
+
+    An :class:`UrsulaError` the block raises, also wrapped, passes at once as it was raised, and so
+    does the error of a framework's own connection that the extension classifies. Every other error
+    waits, then passes. The wait covers every database at once, and the first failure on one, such
+    as a reset, raises at once. ``timeout`` (seconds, ``None``: no limit) bounds the wait and raises
+    ``TimeoutError``. Effects inside the block, such as an outbound call, are not gated: call
+    ``db.synced()`` before them.
+    """
+    targets = _databases(dbs)
+    bound = limit(timeout)
+    try:
+        yield
+    except Exception as error:
+        passing = _passing(error, targets)
+        if passing is error:
+            raise
+        if passing is not None:
+            raise passing from error
+        _wait(targets, bound)
+        raise
+    _wait(targets, bound)
+
+
+def _wait(dbs: tuple[UrsulaConnection, ...], timeout: float | None) -> None:
+    try:
+        _events.synced(dbs, timeout)
+    except TimeoutError as error:
+        raise _gave_up(timeout, dbs) from error
+
+
+@contextlib.asynccontextmanager
+async def agate(*dbs: UrsulaConnection, timeout: float | None = None) -> AsyncIterator[None]:
+    """:func:`gate` for asyncio: ``async with agate(db):``. It waits on the event loop instead of
+    blocking it, and cancelling the task ends the wait."""
+    targets = _databases(dbs)
+    bound = limit(timeout)
+    try:
+        yield
+    except Exception as error:
+        passing = _passing(error, targets)
+        if passing is error:
+            raise
+        if passing is not None:
+            raise passing from error
+        await _await(targets, bound)
+        raise
+    await _await(targets, bound)
+
+
+async def _await(dbs: tuple[UrsulaConnection, ...], timeout: float | None) -> None:
+    try:
+        await _events.asynced(dbs, timeout)
+    except TimeoutError as error:
+        raise _gave_up(timeout, dbs) from error