sqlite-ursula 0.11.3.dev0__py3-none-macosx_11_0_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlite_ursula/__init__.py +77 -0
- sqlite_ursula/__main__.py +7 -0
- sqlite_ursula/_args.py +39 -0
- sqlite_ursula/_connection.py +368 -0
- sqlite_ursula/_control.py +199 -0
- sqlite_ursula/_doctor.py +37 -0
- sqlite_ursula/_events.py +477 -0
- sqlite_ursula/_gate.py +136 -0
- sqlite_ursula/_host.py +142 -0
- sqlite_ursula/_lib/libsqlite_ursula_vfs.dylib +0 -0
- sqlite_ursula/_open.py +432 -0
- sqlite_ursula/errors.py +171 -0
- sqlite_ursula/langgraph.py +474 -0
- sqlite_ursula/py.typed +0 -0
- sqlite_ursula-0.11.3.dev0.dist-info/METADATA +183 -0
- sqlite_ursula-0.11.3.dev0.dist-info/RECORD +18 -0
- sqlite_ursula-0.11.3.dev0.dist-info/WHEEL +5 -0
- sqlite_ursula-0.11.3.dev0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""SQLite replicated through an Ursula stream by the sqlite-ursula-vfs loadable extension
|
|
2
|
+
(clients/sqlite-vfs), for the stdlib ``sqlite3`` module. The Python peer of the npm package
|
|
3
|
+
``@tonbo/sqlite-ursula``.
|
|
4
|
+
|
|
5
|
+
A commit is local: the extension appends it to a log it owns and replicates the log to the stream
|
|
6
|
+
in the background. ``db.synced()`` is the output gate::
|
|
7
|
+
|
|
8
|
+
import sqlite_ursula
|
|
9
|
+
|
|
10
|
+
db = sqlite_ursula.open("https://streams.example.com/acme/notes")
|
|
11
|
+
db.execute("CREATE TABLE IF NOT EXISTS notes(body TEXT)")
|
|
12
|
+
db.execute("INSERT INTO notes VALUES (?)", ("hello",))
|
|
13
|
+
db.commit()
|
|
14
|
+
db.synced() # this database's commits are in the stream, and this process still owns it
|
|
15
|
+
|
|
16
|
+
``aopen`` and ``db.asynced()`` are the same calls for asyncio, and ``gate`` / ``agate`` wrap a
|
|
17
|
+
block of work in the gate.
|
|
18
|
+
|
|
19
|
+
Module map:
|
|
20
|
+
|
|
21
|
+
- ``_host``: where the extension is, whether this Python can load it (``host_unsupported``), and
|
|
22
|
+
loading it.
|
|
23
|
+
- ``_control``: the process's control connection, calls of the extension's SQL functions,
|
|
24
|
+
``classify``, which asks the extension (``ursula_classify``) what a failure is, and the fork
|
|
25
|
+
state.
|
|
26
|
+
- ``_events``: the events thread, which waits for the extension's events and resolves
|
|
27
|
+
``synced()`` and ``asynced()``, completes ``open()`` and ``aopen()``, and runs the reset
|
|
28
|
+
callbacks.
|
|
29
|
+
- ``_connection``: :class:`UrsulaConnection`, the ``sqlite3.Connection`` subclass ``open``
|
|
30
|
+
returns, and its cursor, whose statements classify their errors.
|
|
31
|
+
- ``_open``: :func:`open`, :func:`aopen`, :func:`import_sqlite` and :func:`set_token`.
|
|
32
|
+
- ``_gate``: :func:`gate` and :func:`agate`, which hold a block's outcome until ``synced()``.
|
|
33
|
+
- ``_args``: the checks of their arguments, and of ``synced()``'s, as ``invalid_argument``.
|
|
34
|
+
- ``_doctor``: ``python -m sqlite_ursula``, which says whether this Python can host the extension.
|
|
35
|
+
- ``errors``: the four error classes.
|
|
36
|
+
- ``langgraph``: LangGraph's SQLite checkpointers and stores on a replicated database, for the
|
|
37
|
+
``langgraph`` extra. The package never imports it.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
from ._connection import UrsulaConnection
|
|
43
|
+
from ._connection import UrsulaCursor
|
|
44
|
+
from ._gate import agate
|
|
45
|
+
from ._gate import gate
|
|
46
|
+
from ._open import Imported
|
|
47
|
+
from ._open import aopen
|
|
48
|
+
from ._open import import_sqlite
|
|
49
|
+
from ._open import open # noqa: A004 - the package's entry point, as Node's `open`
|
|
50
|
+
from ._open import set_token
|
|
51
|
+
from .errors import Action
|
|
52
|
+
from .errors import UrsulaError
|
|
53
|
+
from .errors import UrsulaResetError
|
|
54
|
+
from .errors import UrsulaSetupError
|
|
55
|
+
from .errors import UrsulaUnavailableError
|
|
56
|
+
from .errors import UrsulaWriteRefusedError
|
|
57
|
+
|
|
58
|
+
__version__ = "0.11.3.dev0"
|
|
59
|
+
|
|
60
|
+
__all__ = [
|
|
61
|
+
"Action",
|
|
62
|
+
"Imported",
|
|
63
|
+
"UrsulaConnection",
|
|
64
|
+
"UrsulaCursor",
|
|
65
|
+
"UrsulaError",
|
|
66
|
+
"UrsulaResetError",
|
|
67
|
+
"UrsulaSetupError",
|
|
68
|
+
"UrsulaUnavailableError",
|
|
69
|
+
"UrsulaWriteRefusedError",
|
|
70
|
+
"__version__",
|
|
71
|
+
"agate",
|
|
72
|
+
"aopen",
|
|
73
|
+
"gate",
|
|
74
|
+
"import_sqlite",
|
|
75
|
+
"open",
|
|
76
|
+
"set_token",
|
|
77
|
+
]
|
sqlite_ursula/_args.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""The checks of the arguments the package takes, shared by ``open`` and the connection's calls: a
|
|
2
|
+
value it cannot take is ``invalid_argument`` (``fix``), never a ``TypeError`` from deeper down."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import math
|
|
7
|
+
import threading
|
|
8
|
+
|
|
9
|
+
from .errors import UrsulaSetupError
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def invalid(message: str) -> UrsulaSetupError:
|
|
13
|
+
return UrsulaSetupError(f"invalid_argument: {message}", "invalid_argument", "fix")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def seconds(name: str, value: object, *, positive: bool) -> float:
|
|
17
|
+
"""``value``, a number of seconds: positive, or zero or more, and at most what a wait takes
|
|
18
|
+
(``threading.TIMEOUT_MAX``)."""
|
|
19
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or math.isnan(value):
|
|
20
|
+
raise invalid(f"{name} must be a number of seconds, not {value!r}")
|
|
21
|
+
if value < 0 or (positive and value == 0):
|
|
22
|
+
raise invalid(f"{name} must be {'positive' if positive else 'zero or more'}, not {value!r}")
|
|
23
|
+
if value > threading.TIMEOUT_MAX:
|
|
24
|
+
raise invalid(
|
|
25
|
+
f"{name} must be at most {threading.TIMEOUT_MAX:g} s, not {value!r}: for no limit, "
|
|
26
|
+
"pass None where the call takes it"
|
|
27
|
+
)
|
|
28
|
+
return float(value)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def limit(timeout: object) -> float | None:
|
|
32
|
+
"""A wait's ``timeout``: zero or more seconds, or ``None`` for no limit."""
|
|
33
|
+
return None if timeout is None else seconds("timeout", timeout, positive=False)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def flag(name: str, value: object) -> bool:
|
|
37
|
+
if not isinstance(value, bool):
|
|
38
|
+
raise invalid(f"{name} must be a boolean, not {value!r}")
|
|
39
|
+
return value
|
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
"""The connection ``open`` returns: a ``sqlite3.Connection`` subclass, so every library that takes
|
|
2
|
+
one (LangGraph's ``SqliteSaver`` among them) takes it, and ``isinstance`` holds.
|
|
3
|
+
|
|
4
|
+
``Connection.execute``, ``commit`` and ``__exit__`` call SQLite directly, not through
|
|
5
|
+
``cursor()``, so the subclass overrides them along with ``cursor``, and its cursor overrides
|
|
6
|
+
``execute*`` and ``fetch*``. Each raises what ``classify`` (``_control``) makes of its error: an
|
|
7
|
+
:class:`UrsulaError` for a failure of the extension, every other error unchanged. On top of
|
|
8
|
+
``sqlite3.Connection`` it has ``name``, ``path``, ``synced()``, ``asynced()``, ``status()`` and the
|
|
9
|
+
reset callbacks.
|
|
10
|
+
|
|
11
|
+
A refused write or a reset is one of SQLite's I/O errors, on which SQLite rolls back the whole
|
|
12
|
+
transaction. A transaction helper's ``ROLLBACK`` then fails with "cannot rollback - no transaction
|
|
13
|
+
is active", and one that runs ``COMMIT`` in a ``finally`` block, as LangGraph's ``SqliteStore``
|
|
14
|
+
does, fails with "cannot commit - no transaction is active". Either would replace the error the
|
|
15
|
+
caller needs. So the connection remembers an :class:`UrsulaError` after which no transaction is
|
|
16
|
+
open, until a statement succeeds. A ``ROLLBACK`` run through ``execute`` that finds no transaction
|
|
17
|
+
then succeeds, since nothing is left to roll back. Through ``executescript``, which may hold more
|
|
18
|
+
statements after it, it raises the remembered error instead, and so do a ``COMMIT`` or ``END`` that
|
|
19
|
+
finds no transaction and a ``ROLLBACK TO`` or ``RELEASE`` that finds no savepoint. The legacy
|
|
20
|
+
``rollback()`` and ``commit()`` methods run nothing when no transaction is open. ``open()`` refuses
|
|
21
|
+
``autocommit=False``, the mode in which the connection is always in a transaction (see ``_open``),
|
|
22
|
+
and so does setting ``autocommit`` to ``False`` on a connection ``open()`` made.
|
|
23
|
+
|
|
24
|
+
In a child forked after the parent made the connection, its overridden calls raise ``forked``,
|
|
25
|
+
and ``close()`` does nothing: the file and its locks are the parent's. Its other methods
|
|
26
|
+
(``backup``, ``blobopen``, ``serialize`` and the like) are ``sqlite3.Connection``'s, and the
|
|
27
|
+
extension guards the bound file under them.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import logging
|
|
33
|
+
import sqlite3
|
|
34
|
+
import urllib.parse
|
|
35
|
+
from collections.abc import Callable
|
|
36
|
+
from typing import Any
|
|
37
|
+
from typing import TypeVar
|
|
38
|
+
|
|
39
|
+
from . import _control
|
|
40
|
+
from . import _events
|
|
41
|
+
from ._args import invalid
|
|
42
|
+
from .errors import UrsulaError
|
|
43
|
+
from .errors import UrsulaResetError
|
|
44
|
+
from .errors import again
|
|
45
|
+
|
|
46
|
+
log = logging.getLogger("sqlite_ursula")
|
|
47
|
+
|
|
48
|
+
#: SQLITE_ERROR: a SQL error, and every error of an Ursula SQL function.
|
|
49
|
+
SQLITE_ERROR = 1
|
|
50
|
+
#: SQLite's message for a ROLLBACK and for a COMMIT or END, and the start of its message for a
|
|
51
|
+
#: ROLLBACK TO or RELEASE, that finds no transaction or savepoint.
|
|
52
|
+
NO_TRANSACTION = "cannot rollback - no transaction is active"
|
|
53
|
+
NO_COMMIT = "cannot commit - no transaction is active"
|
|
54
|
+
NO_SAVEPOINT = "no such savepoint: "
|
|
55
|
+
|
|
56
|
+
T = TypeVar("T")
|
|
57
|
+
|
|
58
|
+
#: ``sqlite3.Connection``'s ``autocommit`` attribute (Python 3.12 or newer), or ``None``.
|
|
59
|
+
_AUTOCOMMIT: Any = vars(sqlite3.Connection).get("autocommit")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def autocommit_false() -> UrsulaError:
|
|
63
|
+
"""The refusal of ``autocommit=False``, as an ``open()`` argument or set later."""
|
|
64
|
+
return invalid(
|
|
65
|
+
"autocommit=False keeps the connection in a transaction, where a refused COMMIT is "
|
|
66
|
+
"hidden by the ROLLBACK that follows it: use the default transaction control, "
|
|
67
|
+
"isolation_level=None, or autocommit=True"
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
ResetCallback = Callable[[UrsulaResetError], object]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def database_of(path: str, read_only: bool) -> str:
|
|
75
|
+
"""``sqlite3.connect``'s ``database`` for the bound file ``path``: a ``mode=ro`` URI, which
|
|
76
|
+
takes ``uri=True``, for a read-only connection."""
|
|
77
|
+
return f"file:{urllib.parse.quote(path)}?mode=ro" if read_only else path
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class UrsulaCursor(sqlite3.Cursor):
|
|
81
|
+
"""A ``sqlite3.Cursor`` whose statements raise what their errors classify as."""
|
|
82
|
+
|
|
83
|
+
def _run(
|
|
84
|
+
self, f: Callable[..., T], *args: Any, rolled_back: Callable[[], T] | None = None
|
|
85
|
+
) -> T:
|
|
86
|
+
run = getattr(self.connection, "_run", None)
|
|
87
|
+
if run is None:
|
|
88
|
+
return f(*args)
|
|
89
|
+
return run(f, *args, rolled_back=rolled_back)
|
|
90
|
+
|
|
91
|
+
def execute(self, sql: str, parameters: Any = (), /) -> UrsulaCursor:
|
|
92
|
+
return self._run(super().execute, sql, parameters, rolled_back=lambda: self)
|
|
93
|
+
|
|
94
|
+
def executemany(self, sql: str, parameters: Any, /) -> UrsulaCursor:
|
|
95
|
+
return self._run(super().executemany, sql, parameters, rolled_back=lambda: self)
|
|
96
|
+
|
|
97
|
+
def executescript(self, script: str, /) -> UrsulaCursor:
|
|
98
|
+
return self._run(super().executescript, script)
|
|
99
|
+
|
|
100
|
+
def fetchone(self) -> Any:
|
|
101
|
+
return self._run(super().fetchone)
|
|
102
|
+
|
|
103
|
+
def fetchmany(self, size: int | None = None) -> list[Any]:
|
|
104
|
+
if size is None:
|
|
105
|
+
return self._run(super().fetchmany)
|
|
106
|
+
return self._run(super().fetchmany, size)
|
|
107
|
+
|
|
108
|
+
def fetchall(self) -> list[Any]:
|
|
109
|
+
return self._run(super().fetchall)
|
|
110
|
+
|
|
111
|
+
def __next__(self) -> Any:
|
|
112
|
+
return self._run(super().__next__)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class UrsulaConnection(sqlite3.Connection):
|
|
116
|
+
"""A connection to a database a stream holds.
|
|
117
|
+
|
|
118
|
+
Statements are local: commit, then call :meth:`synced` before anything outside the process
|
|
119
|
+
acts on what you wrote or read. Keep it open while a framework uses :attr:`path`: the binding
|
|
120
|
+
retires once its last connection closes.
|
|
121
|
+
|
|
122
|
+
Made with ``sqlite3.connect(..., factory=UrsulaConnection)`` outside ``open()`` it binds
|
|
123
|
+
nothing, classifies nothing, and works as a ``sqlite3.Connection``.
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
#: The stream's path, such as ``/acme/notes``, or ``""`` for a connection ``open()`` did not
|
|
127
|
+
#: make.
|
|
128
|
+
name: str
|
|
129
|
+
#: The local database file the extension bound, as ``status()["path"]`` reports it: the path
|
|
130
|
+
#: a framework opens with ``sqlite3.connect`` to use the same database.
|
|
131
|
+
path: str
|
|
132
|
+
|
|
133
|
+
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
134
|
+
super().__init__(*args, **kwargs)
|
|
135
|
+
database = args[0] if args else kwargs.get("database", "")
|
|
136
|
+
self.name = ""
|
|
137
|
+
self.path = str(database)
|
|
138
|
+
self._bound = False
|
|
139
|
+
self._read_only = False
|
|
140
|
+
self._busy = 5.0
|
|
141
|
+
self._generation = _control.generation()
|
|
142
|
+
self._closed = False
|
|
143
|
+
self._listeners: list[ResetCallback] = []
|
|
144
|
+
#: The reset this connection knows of, from an event, a statement or ``synced()``.
|
|
145
|
+
self._reset: UrsulaResetError | None = None
|
|
146
|
+
self._fired = False
|
|
147
|
+
#: An UrsulaError after which no transaction was open, until a statement succeeds.
|
|
148
|
+
self._rolled_back: UrsulaError | None = None
|
|
149
|
+
|
|
150
|
+
def _bind(self, name: str, path: str, read_only: bool, busy: float) -> None:
|
|
151
|
+
"""``open()``: the connection is on the binding of stream ``name``, whose file is
|
|
152
|
+
``path``, read-only when ``read_only``, with the busy timeout ``busy`` in seconds."""
|
|
153
|
+
self.name = name
|
|
154
|
+
self.path = path
|
|
155
|
+
self._bound = True
|
|
156
|
+
self._read_only = read_only
|
|
157
|
+
self._busy = busy
|
|
158
|
+
_events.track(self)
|
|
159
|
+
|
|
160
|
+
if _AUTOCOMMIT is not None:
|
|
161
|
+
|
|
162
|
+
@property
|
|
163
|
+
def autocommit(self) -> Any:
|
|
164
|
+
return _AUTOCOMMIT.__get__(self, type(self))
|
|
165
|
+
|
|
166
|
+
@autocommit.setter
|
|
167
|
+
def autocommit(self, value: Any) -> None:
|
|
168
|
+
if self._bound and value is False:
|
|
169
|
+
raise autocommit_false()
|
|
170
|
+
_AUTOCOMMIT.__set__(self, value)
|
|
171
|
+
|
|
172
|
+
# The statements.
|
|
173
|
+
|
|
174
|
+
def cursor(self, factory: Any = None) -> Any:
|
|
175
|
+
self._check_fork()
|
|
176
|
+
return super().cursor(UrsulaCursor if factory is None else factory)
|
|
177
|
+
|
|
178
|
+
def execute(self, sql: str, parameters: Any = (), /) -> Any:
|
|
179
|
+
return self.cursor().execute(sql, parameters)
|
|
180
|
+
|
|
181
|
+
def executemany(self, sql: str, parameters: Any, /) -> Any:
|
|
182
|
+
return self.cursor().executemany(sql, parameters)
|
|
183
|
+
|
|
184
|
+
def executescript(self, script: str, /) -> Any:
|
|
185
|
+
return self.cursor().executescript(script)
|
|
186
|
+
|
|
187
|
+
def commit(self) -> None:
|
|
188
|
+
self._run(super().commit)
|
|
189
|
+
|
|
190
|
+
def rollback(self) -> None:
|
|
191
|
+
self._run(super().rollback)
|
|
192
|
+
|
|
193
|
+
def __exit__(self, *exc: Any) -> Any:
|
|
194
|
+
return self._run(super().__exit__, *exc)
|
|
195
|
+
|
|
196
|
+
def close(self) -> None:
|
|
197
|
+
"""Closes the connection. Once a binding's last connection closes, replication drains in the
|
|
198
|
+
background and the binding retires. Use ``open()`` to use the database again.
|
|
199
|
+
|
|
200
|
+
In a child forked after the parent made the connection it closes nothing, since a thread of
|
|
201
|
+
the parent may have held the file's locks at the fork, and keeps the connection so that
|
|
202
|
+
collecting it closes nothing either.
|
|
203
|
+
"""
|
|
204
|
+
if self._inherited():
|
|
205
|
+
if not self._closed:
|
|
206
|
+
self._closed = True
|
|
207
|
+
_control._inherited.append(self)
|
|
208
|
+
return
|
|
209
|
+
super().close()
|
|
210
|
+
if not self._closed:
|
|
211
|
+
self._closed = True
|
|
212
|
+
if self._bound:
|
|
213
|
+
_events.untrack(self)
|
|
214
|
+
|
|
215
|
+
# What the extension adds.
|
|
216
|
+
|
|
217
|
+
def synced(self, timeout: float | None = None) -> None:
|
|
218
|
+
"""Returns once the stream holds every commit this process logged on this database before
|
|
219
|
+
the call, and a round trip that started after the call found this process still the owner
|
|
220
|
+
(design: Synced). Call it before anything outside the process acts on what you wrote or
|
|
221
|
+
read.
|
|
222
|
+
|
|
223
|
+
It covers every connection of the process on this database (:attr:`path`), a framework's
|
|
224
|
+
included, and no other database: gate each one an output depends on. It raises
|
|
225
|
+
``TimeoutError`` after ``timeout`` seconds (``None``: no limit), while the commits keep
|
|
226
|
+
replicating, and an :class:`UrsulaResetError` on a reset. Network failures never raise: it
|
|
227
|
+
waits. Ctrl-C interrupts it. On a reader cache it returns at once.
|
|
228
|
+
"""
|
|
229
|
+
_events.synced((self,), timeout)
|
|
230
|
+
|
|
231
|
+
async def asynced(self, timeout: float | None = None) -> None:
|
|
232
|
+
""":meth:`synced` for asyncio: the same guarantee and the same errors, and it waits on the
|
|
233
|
+
event loop instead of blocking it. Cancelling it ends the wait, while the commits keep
|
|
234
|
+
replicating."""
|
|
235
|
+
await _events.asynced((self,), timeout)
|
|
236
|
+
|
|
237
|
+
def status(self) -> dict[str, Any]:
|
|
238
|
+
"""``ursula_status`` of the database (design: Status)."""
|
|
239
|
+
self._check_fork()
|
|
240
|
+
return _control.call("SELECT ursula_status(?)", self.path)
|
|
241
|
+
|
|
242
|
+
def on_reset(self, callback: ResetCallback) -> ResetCallback:
|
|
243
|
+
"""Runs ``callback`` once when the database is reset, with the reset error, after its
|
|
244
|
+
pending ``synced()`` calls were rejected. It runs on a thread of the package's. Returns
|
|
245
|
+
``callback``, so it serves as a decorator."""
|
|
246
|
+
self._check_fork()
|
|
247
|
+
self._listeners.append(callback)
|
|
248
|
+
return callback
|
|
249
|
+
|
|
250
|
+
def off_reset(self, callback: ResetCallback) -> None:
|
|
251
|
+
"""Removes a callback :meth:`on_reset` added."""
|
|
252
|
+
self._check_fork()
|
|
253
|
+
if callback in self._listeners:
|
|
254
|
+
self._listeners.remove(callback)
|
|
255
|
+
|
|
256
|
+
# The events thread's.
|
|
257
|
+
|
|
258
|
+
def _reset_by(self, error: UrsulaResetError) -> None:
|
|
259
|
+
"""The database was reset: its callbacks run once."""
|
|
260
|
+
self._remember(error)
|
|
261
|
+
if self._fired or self._closed:
|
|
262
|
+
return
|
|
263
|
+
self._fired = True
|
|
264
|
+
for callback in list(self._listeners):
|
|
265
|
+
try:
|
|
266
|
+
callback(self._reset)
|
|
267
|
+
except Exception:
|
|
268
|
+
log.exception("an on_reset callback of %s raised", self.name)
|
|
269
|
+
|
|
270
|
+
def _current_reset(self) -> UrsulaResetError | None:
|
|
271
|
+
"""The reset ``ursula_status`` reports, if any."""
|
|
272
|
+
try:
|
|
273
|
+
reset = self.status().get("reset")
|
|
274
|
+
except Exception:
|
|
275
|
+
return None
|
|
276
|
+
return None if reset is None else _events.reset_error(reset)
|
|
277
|
+
|
|
278
|
+
# Classification.
|
|
279
|
+
|
|
280
|
+
def _inherited(self) -> bool:
|
|
281
|
+
"""Whether the connection is the parent's: bound before a fork this process came from."""
|
|
282
|
+
return self._bound and self._generation != _control.generation()
|
|
283
|
+
|
|
284
|
+
def _check_fork(self) -> None:
|
|
285
|
+
if self._inherited():
|
|
286
|
+
raise _control.forked_error()
|
|
287
|
+
|
|
288
|
+
def _usable(self) -> None:
|
|
289
|
+
"""Before a wait: raises ``forked`` in a forked child, and the reset the connection knows
|
|
290
|
+
of."""
|
|
291
|
+
self._check_fork()
|
|
292
|
+
if self._reset is not None:
|
|
293
|
+
raise again(self._reset)
|
|
294
|
+
|
|
295
|
+
def _remember(self, error: UrsulaResetError) -> None:
|
|
296
|
+
"""The first reset the connection learns of, which every later wait raises."""
|
|
297
|
+
if self._reset is None:
|
|
298
|
+
self._reset = error
|
|
299
|
+
|
|
300
|
+
def _in_transaction(self) -> bool:
|
|
301
|
+
try:
|
|
302
|
+
return self.in_transaction
|
|
303
|
+
except sqlite3.ProgrammingError:
|
|
304
|
+
return True
|
|
305
|
+
|
|
306
|
+
def _classify(self, error: BaseException) -> BaseException:
|
|
307
|
+
if not self._bound:
|
|
308
|
+
return error
|
|
309
|
+
classified = _control.classify(error, self.path)
|
|
310
|
+
if isinstance(classified, UrsulaResetError):
|
|
311
|
+
self._remember(classified)
|
|
312
|
+
return classified
|
|
313
|
+
|
|
314
|
+
def _run(
|
|
315
|
+
self, f: Callable[..., T], *args: Any, rolled_back: Callable[[], T] | None = None
|
|
316
|
+
) -> T:
|
|
317
|
+
"""Runs ``f``, a method that runs a statement, and raises what its error classifies as.
|
|
318
|
+
``rolled_back`` is what it returns for a ROLLBACK that found no transaction after an
|
|
319
|
+
UrsulaError, when it can report one."""
|
|
320
|
+
self._check_fork()
|
|
321
|
+
try:
|
|
322
|
+
result = f(*args)
|
|
323
|
+
except sqlite3.Error as error:
|
|
324
|
+
remembered = self._rolled_back
|
|
325
|
+
# SQLite tells these errors from others by their message alone: they are SQLITE_ERROR.
|
|
326
|
+
if remembered is not None and getattr(error, "sqlite_errorcode", None) == SQLITE_ERROR:
|
|
327
|
+
message = str(error)
|
|
328
|
+
if message == NO_TRANSACTION and rolled_back is not None:
|
|
329
|
+
self._rolled_back = None
|
|
330
|
+
return rolled_back()
|
|
331
|
+
if message in (NO_TRANSACTION, NO_COMMIT) or message.startswith(NO_SAVEPOINT):
|
|
332
|
+
raise again(remembered) from remembered.__cause__
|
|
333
|
+
classified = self._classify(error)
|
|
334
|
+
if isinstance(classified, UrsulaError) and not self._in_transaction():
|
|
335
|
+
self._rolled_back = classified
|
|
336
|
+
if classified is error:
|
|
337
|
+
raise
|
|
338
|
+
raise classified from error
|
|
339
|
+
self._rolled_back = None
|
|
340
|
+
return result
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def bound(db: object, caller: str) -> UrsulaConnection:
|
|
344
|
+
"""``db``, which ``caller`` (a gate, an adapter) takes: ``invalid_argument`` unless ``open()``
|
|
345
|
+
made it, and ``forked`` in a child forked after the parent made it."""
|
|
346
|
+
if not isinstance(db, UrsulaConnection) or not db._bound:
|
|
347
|
+
raise invalid(f"{caller} takes connections open() made, not {db!r}")
|
|
348
|
+
db._check_fork()
|
|
349
|
+
return db
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def on_binding(db: UrsulaConnection, cls: type[UrsulaConnection]) -> tuple[str, dict[str, Any]]:
|
|
353
|
+
"""``sqlite3.connect``'s ``database`` and keywords for another connection on the binding of
|
|
354
|
+
``db``, for an adapter's own connection, also one a library makes on a thread of its own, as
|
|
355
|
+
aiosqlite does.
|
|
356
|
+
|
|
357
|
+
The connection is of class ``cls``, a subclass of ``UrsulaConnection``, and bound as ``db``
|
|
358
|
+
is, so its statements raise what their errors classify as and it learns of a reset, it is
|
|
359
|
+
read-only when ``db`` is, and its busy timeout is ``db``'s. Like any connection on
|
|
360
|
+
``db.path``, it keeps the binding while it is open."""
|
|
361
|
+
name, path, read_only, busy = db.name, db.path, db._read_only, db._busy
|
|
362
|
+
|
|
363
|
+
def factory(*args: Any, **kwargs: Any) -> UrsulaConnection:
|
|
364
|
+
conn = cls(*args, **kwargs)
|
|
365
|
+
conn._bind(name, path, read_only, busy)
|
|
366
|
+
return conn
|
|
367
|
+
|
|
368
|
+
return database_of(path, read_only), {"factory": factory, "uri": read_only, "timeout": busy}
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""The process's control connection, the one way the package calls the extension's SQL functions
|
|
2
|
+
on it, ``classify``, and what a fork leaves of the process's state.
|
|
3
|
+
|
|
4
|
+
The registry of bound databases and the replication driver are the process's: one control
|
|
5
|
+
connection serves every thread. Its calls are short, since every call that could wait is made with
|
|
6
|
+
timeout 0, or on a connection of its own outside the lock (:func:`call_waiting`), so one lock
|
|
7
|
+
serializes them. The events thread (``_events``) takes the same lock while it applies events, so a
|
|
8
|
+
request or a ticket is registered before any event about it applies.
|
|
9
|
+
|
|
10
|
+
A fork copies the process without its threads. A child forked after the parent started the
|
|
11
|
+
extension (an open or an import) is marked forked, and the package's calls in it (``open``,
|
|
12
|
+
``import_sqlite``, ``set_token``) raise ``forked`` at once, before they could touch a lock that
|
|
13
|
+
another thread of the parent held at the fork. So do the calls of a connection the parent made
|
|
14
|
+
(``_connection``), whose ``close()`` touches nothing. A child forked before that starts the
|
|
15
|
+
extension as any process does. A platform without ``fork`` (Windows) registers no hook.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import os
|
|
22
|
+
import sqlite3
|
|
23
|
+
import threading
|
|
24
|
+
from collections.abc import Sequence
|
|
25
|
+
from typing import Any
|
|
26
|
+
|
|
27
|
+
from . import _host
|
|
28
|
+
from .errors import UrsulaError
|
|
29
|
+
from .errors import UrsulaSetupError
|
|
30
|
+
from .errors import error_of
|
|
31
|
+
from .errors import find_wrapped
|
|
32
|
+
|
|
33
|
+
_lock = threading.RLock()
|
|
34
|
+
_control: sqlite3.Connection | None = None
|
|
35
|
+
_library: _host.Library | None = None
|
|
36
|
+
#: Whether this process started the extension: an open or an import ran.
|
|
37
|
+
_started = False
|
|
38
|
+
#: Whether this process is a child forked after its parent started the extension.
|
|
39
|
+
_forked = False
|
|
40
|
+
#: Increases at each fork in the child: a connection made before it is the parent's.
|
|
41
|
+
_generation = 0
|
|
42
|
+
#: What a fork left in the child: kept, never closed, since a thread of the parent may have held
|
|
43
|
+
#: their locks at the fork.
|
|
44
|
+
_inherited: list[object] = []
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def lock() -> threading.RLock:
|
|
48
|
+
"""The lock of the control connection and of the events' state."""
|
|
49
|
+
return _lock
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def generation() -> int:
|
|
53
|
+
"""This process's generation: a connection made under another one belongs to the parent."""
|
|
54
|
+
return _generation
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def forked_error() -> UrsulaSetupError:
|
|
58
|
+
"""``forked``: what the guarded calls (``open``, ``import_sqlite``, ``set_token`` and the
|
|
59
|
+
overridden calls of an inherited connection) raise in a child forked after the parent started
|
|
60
|
+
the extension."""
|
|
61
|
+
return UrsulaSetupError(
|
|
62
|
+
"forked: this process forked after its parent opened an Ursula database, and the "
|
|
63
|
+
"databases and the replication of the parent are not this process's: open databases "
|
|
64
|
+
"after the fork (gunicorn's post_fork hook, Celery's worker_process_init signal, or "
|
|
65
|
+
"multiprocessing's spawn or forkserver start method)",
|
|
66
|
+
"forked",
|
|
67
|
+
"fix",
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def check_fork() -> None:
|
|
72
|
+
"""Raises ``forked`` in a child forked after the parent started the extension."""
|
|
73
|
+
if _forked:
|
|
74
|
+
raise forked_error()
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def start() -> None:
|
|
78
|
+
"""Marks the extension started: a child forked from now on is forked."""
|
|
79
|
+
global _started
|
|
80
|
+
check_fork()
|
|
81
|
+
_started = True
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def library() -> _host.Library:
|
|
85
|
+
"""The library the process loaded, loading it first."""
|
|
86
|
+
control()
|
|
87
|
+
assert _library is not None
|
|
88
|
+
return _library
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def control() -> sqlite3.Connection:
|
|
92
|
+
"""The control connection, loading the extension at the first call."""
|
|
93
|
+
global _control, _library
|
|
94
|
+
check_fork()
|
|
95
|
+
with _lock:
|
|
96
|
+
if _control is None:
|
|
97
|
+
lib = _host.library()
|
|
98
|
+
_control = _host.connection(lib)
|
|
99
|
+
_library = lib
|
|
100
|
+
return _control
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def connection() -> sqlite3.Connection:
|
|
104
|
+
"""A new connection with the process's extension loaded, for a call that waits."""
|
|
105
|
+
return _host.connection(library())
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def parsed(text: object) -> Any:
|
|
109
|
+
"""A SQL function's JSON result."""
|
|
110
|
+
return json.loads(text) if isinstance(text, str) and text else None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def call(sql: str, *args: object) -> Any:
|
|
114
|
+
"""One of the extension's SQL functions on the control connection: its JSON result, or its
|
|
115
|
+
error as what :func:`classify` makes of it."""
|
|
116
|
+
db = control()
|
|
117
|
+
with _lock:
|
|
118
|
+
return _result(db, sql, args)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def call_waiting(sql: str, *args: object) -> Any:
|
|
122
|
+
""":func:`call` for a function that can wait (``ursula_cancel`` while a disk job of the open
|
|
123
|
+
runs, ``ursula_import``): on a connection of its own, outside the lock, so the events thread
|
|
124
|
+
and the other calls go on meanwhile."""
|
|
125
|
+
db = connection()
|
|
126
|
+
try:
|
|
127
|
+
return _result(db, sql, args)
|
|
128
|
+
finally:
|
|
129
|
+
db.close()
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _result(db: sqlite3.Connection, sql: str, args: tuple[object, ...]) -> Any:
|
|
133
|
+
try:
|
|
134
|
+
row = db.execute(sql, args).fetchone()
|
|
135
|
+
except sqlite3.Error as error:
|
|
136
|
+
classified = classify(error, None)
|
|
137
|
+
if classified is error:
|
|
138
|
+
raise
|
|
139
|
+
raise classified from error
|
|
140
|
+
return parsed(row[0] if row else None)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def classify(error: BaseException, path: str | None) -> BaseException:
|
|
144
|
+
"""What to raise for ``error``, which a statement or an Ursula SQL function raised.
|
|
145
|
+
|
|
146
|
+
The :class:`UrsulaError` of what the extension's ``ursula_classify`` makes of it, or ``error``
|
|
147
|
+
itself when the extension says it is not its failure (a constraint, a syntax error, an I/O
|
|
148
|
+
error of a database used through ``ATTACH DATABASE``). ``path`` is the bound database the
|
|
149
|
+
statement ran on, whose state names a statement's I/O error, which says nothing.
|
|
150
|
+
"""
|
|
151
|
+
if isinstance(error, UrsulaError) or not isinstance(error, sqlite3.Error):
|
|
152
|
+
return error
|
|
153
|
+
code = getattr(error, "sqlite_errorcode", None)
|
|
154
|
+
db = _control
|
|
155
|
+
if not isinstance(code, int) or db is None or _forked:
|
|
156
|
+
return error
|
|
157
|
+
try:
|
|
158
|
+
with _lock:
|
|
159
|
+
row = db.execute("SELECT ursula_classify(?, ?, ?)", (str(error), code, path)).fetchone()
|
|
160
|
+
named = parsed(row[0] if row else None)
|
|
161
|
+
except (sqlite3.Error, ValueError):
|
|
162
|
+
return error
|
|
163
|
+
if not isinstance(named, dict):
|
|
164
|
+
return error
|
|
165
|
+
return error_of(named, str(named.get("message", error)), error)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def classify_wrapped(error: BaseException, paths: Sequence[str | None]) -> BaseException:
|
|
169
|
+
""":func:`classify` for an error a library may have wrapped: the first error of the chain
|
|
170
|
+
:func:`find_wrapped` walks that is an :class:`UrsulaError`, or that the extension classifies
|
|
171
|
+
on one of ``paths`` (the first that names it), as that :class:`UrsulaError`. So a failure of the
|
|
172
|
+
extension never passes for the wrapper's own kind of error. ``error`` itself otherwise."""
|
|
173
|
+
|
|
174
|
+
def match(next_: BaseException) -> UrsulaError | None:
|
|
175
|
+
if isinstance(next_, UrsulaError):
|
|
176
|
+
return next_
|
|
177
|
+
for path in paths:
|
|
178
|
+
classified = classify(next_, path)
|
|
179
|
+
if isinstance(classified, UrsulaError):
|
|
180
|
+
return classified
|
|
181
|
+
return None
|
|
182
|
+
|
|
183
|
+
found = find_wrapped(error, match)
|
|
184
|
+
return error if found is None else found
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _after_fork_in_child() -> None:
|
|
188
|
+
global _lock, _control, _forked, _generation
|
|
189
|
+
_lock = threading.RLock()
|
|
190
|
+
_generation += 1
|
|
191
|
+
if _control is not None:
|
|
192
|
+
_inherited.append(_control)
|
|
193
|
+
_control = None
|
|
194
|
+
if _started:
|
|
195
|
+
_forked = True
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
if hasattr(os, "register_at_fork"):
|
|
199
|
+
os.register_at_fork(after_in_child=_after_fork_in_child)
|