pi-durable-python 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pi_durable/__init__.py +320 -0
- pi_durable/context.py +328 -0
- pi_durable/delta.py +269 -0
- pi_durable/documents.py +366 -0
- pi_durable/entries.py +66 -0
- pi_durable/env/__init__.py +408 -0
- pi_durable/env/decode.py +48 -0
- pi_durable/env/line_scan.py +190 -0
- pi_durable/env/posix.py +1547 -0
- pi_durable/errors.py +89 -0
- pi_durable/harness/__init__.py +1 -0
- pi_durable/harness/agent.py +344 -0
- pi_durable/harness/compaction.py +673 -0
- pi_durable/harness/context.py +515 -0
- pi_durable/harness/define.py +111 -0
- pi_durable/harness/events.py +766 -0
- pi_durable/harness/generation.py +1079 -0
- pi_durable/harness/harness.py +570 -0
- pi_durable/harness/inbox.py +255 -0
- pi_durable/harness/live.py +374 -0
- pi_durable/harness/output.py +467 -0
- pi_durable/harness/prompt.py +288 -0
- pi_durable/harness/provider.py +78 -0
- pi_durable/harness/registry.py +136 -0
- pi_durable/harness/scheduler.py +1977 -0
- pi_durable/harness/submissions.py +357 -0
- pi_durable/harness/task_graph.py +334 -0
- pi_durable/harness/tool.py +1365 -0
- pi_durable/harness/types.py +772 -0
- pi_durable/harness/usage.py +88 -0
- pi_durable/harness/util.py +204 -0
- pi_durable/harness/validation.py +460 -0
- pi_durable/harness/view.py +324 -0
- pi_durable/ids.py +63 -0
- pi_durable/json.py +186 -0
- pi_durable/providers/__init__.py +362 -0
- pi_durable/providers/anthropic.py +603 -0
- pi_durable/providers/base.py +97 -0
- pi_durable/providers/json_parse.py +117 -0
- pi_durable/providers/openai.py +574 -0
- pi_durable/providers/request.py +35 -0
- pi_durable/providers/retry.py +127 -0
- pi_durable/providers/stream.py +183 -0
- pi_durable/providers/transcript.py +224 -0
- pi_durable/providers/usage.py +58 -0
- pi_durable/py.typed +0 -0
- pi_durable/search.py +115 -0
- pi_durable/session/__init__.py +5 -0
- pi_durable/session/drafts.py +762 -0
- pi_durable/session/forks.py +109 -0
- pi_durable/session/observation.py +374 -0
- pi_durable/session/session.py +1193 -0
- pi_durable/session/transaction.py +1088 -0
- pi_durable/storage/__init__.py +5 -0
- pi_durable/storage/jsonl/__init__.py +15 -0
- pi_durable/storage/jsonl/storage.py +958 -0
- pi_durable/storage/memory.py +889 -0
- pi_durable/storage/scan.py +57 -0
- pi_durable/storage/sqlite/__init__.py +26 -0
- pi_durable/storage/sqlite/database.py +62 -0
- pi_durable/storage/sqlite/migrations.py +191 -0
- pi_durable/storage/sqlite/posix.py +314 -0
- pi_durable/storage/sqlite/schema.py +97 -0
- pi_durable/storage/sqlite/storage.py +1130 -0
- pi_durable/tasks.py +89 -0
- pi_durable/testing/__init__.py +41 -0
- pi_durable/testing/assertions.py +44 -0
- pi_durable/testing/env_conformance.py +828 -0
- pi_durable/testing/runner.py +64 -0
- pi_durable/testing/storage_conformance.py +2008 -0
- pi_durable/testing/types.py +99 -0
- pi_durable/tool_search/__init__.py +159 -0
- pi_durable/tools/__init__.py +81 -0
- pi_durable/tools/_values.py +43 -0
- pi_durable/tools/bash.py +250 -0
- pi_durable/tools/edit.py +214 -0
- pi_durable/tools/edit_diff.py +452 -0
- pi_durable/tools/env.py +19 -0
- pi_durable/tools/file_mutation_queue.py +68 -0
- pi_durable/tools/image.py +248 -0
- pi_durable/tools/image_processor.py +77 -0
- pi_durable/tools/path_utils.py +111 -0
- pi_durable/tools/read.py +506 -0
- pi_durable/tools/write.py +71 -0
- pi_durable/truncate.py +185 -0
- pi_durable/types.py +1297 -0
- pi_durable_python-1.3.0.dist-info/METADATA +276 -0
- pi_durable_python-1.3.0.dist-info/RECORD +90 -0
- pi_durable_python-1.3.0.dist-info/WHEEL +4 -0
- pi_durable_python-1.3.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Cursor and order helpers shared by storage backends."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Any, TypeGuard
|
|
7
|
+
|
|
8
|
+
from ..errors import StorageRequestError
|
|
9
|
+
from ..types import Cursor, ScanOrder
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True, slots=True)
|
|
13
|
+
class ScanStart:
|
|
14
|
+
"""Where a built-in storage scan starts: its order and the last ID a previous page returned."""
|
|
15
|
+
|
|
16
|
+
order: ScanOrder
|
|
17
|
+
after: int | None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _is_safe_int(value: object) -> TypeGuard[int]:
|
|
21
|
+
return isinstance(value, int) and not isinstance(value, bool) and abs(value) <= 9_007_199_254_740_991
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def scan_start(
|
|
25
|
+
requested: ScanOrder | None,
|
|
26
|
+
cursor: Cursor | None,
|
|
27
|
+
fallback: ScanOrder,
|
|
28
|
+
) -> ScanStart:
|
|
29
|
+
"""Resolve a scan's order and position, rejecting malformed and conflicting cursors."""
|
|
30
|
+
|
|
31
|
+
if requested is not None and requested not in ("ascending", "descending"):
|
|
32
|
+
raise StorageRequestError(f"Invalid scan order: {requested}")
|
|
33
|
+
if cursor is None:
|
|
34
|
+
return ScanStart(order=requested or fallback, after=None)
|
|
35
|
+
after = cursor.get("after")
|
|
36
|
+
if not _is_safe_int(after):
|
|
37
|
+
raise StorageRequestError("Invalid storage cursor")
|
|
38
|
+
stored = cursor.get("order")
|
|
39
|
+
if stored is not None and stored not in ("ascending", "descending"):
|
|
40
|
+
raise StorageRequestError("Invalid storage cursor")
|
|
41
|
+
order = valid_order(stored) or fallback
|
|
42
|
+
if requested is not None and requested != order:
|
|
43
|
+
raise StorageRequestError(f"The cursor continues a {order} scan; the query asks for {requested}")
|
|
44
|
+
return ScanStart(order=order, after=after)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def valid_order(value: Any) -> ScanOrder | None:
|
|
48
|
+
return value if value in ("ascending", "descending") else None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def next_cursor(id: int, order: ScanOrder) -> dict[str, Any]:
|
|
52
|
+
"""Return the continuation of a scan whose last returned item has ``id``."""
|
|
53
|
+
|
|
54
|
+
return {"after": id, "order": order}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
__all__ = ["ScanStart", "next_cursor", "scan_start"]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Portable SQLite storage backend and migrations.
|
|
2
|
+
|
|
3
|
+
The host adapter lives in :mod:`pi_durable.storage.sqlite.posix`; this subpath stays free of it,
|
|
4
|
+
so importing the portable backend never selects a driver.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from .database import SqliteDatabase, SqliteExecutor, SqliteValue
|
|
8
|
+
from .migrations import (
|
|
9
|
+
CURRENT_SQLITE_SCHEMA_VERSION,
|
|
10
|
+
SQLITE_MIGRATIONS,
|
|
11
|
+
SqliteMigration,
|
|
12
|
+
apply_sqlite_migrations,
|
|
13
|
+
)
|
|
14
|
+
from .storage import SqliteStorage, SqliteStorageOptions
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"CURRENT_SQLITE_SCHEMA_VERSION",
|
|
18
|
+
"SQLITE_MIGRATIONS",
|
|
19
|
+
"SqliteDatabase",
|
|
20
|
+
"SqliteExecutor",
|
|
21
|
+
"SqliteMigration",
|
|
22
|
+
"SqliteStorage",
|
|
23
|
+
"SqliteStorageOptions",
|
|
24
|
+
"SqliteValue",
|
|
25
|
+
"apply_sqlite_migrations",
|
|
26
|
+
]
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Portable SQLite facade contracts, translated from v1.3.0 ``sqlite/database.ts``.
|
|
2
|
+
|
|
3
|
+
The core never imports a driver: it accepts any :class:`SqliteDatabase`, and the stdlib adapter in
|
|
4
|
+
:mod:`pi_durable.storage.sqlite.posix` supplies one. Operations return awaitables rather than being
|
|
5
|
+
``async`` functions so a caller's call order is the order the adapter observes, matching the
|
|
6
|
+
promise-returning upstream facade.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections.abc import Awaitable, Callable
|
|
12
|
+
from typing import Any, Protocol, TypeAlias, TypeVar
|
|
13
|
+
|
|
14
|
+
SqliteValue: TypeAlias = None | int | float | str | bytes
|
|
15
|
+
"""One positional binding accepted by the portable SQLite storage core."""
|
|
16
|
+
|
|
17
|
+
T = TypeVar("T")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class SqliteExecutor(Protocol):
|
|
21
|
+
"""Asynchronous SQL operations shared by a database and its transaction handles.
|
|
22
|
+
|
|
23
|
+
``exec`` runs SQL text without bindings and may contain several statements. ``run``, ``get``,
|
|
24
|
+
and ``all`` execute one statement with positional bindings. Adapters may cache prepared
|
|
25
|
+
statements by SQL text, so callers pass values as bindings instead of interpolating them.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
def exec(self, sql: str) -> Awaitable[None]:
|
|
29
|
+
"""Run one or more statements without bindings."""
|
|
30
|
+
|
|
31
|
+
def run(self, sql: str, *params: SqliteValue) -> Awaitable[None]:
|
|
32
|
+
"""Run one statement with positional bindings."""
|
|
33
|
+
|
|
34
|
+
def get(self, sql: str, *params: SqliteValue) -> Awaitable[dict[str, Any] | None]:
|
|
35
|
+
"""Return the first row of one statement, or ``None``."""
|
|
36
|
+
|
|
37
|
+
def all(self, sql: str, *params: SqliteValue) -> Awaitable[list[dict[str, Any]]]:
|
|
38
|
+
"""Return every row of one statement."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class SqliteDatabase(SqliteExecutor, Protocol):
|
|
42
|
+
"""Minimal database facade required by :class:`~pi_durable.storage.sqlite.SqliteStorage`.
|
|
43
|
+
|
|
44
|
+
``transaction`` passes the callback a transaction handle. All work in the transaction must use
|
|
45
|
+
that handle; the handle is invalid after the callback settles. Adapters must queue unrelated
|
|
46
|
+
operations and other transactions until the transaction finishes. The returned awaitable
|
|
47
|
+
settles after commit or rollback. Calling the database itself (including ``transaction`` or
|
|
48
|
+
``close``) from inside a callback therefore waits for that transaction and never settles.
|
|
49
|
+
|
|
50
|
+
When the callback raises, the adapter must roll the transaction back before re-raising that
|
|
51
|
+
same error. If rollback fails, it must raise a different error (here an ``ExceptionGroup``) so
|
|
52
|
+
callers cannot mistake the callback error for a guaranteed rollback.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
def transaction(self, callback: Callable[[SqliteExecutor], Awaitable[T]]) -> Awaitable[T]:
|
|
56
|
+
"""Run one callback as one SQL transaction."""
|
|
57
|
+
|
|
58
|
+
def close(self) -> Awaitable[None]:
|
|
59
|
+
"""Release the connection; later operations reject or fail at the driver."""
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
__all__ = ["SqliteDatabase", "SqliteExecutor", "SqliteValue"]
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Ordered SQLite schema history and atomic application, translated from v1.3.0 ``sqlite/migrations.ts``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
|
|
8
|
+
from .database import SqliteDatabase, SqliteExecutor
|
|
9
|
+
from .schema import DEFAULT_SQLITE_SCHEMA, SqliteSchema
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True, slots=True)
|
|
13
|
+
class SqliteMigration:
|
|
14
|
+
"""One versioned, order-sensitive group of schema statements."""
|
|
15
|
+
|
|
16
|
+
version: int
|
|
17
|
+
statements: Sequence[str]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def create_sqlite_migrations(schema: SqliteSchema) -> tuple[SqliteMigration, ...]:
|
|
21
|
+
"""Build immutable, ordered schema history for a set of SQLite identifiers."""
|
|
22
|
+
|
|
23
|
+
tables = schema.tables
|
|
24
|
+
indexes = schema.indexes
|
|
25
|
+
# next_id is TEXT because JavaScript's SQLite driver rejects INTEGER results outside its safe
|
|
26
|
+
# integer range. The Python port keeps the column shape for persisted-record parity.
|
|
27
|
+
initial_schema = (
|
|
28
|
+
f"""CREATE TABLE {tables['durableMetadata']} (
|
|
29
|
+
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
|
30
|
+
next_id TEXT NOT NULL,
|
|
31
|
+
next_seq INTEGER NOT NULL
|
|
32
|
+
) STRICT""",
|
|
33
|
+
f"INSERT INTO {tables['durableMetadata']} (singleton, next_id, next_seq) VALUES (1, '2', 1)",
|
|
34
|
+
f"""CREATE TABLE {tables['recordIds']} (
|
|
35
|
+
id INTEGER PRIMARY KEY,
|
|
36
|
+
record_type TEXT NOT NULL
|
|
37
|
+
CHECK (record_type IN ('conversation', 'entry', 'task', 'submission', 'document'))
|
|
38
|
+
) STRICT""",
|
|
39
|
+
f"""CREATE TABLE {tables['conversations']} (
|
|
40
|
+
id INTEGER PRIMARY KEY,
|
|
41
|
+
owner_conversation_id INTEGER,
|
|
42
|
+
owner_task_id INTEGER,
|
|
43
|
+
record TEXT NOT NULL CHECK (json_valid(record))
|
|
44
|
+
) STRICT""",
|
|
45
|
+
f"CREATE INDEX {indexes['conversationsByOwnerConversation']} "
|
|
46
|
+
f"ON {tables['conversations']} (owner_conversation_id, id)",
|
|
47
|
+
f"CREATE INDEX {indexes['conversationsByOwnerTask']} "
|
|
48
|
+
f"ON {tables['conversations']} (owner_task_id, id)",
|
|
49
|
+
f"""CREATE TABLE {tables['entries']} (
|
|
50
|
+
id INTEGER PRIMARY KEY,
|
|
51
|
+
conversation_id INTEGER NOT NULL,
|
|
52
|
+
head INTEGER,
|
|
53
|
+
commit_seq INTEGER NOT NULL,
|
|
54
|
+
record TEXT NOT NULL CHECK (json_valid(record))
|
|
55
|
+
) STRICT""",
|
|
56
|
+
f"CREATE INDEX {indexes['entriesByConversation']} "
|
|
57
|
+
f"ON {tables['entries']} (conversation_id, id DESC)",
|
|
58
|
+
f"CREATE INDEX {indexes['entryHeadsByConversation']} "
|
|
59
|
+
f"ON {tables['entries']} (conversation_id, id DESC) WHERE head IS NOT NULL",
|
|
60
|
+
f"""CREATE TABLE {tables['tasks']} (
|
|
61
|
+
id INTEGER PRIMARY KEY,
|
|
62
|
+
conversation_id INTEGER NOT NULL,
|
|
63
|
+
kind TEXT NOT NULL,
|
|
64
|
+
status TEXT NOT NULL
|
|
65
|
+
CHECK (status IN ('pending', 'running', 'waiting', 'completing', 'terminal')),
|
|
66
|
+
abort_requested INTEGER NOT NULL CHECK (abort_requested IN (0, 1)),
|
|
67
|
+
background INTEGER NOT NULL CHECK (background IN (0, 1)),
|
|
68
|
+
record TEXT NOT NULL CHECK (json_valid(record))
|
|
69
|
+
) STRICT""",
|
|
70
|
+
f"CREATE INDEX {indexes['tasksByStatus']} ON {tables['tasks']} (status, id)",
|
|
71
|
+
f"CREATE INDEX {indexes['tasksByConversation']} ON {tables['tasks']} (conversation_id, id)",
|
|
72
|
+
f"CREATE INDEX {indexes['tasksByKind']} ON {tables['tasks']} (kind, id)",
|
|
73
|
+
f"CREATE INDEX {indexes['tasksByAbortRequested']} "
|
|
74
|
+
f"ON {tables['tasks']} (abort_requested, id)",
|
|
75
|
+
f"CREATE INDEX {indexes['tasksByBackground']} ON {tables['tasks']} (background, id)",
|
|
76
|
+
f"""CREATE TABLE {tables['submissions']} (
|
|
77
|
+
id INTEGER PRIMARY KEY,
|
|
78
|
+
conversation_id INTEGER NOT NULL,
|
|
79
|
+
request_id TEXT,
|
|
80
|
+
status TEXT NOT NULL CHECK (status IN ('queued', 'placed', 'done', 'unanswered')),
|
|
81
|
+
record TEXT NOT NULL CHECK (json_valid(record))
|
|
82
|
+
) STRICT""",
|
|
83
|
+
f"CREATE INDEX {indexes['submissionsByRequest']} "
|
|
84
|
+
f"ON {tables['submissions']} (conversation_id, request_id)",
|
|
85
|
+
f"CREATE INDEX {indexes['submissionsByConversation']} "
|
|
86
|
+
f"ON {tables['submissions']} (conversation_id, id)",
|
|
87
|
+
f"CREATE INDEX {indexes['submissionsByStatus']} ON {tables['submissions']} (status, id)",
|
|
88
|
+
f"""CREATE TABLE {tables['documents']} (
|
|
89
|
+
id INTEGER PRIMARY KEY,
|
|
90
|
+
kind TEXT NOT NULL,
|
|
91
|
+
family INTEGER NOT NULL CHECK (family IN (0, 1)),
|
|
92
|
+
key_value TEXT NOT NULL,
|
|
93
|
+
scope_kind TEXT NOT NULL CHECK (scope_kind IN ('session', 'conversation', 'task')),
|
|
94
|
+
owner_id INTEGER NOT NULL,
|
|
95
|
+
created_at INTEGER NOT NULL,
|
|
96
|
+
retired_at INTEGER,
|
|
97
|
+
record TEXT NOT NULL CHECK (json_valid(record))
|
|
98
|
+
) STRICT""",
|
|
99
|
+
f"""CREATE INDEX {indexes['documentsByAddress']} ON {tables['documents']}
|
|
100
|
+
(kind, scope_kind, owner_id, family, key_value, created_at DESC, retired_at)""",
|
|
101
|
+
f"CREATE INDEX {indexes['documentsByScope']} "
|
|
102
|
+
f"ON {tables['documents']} (scope_kind, owner_id, id)",
|
|
103
|
+
f"CREATE INDEX {indexes['documentsByScopeKind']} "
|
|
104
|
+
f"ON {tables['documents']} (scope_kind, owner_id, kind, id)",
|
|
105
|
+
f"""CREATE TABLE {tables['documentRevisions']} (
|
|
106
|
+
document_id INTEGER NOT NULL,
|
|
107
|
+
seq INTEGER NOT NULL,
|
|
108
|
+
kind TEXT NOT NULL CHECK (kind IN ('base', 'delta')),
|
|
109
|
+
version INTEGER NOT NULL,
|
|
110
|
+
content TEXT NOT NULL CHECK (json_valid(content)),
|
|
111
|
+
PRIMARY KEY (document_id, seq)
|
|
112
|
+
) STRICT""",
|
|
113
|
+
f"CREATE INDEX {indexes['documentRevisionsByKind']} "
|
|
114
|
+
f"ON {tables['documentRevisions']} (document_id, kind, seq DESC)",
|
|
115
|
+
)
|
|
116
|
+
return (SqliteMigration(version=1, statements=initial_schema),)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
SQLITE_MIGRATIONS = create_sqlite_migrations(DEFAULT_SQLITE_SCHEMA)
|
|
120
|
+
CURRENT_SQLITE_SCHEMA_VERSION = SQLITE_MIGRATIONS[-1].version if SQLITE_MIGRATIONS else 0
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
async def apply_sqlite_migrations(
|
|
124
|
+
database: SqliteDatabase,
|
|
125
|
+
migrations: Sequence[SqliteMigration] | None = None,
|
|
126
|
+
) -> None:
|
|
127
|
+
"""Apply all pending schema migrations atomically."""
|
|
128
|
+
|
|
129
|
+
await apply_sqlite_migrations_for_schema(
|
|
130
|
+
database,
|
|
131
|
+
DEFAULT_SQLITE_SCHEMA,
|
|
132
|
+
SQLITE_MIGRATIONS if migrations is None else migrations,
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
async def apply_sqlite_migrations_for_schema(
|
|
137
|
+
database: SqliteDatabase,
|
|
138
|
+
schema: SqliteSchema,
|
|
139
|
+
migrations: Sequence[SqliteMigration] | None = None,
|
|
140
|
+
) -> None:
|
|
141
|
+
"""Apply migrations rendered for a particular schema."""
|
|
142
|
+
|
|
143
|
+
if migrations is None:
|
|
144
|
+
migrations = create_sqlite_migrations(schema)
|
|
145
|
+
for index, migration in enumerate(migrations):
|
|
146
|
+
if migration.version != index + 1:
|
|
147
|
+
raise RuntimeError("Durable SQLite migrations must have contiguous versions starting at 1")
|
|
148
|
+
|
|
149
|
+
durable_schema = schema.tables["durableSchema"]
|
|
150
|
+
|
|
151
|
+
async def migrate(transaction: SqliteExecutor) -> None:
|
|
152
|
+
await transaction.exec(
|
|
153
|
+
f"""CREATE TABLE IF NOT EXISTS {durable_schema} (
|
|
154
|
+
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
|
155
|
+
version INTEGER NOT NULL CHECK (version >= 0)
|
|
156
|
+
) STRICT"""
|
|
157
|
+
)
|
|
158
|
+
await transaction.run(
|
|
159
|
+
f"INSERT OR IGNORE INTO {durable_schema} (singleton, version) VALUES (1, 0)"
|
|
160
|
+
)
|
|
161
|
+
row = await transaction.get(
|
|
162
|
+
f"SELECT version FROM {durable_schema} WHERE singleton = 1"
|
|
163
|
+
)
|
|
164
|
+
if row is None:
|
|
165
|
+
raise RuntimeError("Durable SQLite schema metadata is missing")
|
|
166
|
+
current_version = migrations[-1].version if migrations else 0
|
|
167
|
+
if row["version"] > current_version:
|
|
168
|
+
raise RuntimeError(
|
|
169
|
+
f"Durable SQLite schema version {row['version']} is newer than supported "
|
|
170
|
+
f"version {current_version}"
|
|
171
|
+
)
|
|
172
|
+
for migration in migrations:
|
|
173
|
+
if migration.version <= row["version"]:
|
|
174
|
+
continue
|
|
175
|
+
for statement in migration.statements:
|
|
176
|
+
await transaction.exec(statement)
|
|
177
|
+
await transaction.run(
|
|
178
|
+
f"UPDATE {durable_schema} SET version = ? WHERE singleton = 1", migration.version
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
await database.transaction(migrate)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
__all__ = [
|
|
185
|
+
"CURRENT_SQLITE_SCHEMA_VERSION",
|
|
186
|
+
"SQLITE_MIGRATIONS",
|
|
187
|
+
"SqliteMigration",
|
|
188
|
+
"apply_sqlite_migrations",
|
|
189
|
+
"apply_sqlite_migrations_for_schema",
|
|
190
|
+
"create_sqlite_migrations",
|
|
191
|
+
]
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
"""Stdlib ``sqlite3`` adapter for the portable SQLite core.
|
|
2
|
+
|
|
3
|
+
Python replaces the host-specific Node driver with this adapter, as it replaces
|
|
4
|
+
``NodeExecutionEnv`` with :class:`~pi_durable.env.posix.PosixExecutionEnv`. It supplies the
|
|
5
|
+
:class:`~pi_durable.storage.sqlite.database.SqliteDatabase` facade: calls run in call order, a
|
|
6
|
+
transaction holds the connection until its callback settles, and prepared statements are reused
|
|
7
|
+
across ordinary operations and transaction handles.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
import sqlite3
|
|
14
|
+
from collections.abc import Awaitable, Callable
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, NotRequired, TypedDict, TypeVar
|
|
17
|
+
|
|
18
|
+
from .database import SqliteExecutor, SqliteValue
|
|
19
|
+
from .schema import validate_sqlite_schema_prefix
|
|
20
|
+
from .storage import SqliteStorage, SqliteStorageOptions
|
|
21
|
+
|
|
22
|
+
T = TypeVar("T")
|
|
23
|
+
|
|
24
|
+
_DEFAULT_WAL_AUTO_CHECKPOINT_PAGES = 1_000
|
|
25
|
+
_DEFAULT_BUSY_TIMEOUT_MS = 5_000
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class PosixSqliteDatabaseOptions(TypedDict):
|
|
29
|
+
"""Connection settings for a stdlib-backed SQLite file."""
|
|
30
|
+
|
|
31
|
+
wal_auto_checkpoint_pages: NotRequired[int]
|
|
32
|
+
"""SQLite WAL auto-checkpoint threshold in pages; 0 disables it (SQLite default: 1000)."""
|
|
33
|
+
|
|
34
|
+
busy_timeout_ms: NotRequired[int]
|
|
35
|
+
"""Time SQLite waits for a competing file lock (SQLite default: 0; here: 5000 ms)."""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class PosixSqliteStorageOptions(PosixSqliteDatabaseOptions):
|
|
39
|
+
"""Connection settings plus durable storage options for a stdlib-backed file."""
|
|
40
|
+
|
|
41
|
+
prefix: NotRequired[str]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class _TransactionScope:
|
|
45
|
+
"""Whether a transaction handle still belongs to a live transaction."""
|
|
46
|
+
|
|
47
|
+
__slots__ = ("active",)
|
|
48
|
+
|
|
49
|
+
def __init__(self) -> None:
|
|
50
|
+
self.active = True
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class _SerialOperationQueue:
|
|
54
|
+
"""Runs operations in call order; an asynchronous operation holds the queue until it settles."""
|
|
55
|
+
|
|
56
|
+
__slots__ = ("_tail", "_tasks")
|
|
57
|
+
|
|
58
|
+
def __init__(self) -> None:
|
|
59
|
+
self._tail: asyncio.Future[None] | None = None
|
|
60
|
+
self._tasks: set[asyncio.Task[None]] = set()
|
|
61
|
+
|
|
62
|
+
def run(self, operation: Callable[[], Awaitable[T]]) -> asyncio.Future[T]:
|
|
63
|
+
loop = asyncio.get_running_loop()
|
|
64
|
+
result: asyncio.Future[T] = loop.create_future()
|
|
65
|
+
barrier: asyncio.Future[None] = loop.create_future()
|
|
66
|
+
previous, self._tail = self._tail, barrier
|
|
67
|
+
|
|
68
|
+
async def runner() -> None:
|
|
69
|
+
if previous is not None:
|
|
70
|
+
await previous
|
|
71
|
+
try:
|
|
72
|
+
value = await operation()
|
|
73
|
+
except BaseException as error:
|
|
74
|
+
if not result.done():
|
|
75
|
+
result.set_exception(error)
|
|
76
|
+
else:
|
|
77
|
+
if not result.done():
|
|
78
|
+
result.set_result(value)
|
|
79
|
+
finally:
|
|
80
|
+
if not barrier.done():
|
|
81
|
+
barrier.set_result(None)
|
|
82
|
+
|
|
83
|
+
task = loop.create_task(runner())
|
|
84
|
+
self._tasks.add(task)
|
|
85
|
+
task.add_done_callback(self._tasks.discard)
|
|
86
|
+
return result
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class _PosixSqliteExecutor:
|
|
90
|
+
"""One statement per ``run``/``get``/``all`` on a shared connection and statement cache."""
|
|
91
|
+
|
|
92
|
+
__slots__ = ("_connection", "_statements", "_preparer")
|
|
93
|
+
|
|
94
|
+
def __init__(
|
|
95
|
+
self,
|
|
96
|
+
connection: sqlite3.Connection,
|
|
97
|
+
statements: dict[str, sqlite3.Cursor],
|
|
98
|
+
prepare: Callable[[str], sqlite3.Cursor] | None = None,
|
|
99
|
+
) -> None:
|
|
100
|
+
self._connection = connection
|
|
101
|
+
self._statements = statements
|
|
102
|
+
self._preparer = self.prepare if prepare is None else prepare
|
|
103
|
+
|
|
104
|
+
def prepare(self, sql: str) -> sqlite3.Cursor:
|
|
105
|
+
"""Return a cursor for ``sql``; a subclass may count preparation misses."""
|
|
106
|
+
|
|
107
|
+
return self._connection.cursor()
|
|
108
|
+
|
|
109
|
+
def exec(self, sql: str) -> asyncio.Future[None]:
|
|
110
|
+
return self._run(lambda: self._execute_script(sql))
|
|
111
|
+
|
|
112
|
+
def run(self, sql: str, *params: SqliteValue) -> asyncio.Future[None]:
|
|
113
|
+
def execute() -> None:
|
|
114
|
+
self._statement(sql).execute(sql, params)
|
|
115
|
+
|
|
116
|
+
return self._run(execute)
|
|
117
|
+
|
|
118
|
+
def get(self, sql: str, *params: SqliteValue) -> asyncio.Future[dict[str, Any] | None]:
|
|
119
|
+
def execute() -> dict[str, Any] | None:
|
|
120
|
+
cursor = self._statement(sql)
|
|
121
|
+
cursor.execute(sql, params)
|
|
122
|
+
row = cursor.fetchone()
|
|
123
|
+
return None if row is None else dict(row)
|
|
124
|
+
|
|
125
|
+
return self._run(execute)
|
|
126
|
+
|
|
127
|
+
def all(self, sql: str, *params: SqliteValue) -> asyncio.Future[list[dict[str, Any]]]:
|
|
128
|
+
def execute() -> list[dict[str, Any]]:
|
|
129
|
+
cursor = self._statement(sql)
|
|
130
|
+
cursor.execute(sql, params)
|
|
131
|
+
return [dict(row) for row in cursor.fetchall()]
|
|
132
|
+
|
|
133
|
+
return self._run(execute)
|
|
134
|
+
|
|
135
|
+
def _execute_script(self, sql: str) -> None:
|
|
136
|
+
self._connection.executescript(sql)
|
|
137
|
+
|
|
138
|
+
def _statement(self, sql: str) -> sqlite3.Cursor:
|
|
139
|
+
cursor = self._statements.get(sql)
|
|
140
|
+
if cursor is None:
|
|
141
|
+
cursor = self._preparer(sql)
|
|
142
|
+
self._statements[sql] = cursor
|
|
143
|
+
return cursor
|
|
144
|
+
|
|
145
|
+
def _run(self, operation: Callable[[], T]) -> asyncio.Future[T]:
|
|
146
|
+
raise NotImplementedError
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class _PosixSqliteTransaction(_PosixSqliteExecutor):
|
|
150
|
+
"""A transaction handle: runs synchronously inside the owning transaction until it settles."""
|
|
151
|
+
|
|
152
|
+
__slots__ = ("_scope",)
|
|
153
|
+
|
|
154
|
+
def __init__(
|
|
155
|
+
self,
|
|
156
|
+
connection: sqlite3.Connection,
|
|
157
|
+
statements: dict[str, sqlite3.Cursor],
|
|
158
|
+
prepare: Callable[[str], sqlite3.Cursor],
|
|
159
|
+
scope: _TransactionScope,
|
|
160
|
+
) -> None:
|
|
161
|
+
super().__init__(connection, statements, prepare)
|
|
162
|
+
self._scope = scope
|
|
163
|
+
|
|
164
|
+
def _execute_script(self, sql: str) -> None:
|
|
165
|
+
for statement in _complete_statements(sql):
|
|
166
|
+
self._connection.execute(statement)
|
|
167
|
+
|
|
168
|
+
def _run(self, operation: Callable[[], T]) -> asyncio.Future[T]:
|
|
169
|
+
future: asyncio.Future[T] = asyncio.get_running_loop().create_future()
|
|
170
|
+
if not self._scope.active:
|
|
171
|
+
future.set_exception(RuntimeError("SQLite transaction handle is no longer active"))
|
|
172
|
+
return future
|
|
173
|
+
try:
|
|
174
|
+
future.set_result(operation())
|
|
175
|
+
except BaseException as error:
|
|
176
|
+
future.set_exception(error)
|
|
177
|
+
return future
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
class PosixSqliteDatabase(_PosixSqliteExecutor):
|
|
181
|
+
"""``SqliteDatabase`` facade backed by Python's stdlib :mod:`sqlite3`."""
|
|
182
|
+
|
|
183
|
+
__slots__ = ("_closing", "_closed", "_queue")
|
|
184
|
+
|
|
185
|
+
def __init__(self, connection: sqlite3.Connection) -> None:
|
|
186
|
+
super().__init__(connection, {})
|
|
187
|
+
self._queue = _SerialOperationQueue()
|
|
188
|
+
self._closed = False
|
|
189
|
+
self._closing: asyncio.Future[None] | None = None
|
|
190
|
+
|
|
191
|
+
def transaction(self, callback: Callable[[SqliteExecutor], Awaitable[T]]) -> asyncio.Future[T]:
|
|
192
|
+
async def run_transaction() -> T:
|
|
193
|
+
scope = _TransactionScope()
|
|
194
|
+
transaction = _PosixSqliteTransaction(
|
|
195
|
+
self._connection, self._statements, self._preparer, scope
|
|
196
|
+
)
|
|
197
|
+
self._connection.execute("BEGIN IMMEDIATE")
|
|
198
|
+
try:
|
|
199
|
+
result = await callback(transaction)
|
|
200
|
+
scope.active = False
|
|
201
|
+
self._connection.execute("COMMIT")
|
|
202
|
+
return result
|
|
203
|
+
except BaseException as error:
|
|
204
|
+
scope.active = False
|
|
205
|
+
try:
|
|
206
|
+
self._connection.execute("ROLLBACK")
|
|
207
|
+
except Exception as rollback_error:
|
|
208
|
+
if isinstance(error, Exception):
|
|
209
|
+
raise ExceptionGroup(
|
|
210
|
+
"SQLite transaction failed and rollback failed",
|
|
211
|
+
[error, rollback_error],
|
|
212
|
+
) from None
|
|
213
|
+
raise rollback_error from None
|
|
214
|
+
raise
|
|
215
|
+
|
|
216
|
+
return self._queue.run(run_transaction)
|
|
217
|
+
|
|
218
|
+
def close(self) -> asyncio.Future[None]:
|
|
219
|
+
if self._closing is None:
|
|
220
|
+
async def close_database() -> None:
|
|
221
|
+
self._close()
|
|
222
|
+
|
|
223
|
+
self._closing = self._queue.run(close_database)
|
|
224
|
+
return self._closing
|
|
225
|
+
|
|
226
|
+
def _close(self) -> None:
|
|
227
|
+
if self._closed:
|
|
228
|
+
return
|
|
229
|
+
self._closed = True
|
|
230
|
+
self._statements.clear()
|
|
231
|
+
try:
|
|
232
|
+
self._connection.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
233
|
+
finally:
|
|
234
|
+
self._connection.close()
|
|
235
|
+
|
|
236
|
+
def _run(self, operation: Callable[[], T]) -> asyncio.Future[T]:
|
|
237
|
+
async def call() -> T:
|
|
238
|
+
return operation()
|
|
239
|
+
|
|
240
|
+
return self._queue.run(call)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _complete_statements(sql: str) -> list[str]:
|
|
244
|
+
"""Split SQL text into complete statements, so a transaction handle can run a script."""
|
|
245
|
+
|
|
246
|
+
statements: list[str] = []
|
|
247
|
+
pending = ""
|
|
248
|
+
for character in sql:
|
|
249
|
+
pending += character
|
|
250
|
+
if character == ";" and sqlite3.complete_statement(pending):
|
|
251
|
+
statements.append(pending)
|
|
252
|
+
pending = ""
|
|
253
|
+
if pending.strip():
|
|
254
|
+
statements.append(pending)
|
|
255
|
+
return statements
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
async def open_posix_sqlite_database(
|
|
259
|
+
path: str,
|
|
260
|
+
options: PosixSqliteDatabaseOptions | None = None,
|
|
261
|
+
) -> PosixSqliteDatabase:
|
|
262
|
+
"""Open and configure a stdlib-backed SQLite database facade."""
|
|
263
|
+
|
|
264
|
+
checkpoint_pages = (
|
|
265
|
+
_DEFAULT_WAL_AUTO_CHECKPOINT_PAGES
|
|
266
|
+
if options is None
|
|
267
|
+
else options.get("wal_auto_checkpoint_pages", _DEFAULT_WAL_AUTO_CHECKPOINT_PAGES)
|
|
268
|
+
)
|
|
269
|
+
timeout_ms = (
|
|
270
|
+
_DEFAULT_BUSY_TIMEOUT_MS
|
|
271
|
+
if options is None
|
|
272
|
+
else options.get("busy_timeout_ms", _DEFAULT_BUSY_TIMEOUT_MS)
|
|
273
|
+
)
|
|
274
|
+
if path != ":memory:":
|
|
275
|
+
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
|
276
|
+
connection = sqlite3.connect(path, timeout=timeout_ms / 1000, isolation_level=None)
|
|
277
|
+
connection.row_factory = sqlite3.Row
|
|
278
|
+
database = PosixSqliteDatabase(connection)
|
|
279
|
+
try:
|
|
280
|
+
await database.exec("PRAGMA journal_mode = WAL")
|
|
281
|
+
await database.exec("PRAGMA synchronous = NORMAL")
|
|
282
|
+
await database.exec(f"PRAGMA wal_autocheckpoint = {checkpoint_pages}")
|
|
283
|
+
return database
|
|
284
|
+
except BaseException:
|
|
285
|
+
try:
|
|
286
|
+
await database.close()
|
|
287
|
+
except Exception:
|
|
288
|
+
pass
|
|
289
|
+
raise
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
async def open_posix_sqlite_storage(
|
|
293
|
+
path: str,
|
|
294
|
+
options: PosixSqliteStorageOptions | None = None,
|
|
295
|
+
) -> SqliteStorage:
|
|
296
|
+
"""Open or create file-backed durable storage, optionally under a table prefix."""
|
|
297
|
+
|
|
298
|
+
prefix = None if options is None else options.get("prefix")
|
|
299
|
+
if prefix is not None:
|
|
300
|
+
validate_sqlite_schema_prefix(prefix)
|
|
301
|
+
database = await open_posix_sqlite_database(path, options)
|
|
302
|
+
storage_options: SqliteStorageOptions | None = (
|
|
303
|
+
None if prefix is None else {"prefix": prefix}
|
|
304
|
+
)
|
|
305
|
+
return await SqliteStorage.open(database, storage_options)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
__all__ = [
|
|
309
|
+
"PosixSqliteDatabase",
|
|
310
|
+
"PosixSqliteDatabaseOptions",
|
|
311
|
+
"PosixSqliteStorageOptions",
|
|
312
|
+
"open_posix_sqlite_database",
|
|
313
|
+
"open_posix_sqlite_storage",
|
|
314
|
+
]
|