echoact 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- echoact/__init__.py +3 -0
- echoact/__main__.py +117 -0
- echoact/app.py +315 -0
- echoact/audio/__init__.py +0 -0
- echoact/audio/devices.py +192 -0
- echoact/audio/player.py +611 -0
- echoact/audio/wav.py +854 -0
- echoact/config/__init__.py +0 -0
- echoact/config/budget.py +370 -0
- echoact/config/settings.py +1244 -0
- echoact/db/__init__.py +0 -0
- echoact/db/backup.py +2429 -0
- echoact/db/migrations.py +434 -0
- echoact/db/schema.sql +214 -0
- echoact/db/store.py +2062 -0
- echoact/diagnostics.py +902 -0
- echoact/domain.py +487 -0
- echoact/engine/__init__.py +0 -0
- echoact/engine/container.py +843 -0
- echoact/engine/protocol.py +241 -0
- echoact/engine/runtime.py +324 -0
- echoact/engine/supervisor.py +961 -0
- echoact/engine/worker.py +659 -0
- echoact/errors.py +281 -0
- echoact/instance.py +172 -0
- echoact/jobs/__init__.py +0 -0
- echoact/jobs/engine.py +776 -0
- echoact/jobs/request.py +300 -0
- echoact/mcp/__init__.py +0 -0
- echoact/mcp/__main__.py +50 -0
- echoact/mcp/client.py +202 -0
- echoact/mcp/config.py +112 -0
- echoact/mcp/server.py +340 -0
- echoact/models/__init__.py +0 -0
- echoact/models/catalog.py +273 -0
- echoact/models/manifest.py +278 -0
- echoact/models/registry.py +1551 -0
- echoact/paths.py +93 -0
- echoact/policy.py +189 -0
- echoact/security/__init__.py +0 -0
- echoact/security/credentials.py +930 -0
- echoact/security/ratelimit.py +534 -0
- echoact/service/__init__.py +20 -0
- echoact/service/app.py +182 -0
- echoact/service/deps.py +563 -0
- echoact/service/errors.py +241 -0
- echoact/service/routes.py +1125 -0
- echoact/service/schemas.py +509 -0
- echoact/service/server.py +270 -0
- echoact/text/__init__.py +0 -0
- echoact/text/language.py +44 -0
- echoact/text/loader.py +577 -0
- echoact/text/normalize.py +924 -0
- echoact/text/segment.py +499 -0
- echoact/text/sniff.py +1202 -0
- echoact/ui/__init__.py +0 -0
- echoact/ui/bridge.py +50 -0
- echoact/ui/controls.py +360 -0
- echoact/ui/credential_dialog.py +131 -0
- echoact/ui/fonts.py +94 -0
- echoact/ui/i18n.py +260 -0
- echoact/ui/icons.py +440 -0
- echoact/ui/library.py +1642 -0
- echoact/ui/licence.py +162 -0
- echoact/ui/main_window.py +1202 -0
- echoact/ui/mcp_setup.py +494 -0
- echoact/ui/models_view.py +1142 -0
- echoact/ui/notifications.py +202 -0
- echoact/ui/reading.py +494 -0
- echoact/ui/settings_view.py +2258 -0
- echoact/ui/status_view.py +1193 -0
- echoact/ui/theme.py +579 -0
- echoact/util/__init__.py +0 -0
- echoact/util/ids.py +62 -0
- echoact/util/logging.py +127 -0
- echoact-0.1.0.dist-info/METADATA +162 -0
- echoact-0.1.0.dist-info/RECORD +80 -0
- echoact-0.1.0.dist-info/WHEEL +4 -0
- echoact-0.1.0.dist-info/entry_points.txt +3 -0
- echoact-0.1.0.dist-info/licenses/LICENSE +21 -0
echoact/db/migrations.py
ADDED
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
"""Versioned, forward-only schema migrations.
|
|
2
|
+
|
|
3
|
+
N-15 sets the two rules this module exists for:
|
|
4
|
+
|
|
5
|
+
* A recoverable backup is secured *before* any data format change. The copy
|
|
6
|
+
is taken with SQLite's online backup API, which yields a consistent file
|
|
7
|
+
even while the write-ahead log is in use, and it is committed to the
|
|
8
|
+
``backups`` table before the first migration runs so the record survives a
|
|
9
|
+
migration that then fails.
|
|
10
|
+
* An older application that opens a newer data format must not modify it
|
|
11
|
+
destructively. Every open therefore reads the version first and refuses
|
|
12
|
+
outright when it is beyond what this build understands. Refusing is not a
|
|
13
|
+
fallback for "try and see": a v2 build may have added a column this build
|
|
14
|
+
would silently drop out of every ``INSERT``.
|
|
15
|
+
|
|
16
|
+
Forward-only is deliberate. A downgrade path would have to invent what the
|
|
17
|
+
removed information used to be, and N-15's answer to going backwards is the
|
|
18
|
+
backup taken here, not a reverse migration.
|
|
19
|
+
|
|
20
|
+
The error translation for the whole ``echoact.db`` package lives here rather
|
|
21
|
+
than in ``store`` because ``store`` imports this module and both need it;
|
|
22
|
+
rule 3 of the working notes forbids letting ``sqlite3.Error`` escape either.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import re
|
|
28
|
+
import sqlite3
|
|
29
|
+
import time
|
|
30
|
+
from collections.abc import Callable
|
|
31
|
+
from dataclasses import dataclass
|
|
32
|
+
from functools import lru_cache
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
from typing import Final
|
|
35
|
+
|
|
36
|
+
from .. import __version__
|
|
37
|
+
from ..errors import Code, EchoActError
|
|
38
|
+
from ..paths import data_dir
|
|
39
|
+
from ..util.ids import backup_id as new_backup_id
|
|
40
|
+
from ..util.ids import now as wall_now
|
|
41
|
+
|
|
42
|
+
#: The newest schema this build can read and write. A database at a higher
|
|
43
|
+
#: version is refused; a database at a lower one is migrated up.
|
|
44
|
+
SUPPORTED_SCHEMA_VERSION: Final = 1
|
|
45
|
+
|
|
46
|
+
_SCHEMA_FILE: Final = Path(__file__).with_name("schema.sql")
|
|
47
|
+
_SECTION = re.compile(r"^--\s*@section:\s*(\w+)\s*$", re.MULTILINE)
|
|
48
|
+
_WORD = re.compile(r"[A-Za-z_][A-Za-z_0-9]*")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# ======================================================================
|
|
52
|
+
# Error translation
|
|
53
|
+
# ======================================================================
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def translate_sqlite_error(exc: BaseException, *, retry_after_s: float | None = None) -> EchoActError:
|
|
57
|
+
"""Map a raw driver failure onto the one exception type that leaves this
|
|
58
|
+
package, with the code Section 5.3 names for it.
|
|
59
|
+
|
|
60
|
+
N-14 forbids two outcomes in particular: waiting indefinitely on a lock,
|
|
61
|
+
and reporting success. A lock that outlives the busy timeout arrives
|
|
62
|
+
here as ``OperationalError("database is locked")`` and becomes
|
|
63
|
+
``DB_LOCKED`` -- a retryable refusal with a hint -- rather than a stall.
|
|
64
|
+
A full disk arrives as "database or disk is full" and becomes
|
|
65
|
+
``STORAGE_FULL``, which is *not* retryable, because retrying without
|
|
66
|
+
freeing space cannot succeed.
|
|
67
|
+
"""
|
|
68
|
+
text = str(exc).lower()
|
|
69
|
+
if "disk is full" in text or "disk full" in text or "database or disk is full" in text:
|
|
70
|
+
return EchoActError(Code.STORAGE_FULL, detail={"driver": type(exc).__name__}, cause=exc)
|
|
71
|
+
if "locked" in text or "busy" in text:
|
|
72
|
+
return EchoActError(
|
|
73
|
+
Code.DB_LOCKED,
|
|
74
|
+
detail={"driver": type(exc).__name__},
|
|
75
|
+
retry_after_s=retry_after_s,
|
|
76
|
+
cause=exc,
|
|
77
|
+
)
|
|
78
|
+
if isinstance(exc, sqlite3.IntegrityError):
|
|
79
|
+
# A constraint violation is this application contradicting itself --
|
|
80
|
+
# a duplicate identifier, a segment pointing at no job. It is a
|
|
81
|
+
# defect, not an operating condition, so it must not look retryable.
|
|
82
|
+
return EchoActError(
|
|
83
|
+
Code.INTERNAL, "The database rejected an inconsistent write.", cause=exc
|
|
84
|
+
)
|
|
85
|
+
if isinstance(exc, sqlite3.ProgrammingError) or any(
|
|
86
|
+
marker in text for marker in ("no such column", "no such table", "syntax error")
|
|
87
|
+
):
|
|
88
|
+
# A malformed statement is a defect in this application, and
|
|
89
|
+
# DB_UNAVAILABLE would advertise it as worth retrying (N-23).
|
|
90
|
+
return EchoActError(Code.INTERNAL, "The database rejected a malformed query.", cause=exc)
|
|
91
|
+
if isinstance(exc, OSError) and getattr(exc, "errno", None) == 28: # ENOSPC
|
|
92
|
+
return EchoActError(Code.STORAGE_FULL, cause=exc)
|
|
93
|
+
return EchoActError(
|
|
94
|
+
Code.DB_UNAVAILABLE, detail={"driver": type(exc).__name__}, cause=exc
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# ======================================================================
|
|
99
|
+
# schema.sql
|
|
100
|
+
# ======================================================================
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@lru_cache(maxsize=1)
|
|
104
|
+
def _sections() -> dict[str, str]:
|
|
105
|
+
"""Split ``schema.sql`` on its ``-- @section:`` markers."""
|
|
106
|
+
text = _SCHEMA_FILE.read_text(encoding="utf-8")
|
|
107
|
+
marks = list(_SECTION.finditer(text))
|
|
108
|
+
if not marks:
|
|
109
|
+
raise EchoActError(Code.INTERNAL, "schema.sql has no @section markers")
|
|
110
|
+
out: dict[str, str] = {}
|
|
111
|
+
for i, m in enumerate(marks):
|
|
112
|
+
end = marks[i + 1].start() if i + 1 < len(marks) else len(text)
|
|
113
|
+
out[m.group(1)] = text[m.end() : end]
|
|
114
|
+
return out
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def schema_section(name: str) -> str:
|
|
118
|
+
return _sections()[name]
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def split_statements(sql: str) -> list[str]:
|
|
122
|
+
"""Split a script into statements, keeping trigger bodies whole.
|
|
123
|
+
|
|
124
|
+
``executescript`` is not used anywhere in this module: it commits any
|
|
125
|
+
open transaction before it runs, which would defeat the one-transaction-
|
|
126
|
+
per-migration rule this module is built around. So the splitting is done
|
|
127
|
+
here, and it has to understand that the ``;`` inside a ``CREATE TRIGGER
|
|
128
|
+
... BEGIN ... END;`` body does not end a statement.
|
|
129
|
+
"""
|
|
130
|
+
out: list[str] = []
|
|
131
|
+
cur: list[str] = []
|
|
132
|
+
block = 0
|
|
133
|
+
i = 0
|
|
134
|
+
n = len(sql)
|
|
135
|
+
while i < n:
|
|
136
|
+
ch = sql[i]
|
|
137
|
+
if ch == "'":
|
|
138
|
+
j = i + 1
|
|
139
|
+
while j < n:
|
|
140
|
+
if sql[j] == "'":
|
|
141
|
+
if j + 1 < n and sql[j + 1] == "'":
|
|
142
|
+
j += 2
|
|
143
|
+
continue
|
|
144
|
+
break
|
|
145
|
+
j += 1
|
|
146
|
+
cur.append(sql[i : j + 1])
|
|
147
|
+
i = j + 1
|
|
148
|
+
continue
|
|
149
|
+
if sql.startswith("--", i):
|
|
150
|
+
nl = sql.find("\n", i)
|
|
151
|
+
i = n if nl < 0 else nl + 1
|
|
152
|
+
cur.append("\n")
|
|
153
|
+
continue
|
|
154
|
+
word = _WORD.match(sql, i)
|
|
155
|
+
if word:
|
|
156
|
+
upper = word.group(0).upper()
|
|
157
|
+
if upper == "BEGIN":
|
|
158
|
+
block += 1
|
|
159
|
+
elif upper == "END":
|
|
160
|
+
block = max(0, block - 1)
|
|
161
|
+
cur.append(word.group(0))
|
|
162
|
+
i = word.end()
|
|
163
|
+
continue
|
|
164
|
+
if ch == ";" and block == 0:
|
|
165
|
+
stmt = "".join(cur).strip()
|
|
166
|
+
if stmt:
|
|
167
|
+
out.append(stmt)
|
|
168
|
+
cur = []
|
|
169
|
+
i += 1
|
|
170
|
+
continue
|
|
171
|
+
cur.append(ch)
|
|
172
|
+
i += 1
|
|
173
|
+
tail = "".join(cur).strip()
|
|
174
|
+
if tail:
|
|
175
|
+
out.append(tail)
|
|
176
|
+
return out
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _run(conn: sqlite3.Connection, sql: str) -> None:
|
|
180
|
+
for statement in split_statements(sql):
|
|
181
|
+
conn.execute(statement)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
# ======================================================================
|
|
185
|
+
# Capability probes
|
|
186
|
+
# ======================================================================
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def fts5_available(conn: sqlite3.Connection) -> bool:
|
|
190
|
+
"""Whether this SQLite build can create an FTS5 index (F-40).
|
|
191
|
+
|
|
192
|
+
Probed with ``PRAGMA compile_options`` rather than by attempting the
|
|
193
|
+
``CREATE`` and catching the failure, because the attempt would have to
|
|
194
|
+
happen inside the migration's transaction and a failed DDL statement
|
|
195
|
+
there is a rollback we do not want to reason about.
|
|
196
|
+
"""
|
|
197
|
+
try:
|
|
198
|
+
return any(row[0] == "ENABLE_FTS5" for row in conn.execute("PRAGMA compile_options"))
|
|
199
|
+
except sqlite3.Error:
|
|
200
|
+
return False
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def has_fts_index(conn: sqlite3.Connection) -> bool:
|
|
204
|
+
"""Whether *this database* carries the FTS5 objects.
|
|
205
|
+
|
|
206
|
+
Availability is a property of the build; presence is a property of the
|
|
207
|
+
file, and they can disagree. The index is never added to an existing
|
|
208
|
+
database outside a migration: its triggers fire on every document write,
|
|
209
|
+
so a build without FTS5 could not so much as save a document into a file
|
|
210
|
+
that has them. That makes adding them a format change, which N-15 puts
|
|
211
|
+
behind a version bump and a backup.
|
|
212
|
+
"""
|
|
213
|
+
try:
|
|
214
|
+
row = conn.execute(
|
|
215
|
+
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'documents_fts'"
|
|
216
|
+
).fetchone()
|
|
217
|
+
except sqlite3.Error:
|
|
218
|
+
return False
|
|
219
|
+
return row is not None
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
# ======================================================================
|
|
223
|
+
# Migrations
|
|
224
|
+
# ======================================================================
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
@dataclass(frozen=True, slots=True)
|
|
228
|
+
class Migration:
|
|
229
|
+
version: int
|
|
230
|
+
description: str
|
|
231
|
+
apply: Callable[[sqlite3.Connection], None]
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _migration_1(conn: sqlite3.Connection) -> None:
|
|
235
|
+
"""The initial schema.
|
|
236
|
+
|
|
237
|
+
The FTS5 section is applied only where the build supports it; F-40's
|
|
238
|
+
search then runs over the index, and ``Store`` falls back to ``LIKE``
|
|
239
|
+
where it is absent so that search degrades rather than vanishing.
|
|
240
|
+
"""
|
|
241
|
+
_run(conn, schema_section("core"))
|
|
242
|
+
if fts5_available(conn):
|
|
243
|
+
_run(conn, schema_section("fts5"))
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
MIGRATIONS: Final[tuple[Migration, ...]] = (
|
|
247
|
+
Migration(1, "initial schema", _migration_1),
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
@dataclass(frozen=True, slots=True)
|
|
252
|
+
class MigrationReport:
|
|
253
|
+
from_version: int
|
|
254
|
+
to_version: int
|
|
255
|
+
applied: tuple[int, ...]
|
|
256
|
+
backup_path: Path | None
|
|
257
|
+
fts5: bool
|
|
258
|
+
|
|
259
|
+
@property
|
|
260
|
+
def changed(self) -> bool:
|
|
261
|
+
return bool(self.applied)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def current_version(conn: sqlite3.Connection) -> int:
|
|
265
|
+
"""0 for a database that has never been migrated."""
|
|
266
|
+
try:
|
|
267
|
+
row = conn.execute(
|
|
268
|
+
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'schema_version'"
|
|
269
|
+
).fetchone()
|
|
270
|
+
if row is None:
|
|
271
|
+
return 0
|
|
272
|
+
found = conn.execute("SELECT max(version) FROM schema_version").fetchone()[0]
|
|
273
|
+
except sqlite3.Error as exc:
|
|
274
|
+
raise translate_sqlite_error(exc) from exc
|
|
275
|
+
return int(found or 0)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def assert_compatible(conn: sqlite3.Connection) -> int:
|
|
279
|
+
"""N-15: refuse a database written by a newer build, without touching it.
|
|
280
|
+
|
|
281
|
+
Returns the version so a caller can decide whether to migrate. The code
|
|
282
|
+
is ``BACKUP_INCOMPATIBLE`` because it is the only non-retryable
|
|
283
|
+
"incompatible version" code in the catalogue; ``DB_UNAVAILABLE`` is
|
|
284
|
+
marked retryable and would invite a client to try again at something that
|
|
285
|
+
will never change on its own, which N-23 forbids.
|
|
286
|
+
"""
|
|
287
|
+
version = current_version(conn)
|
|
288
|
+
if version > SUPPORTED_SCHEMA_VERSION:
|
|
289
|
+
raise EchoActError(
|
|
290
|
+
Code.BACKUP_INCOMPATIBLE,
|
|
291
|
+
"This database was written by a newer version of EchoAct "
|
|
292
|
+
f"(data format v{version}); this build understands v{SUPPORTED_SCHEMA_VERSION}. "
|
|
293
|
+
"It has not been modified.",
|
|
294
|
+
detail={"found_version": version, "supported_version": SUPPORTED_SCHEMA_VERSION},
|
|
295
|
+
)
|
|
296
|
+
return version
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def default_backup_dir() -> Path:
|
|
300
|
+
"""Where a pre-migration copy goes.
|
|
301
|
+
|
|
302
|
+
``echoact.paths`` names every other location; it has no backup directory
|
|
303
|
+
yet, so this module keeps the choice in one place until it does.
|
|
304
|
+
"""
|
|
305
|
+
return data_dir() / "backups"
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _item_count(conn: sqlite3.Connection) -> int:
|
|
309
|
+
total = 0
|
|
310
|
+
for table in ("documents", "jobs", "results"):
|
|
311
|
+
try:
|
|
312
|
+
total += int(conn.execute(f"SELECT count(*) FROM {table}").fetchone()[0])
|
|
313
|
+
except sqlite3.Error:
|
|
314
|
+
continue
|
|
315
|
+
return total
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def take_backup(
|
|
319
|
+
conn: sqlite3.Connection,
|
|
320
|
+
*,
|
|
321
|
+
directory: Path | None = None,
|
|
322
|
+
kind: str = "pre_migration",
|
|
323
|
+
at: float | None = None,
|
|
324
|
+
note: str | None = None,
|
|
325
|
+
) -> Path:
|
|
326
|
+
"""Copy the live database to a new file and record it (N-15).
|
|
327
|
+
|
|
328
|
+
The online backup API is used rather than a file copy because the source
|
|
329
|
+
is open in write-ahead mode: a byte copy of the main file alone would
|
|
330
|
+
miss everything still in the log.
|
|
331
|
+
"""
|
|
332
|
+
moment = wall_now() if at is None else at
|
|
333
|
+
version = current_version(conn)
|
|
334
|
+
target_dir = directory or default_backup_dir()
|
|
335
|
+
stamp = time.strftime("%Y%m%dT%H%M%S", time.gmtime(moment))
|
|
336
|
+
dest = target_dir / f"{kind}-v{version}-{stamp}-{new_backup_id()}.sqlite3"
|
|
337
|
+
try:
|
|
338
|
+
target_dir.mkdir(parents=True, exist_ok=True)
|
|
339
|
+
destination = sqlite3.connect(dest)
|
|
340
|
+
try:
|
|
341
|
+
conn.backup(destination)
|
|
342
|
+
finally:
|
|
343
|
+
destination.close()
|
|
344
|
+
size = dest.stat().st_size
|
|
345
|
+
items = _item_count(conn)
|
|
346
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
347
|
+
conn.execute(
|
|
348
|
+
"INSERT INTO backups (backup_id, kind, created_at, location, byte_size,"
|
|
349
|
+
" item_count, schema_version, app_version, verified, note)"
|
|
350
|
+
" VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?)",
|
|
351
|
+
(
|
|
352
|
+
new_backup_id(),
|
|
353
|
+
kind,
|
|
354
|
+
moment,
|
|
355
|
+
str(dest),
|
|
356
|
+
size,
|
|
357
|
+
items,
|
|
358
|
+
version,
|
|
359
|
+
__version__,
|
|
360
|
+
note,
|
|
361
|
+
),
|
|
362
|
+
)
|
|
363
|
+
conn.execute("COMMIT")
|
|
364
|
+
except (sqlite3.Error, OSError) as exc:
|
|
365
|
+
raise translate_sqlite_error(exc) from exc
|
|
366
|
+
return dest
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def migrate(
|
|
370
|
+
conn: sqlite3.Connection,
|
|
371
|
+
*,
|
|
372
|
+
backup_dir: Path | None = None,
|
|
373
|
+
at: float | None = None,
|
|
374
|
+
) -> MigrationReport:
|
|
375
|
+
"""Bring a database up to ``SUPPORTED_SCHEMA_VERSION``.
|
|
376
|
+
|
|
377
|
+
Each pending migration runs in its own transaction together with its
|
|
378
|
+
``schema_version`` row, so a failure half way through leaves the database
|
|
379
|
+
at the last version that completed rather than in a shape no build
|
|
380
|
+
recognises -- which is N-14's "previously sound data is preserved even if
|
|
381
|
+
a save fails midway" applied to the schema itself.
|
|
382
|
+
"""
|
|
383
|
+
start = assert_compatible(conn)
|
|
384
|
+
pending = tuple(m for m in MIGRATIONS if m.version > start)
|
|
385
|
+
backup: Path | None = None
|
|
386
|
+
|
|
387
|
+
# Creating an empty database is not a format change: there is nothing
|
|
388
|
+
# recoverable to secure. Every later step is.
|
|
389
|
+
if pending and start > 0:
|
|
390
|
+
backup = take_backup(conn, directory=backup_dir, at=at)
|
|
391
|
+
|
|
392
|
+
applied: list[int] = []
|
|
393
|
+
for migration in pending:
|
|
394
|
+
try:
|
|
395
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
396
|
+
migration.apply(conn)
|
|
397
|
+
conn.execute(
|
|
398
|
+
"INSERT INTO schema_version (version, applied_at, description) VALUES (?, ?, ?)",
|
|
399
|
+
(migration.version, wall_now() if at is None else at, migration.description),
|
|
400
|
+
)
|
|
401
|
+
conn.execute("COMMIT")
|
|
402
|
+
except (sqlite3.Error, OSError) as exc:
|
|
403
|
+
try:
|
|
404
|
+
conn.execute("ROLLBACK")
|
|
405
|
+
except sqlite3.Error:
|
|
406
|
+
pass
|
|
407
|
+
raise translate_sqlite_error(exc) from exc
|
|
408
|
+
applied.append(migration.version)
|
|
409
|
+
|
|
410
|
+
return MigrationReport(
|
|
411
|
+
from_version=start,
|
|
412
|
+
to_version=current_version(conn),
|
|
413
|
+
applied=tuple(applied),
|
|
414
|
+
backup_path=backup,
|
|
415
|
+
fts5=has_fts_index(conn),
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
__all__ = [
|
|
420
|
+
"MIGRATIONS",
|
|
421
|
+
"SUPPORTED_SCHEMA_VERSION",
|
|
422
|
+
"Migration",
|
|
423
|
+
"MigrationReport",
|
|
424
|
+
"assert_compatible",
|
|
425
|
+
"current_version",
|
|
426
|
+
"default_backup_dir",
|
|
427
|
+
"fts5_available",
|
|
428
|
+
"has_fts_index",
|
|
429
|
+
"migrate",
|
|
430
|
+
"schema_section",
|
|
431
|
+
"split_statements",
|
|
432
|
+
"take_backup",
|
|
433
|
+
"translate_sqlite_error",
|
|
434
|
+
]
|
echoact/db/schema.sql
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
-- EchoAct schema, version 1.
|
|
2
|
+
--
|
|
3
|
+
-- Section 4.2 fixes what each subject holds; this file adds nothing to that
|
|
4
|
+
-- list except the three things a requirement forces:
|
|
5
|
+
--
|
|
6
|
+
-- * jobs.model_id, denormalised out of the settings JSON, because F-40
|
|
7
|
+
-- filters history by model and a filter over a JSON blob cannot use an
|
|
8
|
+
-- index.
|
|
9
|
+
-- * results.integrity_state / verified_at, because F-45 requires a missing
|
|
10
|
+
-- or corrupted result file to be *detected* and offered for delete or
|
|
11
|
+
-- regenerate; a finding that lives only in the reconciling process is
|
|
12
|
+
-- lost the moment the window closes.
|
|
13
|
+
-- * schema_version and backups, which N-15 needs so that a format change
|
|
14
|
+
-- can be preceded by a recoverable backup and an older build can tell
|
|
15
|
+
-- that it is looking at a newer format.
|
|
16
|
+
--
|
|
17
|
+
-- Read by echoact.db.migrations, which splits the file on the
|
|
18
|
+
-- "-- @section:" markers. The fts5 section is applied only where the
|
|
19
|
+
-- SQLite build has FTS5; echoact.db.store falls back to LIKE where it does
|
|
20
|
+
-- not, so search degrades rather than disappearing (F-40).
|
|
21
|
+
|
|
22
|
+
-- @section: core
|
|
23
|
+
|
|
24
|
+
CREATE TABLE schema_version (
|
|
25
|
+
version INTEGER PRIMARY KEY,
|
|
26
|
+
applied_at REAL NOT NULL,
|
|
27
|
+
description TEXT NOT NULL
|
|
28
|
+
);
|
|
29
|
+
|
|
30
|
+
-- ---------------------------------------------------------------- 4.2 Document
|
|
31
|
+
CREATE TABLE documents (
|
|
32
|
+
document_id TEXT PRIMARY KEY,
|
|
33
|
+
title TEXT NOT NULL,
|
|
34
|
+
body TEXT NOT NULL,
|
|
35
|
+
created_at REAL NOT NULL,
|
|
36
|
+
modified_at REAL NOT NULL,
|
|
37
|
+
version INTEGER NOT NULL DEFAULT 1
|
|
38
|
+
);
|
|
39
|
+
|
|
40
|
+
-- F-40 searches titles; NOCASE so a title search matches the way a reader
|
|
41
|
+
-- expects. Korean is unaffected by case folding, Latin titles are not.
|
|
42
|
+
CREATE INDEX documents_by_title ON documents (title COLLATE NOCASE);
|
|
43
|
+
CREATE INDEX documents_by_modified ON documents (modified_at DESC);
|
|
44
|
+
|
|
45
|
+
-- ------------------------------------------------- 4.2 Integration permission
|
|
46
|
+
-- Deliberately holds no credential and no verifier. N-17 keeps the verifier
|
|
47
|
+
-- in echoact.security and out of backups; this table is the permission
|
|
48
|
+
-- record F-71 displays and F-61 revokes.
|
|
49
|
+
CREATE TABLE clients (
|
|
50
|
+
client_id TEXT PRIMARY KEY,
|
|
51
|
+
label TEXT NOT NULL,
|
|
52
|
+
capabilities TEXT NOT NULL, -- JSON array of domain.Capability values
|
|
53
|
+
active INTEGER NOT NULL DEFAULT 1,
|
|
54
|
+
created_at REAL NOT NULL,
|
|
55
|
+
revoked_at REAL,
|
|
56
|
+
last_seen_at REAL
|
|
57
|
+
);
|
|
58
|
+
|
|
59
|
+
-- --------------------------------------------------------------------- 4.2 Job
|
|
60
|
+
-- source_text is nullable: a one-off job's snapshot is cleared when its
|
|
61
|
+
-- retention window closes (F-42), while the job row itself survives so that
|
|
62
|
+
-- 4.2's "the existing terminal job and its reason are returned" still holds.
|
|
63
|
+
CREATE TABLE jobs (
|
|
64
|
+
job_id TEXT PRIMARY KEY,
|
|
65
|
+
kind TEXT NOT NULL,
|
|
66
|
+
request_path TEXT NOT NULL,
|
|
67
|
+
owner_client_id TEXT NOT NULL,
|
|
68
|
+
client_label TEXT,
|
|
69
|
+
state TEXT NOT NULL,
|
|
70
|
+
retention TEXT NOT NULL,
|
|
71
|
+
source_text TEXT,
|
|
72
|
+
model_id TEXT NOT NULL,
|
|
73
|
+
settings_json TEXT NOT NULL,
|
|
74
|
+
budget_json TEXT, -- the budget actually applied (F-39, F-78)
|
|
75
|
+
created_at REAL NOT NULL,
|
|
76
|
+
started_at REAL,
|
|
77
|
+
ended_at REAL,
|
|
78
|
+
error_code TEXT,
|
|
79
|
+
error_message TEXT,
|
|
80
|
+
idempotency_key TEXT,
|
|
81
|
+
generated_segments INTEGER NOT NULL DEFAULT 0,
|
|
82
|
+
total_segments INTEGER NOT NULL DEFAULT 0
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
-- There is no foreign key from jobs.owner_client_id to clients.client_id on
|
|
86
|
+
-- purpose: F-43 and F-61 both require a client's permissions to be revocable
|
|
87
|
+
-- and removable without touching the history of jobs it created, and 4.2
|
|
88
|
+
-- keeps the permission record separate from body-text history.
|
|
89
|
+
CREATE INDEX jobs_by_created ON jobs (created_at DESC);
|
|
90
|
+
CREATE INDEX jobs_by_model ON jobs (model_id, created_at DESC);
|
|
91
|
+
CREATE INDEX jobs_by_state ON jobs (state, created_at DESC);
|
|
92
|
+
CREATE INDEX jobs_by_owner ON jobs (owner_client_id, created_at DESC);
|
|
93
|
+
|
|
94
|
+
-- ----------------------------------------------------------------- 4.2 Segment
|
|
95
|
+
-- The two source columns are named for exactly what 4.2 standardises: offsets
|
|
96
|
+
-- in Unicode code points into the job's source text, start inclusive and end
|
|
97
|
+
-- exclusive. A column called "source_start"/"source_end" invites someone to
|
|
98
|
+
-- put a UTF-16 index from Qt in it, which is the defect this naming exists to
|
|
99
|
+
-- prevent.
|
|
100
|
+
CREATE TABLE segments (
|
|
101
|
+
segment_id TEXT PRIMARY KEY,
|
|
102
|
+
job_id TEXT NOT NULL REFERENCES jobs (job_id) ON DELETE CASCADE,
|
|
103
|
+
seq INTEGER NOT NULL,
|
|
104
|
+
source_start_codepoint_inclusive INTEGER NOT NULL,
|
|
105
|
+
source_end_codepoint_exclusive INTEGER NOT NULL,
|
|
106
|
+
spoken_text TEXT NOT NULL,
|
|
107
|
+
language TEXT NOT NULL,
|
|
108
|
+
audio_start_ms INTEGER,
|
|
109
|
+
audio_end_ms INTEGER,
|
|
110
|
+
trailing_silence_ms INTEGER NOT NULL DEFAULT 0,
|
|
111
|
+
audio_path TEXT,
|
|
112
|
+
frame_count INTEGER NOT NULL DEFAULT 0,
|
|
113
|
+
ready INTEGER NOT NULL DEFAULT 0,
|
|
114
|
+
UNIQUE (job_id, seq),
|
|
115
|
+
CHECK (source_start_codepoint_inclusive >= 0),
|
|
116
|
+
CHECK (source_end_codepoint_exclusive >= source_start_codepoint_inclusive),
|
|
117
|
+
CHECK (audio_end_ms IS NULL OR audio_start_ms IS NULL OR audio_end_ms >= audio_start_ms)
|
|
118
|
+
);
|
|
119
|
+
|
|
120
|
+
-- ------------------------------------------------------------------ 4.2 Result
|
|
121
|
+
-- Addressed by result_id, never by path: 4.2 requires a permission-checkable
|
|
122
|
+
-- identifier, and relative_path is resolved inside the app against the audio
|
|
123
|
+
-- directory and is never handed to a client.
|
|
124
|
+
CREATE TABLE results (
|
|
125
|
+
result_id TEXT PRIMARY KEY,
|
|
126
|
+
job_id TEXT NOT NULL UNIQUE REFERENCES jobs (job_id) ON DELETE CASCADE,
|
|
127
|
+
sample_rate INTEGER NOT NULL,
|
|
128
|
+
channels INTEGER NOT NULL,
|
|
129
|
+
sample_width_bits INTEGER NOT NULL,
|
|
130
|
+
frame_count INTEGER NOT NULL,
|
|
131
|
+
byte_size INTEGER NOT NULL,
|
|
132
|
+
digest TEXT NOT NULL, -- SHA-256 of the WAV bytes
|
|
133
|
+
relative_path TEXT NOT NULL,
|
|
134
|
+
created_at REAL NOT NULL,
|
|
135
|
+
expires_at REAL, -- NULL means "kept until deleted" (4.1)
|
|
136
|
+
integrity_state TEXT NOT NULL DEFAULT 'unverified',
|
|
137
|
+
verified_at REAL,
|
|
138
|
+
CHECK (integrity_state IN ('unverified', 'ok', 'missing', 'corrupt'))
|
|
139
|
+
);
|
|
140
|
+
|
|
141
|
+
CREATE INDEX results_by_expiry ON results (expires_at);
|
|
142
|
+
|
|
143
|
+
-- --------------------------------------------------------- 4.2 Re-request record
|
|
144
|
+
-- Client and key, the request-match discriminator, the job, and an expiry.
|
|
145
|
+
-- No source text: 4.2 says only what duplicate prevention needs is kept, and
|
|
146
|
+
-- the discriminator is a digest supplied by the caller of the store.
|
|
147
|
+
CREATE TABLE idempotency (
|
|
148
|
+
client_id TEXT NOT NULL,
|
|
149
|
+
key TEXT NOT NULL,
|
|
150
|
+
request_digest TEXT NOT NULL,
|
|
151
|
+
job_id TEXT NOT NULL REFERENCES jobs (job_id) ON DELETE CASCADE,
|
|
152
|
+
created_at REAL NOT NULL,
|
|
153
|
+
expires_at REAL NOT NULL,
|
|
154
|
+
PRIMARY KEY (client_id, key)
|
|
155
|
+
) WITHOUT ROWID;
|
|
156
|
+
|
|
157
|
+
CREATE INDEX idempotency_by_expiry ON idempotency (expires_at);
|
|
158
|
+
|
|
159
|
+
-- ------------------------------------------------------------- N-15 / F-44 Backups
|
|
160
|
+
-- Bookkeeping only. Bundling, verification, and restore live in the backup
|
|
161
|
+
-- module; what belongs to the schema is the record that a recoverable copy
|
|
162
|
+
-- exists, which schema version it was taken at, and whether it was verified,
|
|
163
|
+
-- because 4.1 rotates scheduled backups only after a new one is verified.
|
|
164
|
+
CREATE TABLE backups (
|
|
165
|
+
backup_id TEXT PRIMARY KEY,
|
|
166
|
+
kind TEXT NOT NULL, -- 'manual' | 'scheduled' | 'pre_migration'
|
|
167
|
+
created_at REAL NOT NULL,
|
|
168
|
+
location TEXT NOT NULL,
|
|
169
|
+
byte_size INTEGER NOT NULL DEFAULT 0,
|
|
170
|
+
item_count INTEGER NOT NULL DEFAULT 0,
|
|
171
|
+
schema_version INTEGER NOT NULL,
|
|
172
|
+
app_version TEXT NOT NULL,
|
|
173
|
+
verified INTEGER NOT NULL DEFAULT 0,
|
|
174
|
+
note TEXT,
|
|
175
|
+
CHECK (kind IN ('manual', 'scheduled', 'pre_migration'))
|
|
176
|
+
);
|
|
177
|
+
|
|
178
|
+
CREATE INDEX backups_by_created ON backups (kind, created_at DESC);
|
|
179
|
+
|
|
180
|
+
-- @section: fts5
|
|
181
|
+
-- F-40's full-text index over document titles and retained body text.
|
|
182
|
+
--
|
|
183
|
+
-- The tokenizer is trigram rather than the default unicode61 because
|
|
184
|
+
-- unicode61 splits on whitespace, and Korean search terms are routinely a
|
|
185
|
+
-- substring of an eojeol rather than a whole one; with unicode61 a search for
|
|
186
|
+
-- a two-syllable stem inside a longer word finds nothing. Trigram matches
|
|
187
|
+
-- substrings, at the cost of needing at least three characters -- the store
|
|
188
|
+
-- answers shorter queries with LIKE for that reason.
|
|
189
|
+
--
|
|
190
|
+
-- External content: the index stores no second copy of the body, so N-16's
|
|
191
|
+
-- retention accounting stays truthful about how much space a document costs.
|
|
192
|
+
|
|
193
|
+
CREATE VIRTUAL TABLE documents_fts USING fts5 (
|
|
194
|
+
title,
|
|
195
|
+
body,
|
|
196
|
+
content = 'documents',
|
|
197
|
+
content_rowid = 'rowid',
|
|
198
|
+
tokenize = 'trigram'
|
|
199
|
+
);
|
|
200
|
+
|
|
201
|
+
CREATE TRIGGER documents_fts_ai AFTER INSERT ON documents BEGIN
|
|
202
|
+
INSERT INTO documents_fts (rowid, title, body) VALUES (new.rowid, new.title, new.body);
|
|
203
|
+
END;
|
|
204
|
+
|
|
205
|
+
CREATE TRIGGER documents_fts_ad AFTER DELETE ON documents BEGIN
|
|
206
|
+
INSERT INTO documents_fts (documents_fts, rowid, title, body)
|
|
207
|
+
VALUES ('delete', old.rowid, old.title, old.body);
|
|
208
|
+
END;
|
|
209
|
+
|
|
210
|
+
CREATE TRIGGER documents_fts_au AFTER UPDATE ON documents BEGIN
|
|
211
|
+
INSERT INTO documents_fts (documents_fts, rowid, title, body)
|
|
212
|
+
VALUES ('delete', old.rowid, old.title, old.body);
|
|
213
|
+
INSERT INTO documents_fts (rowid, title, body) VALUES (new.rowid, new.title, new.body);
|
|
214
|
+
END;
|