superlocalmemory 3.8.5 → 3.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +3 -3
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +9 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +139 -404
- package/src/superlocalmemory/core/backend_orchestrator.py +7 -1
- package/src/superlocalmemory/core/component_registry.py +4 -2
- package/src/superlocalmemory/core/embeddings.py +33 -6
- package/src/superlocalmemory/core/engine.py +94 -49
- package/src/superlocalmemory/core/engine_ingestion.py +150 -63
- package/src/superlocalmemory/core/ingestion_command.py +133 -21
- package/src/superlocalmemory/core/mutations.py +32 -10
- package/src/superlocalmemory/core/recall_pipeline.py +111 -77
- package/src/superlocalmemory/core/remember_admission.py +152 -0
- package/src/superlocalmemory/core/remember_runtime.py +712 -0
- package/src/superlocalmemory/graph/cozo_backend.py +5 -5
- package/src/superlocalmemory/learning/bandit.py +50 -1
- package/src/superlocalmemory/learning/source_quality.py +38 -35
- package/src/superlocalmemory/mcp/_daemon_proxy.py +38 -15
- package/src/superlocalmemory/mcp/tools_active.py +4 -41
- package/src/superlocalmemory/mcp/tools_core.py +26 -87
- package/src/superlocalmemory/mcp/tools_evolution.py +5 -10
- package/src/superlocalmemory/optimize/proxy/capture.py +196 -8
- package/src/superlocalmemory/retrieval/engine.py +8 -3
- package/src/superlocalmemory/retrieval/reranker.py +35 -10
- package/src/superlocalmemory/server/loopback.py +7 -13
- package/src/superlocalmemory/server/profile_runtime.py +14 -0
- package/src/superlocalmemory/server/routes/abstraction.py +2 -4
- package/src/superlocalmemory/server/routes/agents.py +3 -5
- package/src/superlocalmemory/server/routes/behavioral.py +5 -13
- package/src/superlocalmemory/server/routes/brain.py +6 -9
- package/src/superlocalmemory/server/routes/entity.py +3 -7
- package/src/superlocalmemory/server/routes/evolution.py +3 -5
- package/src/superlocalmemory/server/routes/helpers.py +44 -23
- package/src/superlocalmemory/server/routes/insights.py +2 -4
- package/src/superlocalmemory/server/routes/learning.py +2 -5
- package/src/superlocalmemory/server/routes/lifecycle.py +2 -4
- package/src/superlocalmemory/server/routes/memories.py +122 -100
- package/src/superlocalmemory/server/routes/tiers.py +3 -22
- package/src/superlocalmemory/server/routes/timeline.py +2 -4
- package/src/superlocalmemory/server/routes/v3_api.py +18 -16
- package/src/superlocalmemory/server/unified_daemon.py +200 -109
- package/src/superlocalmemory/storage/admission_codec.py +119 -0
- package/src/superlocalmemory/storage/admission_journal.py +728 -0
- package/src/superlocalmemory/storage/database.py +59 -0
- package/src/superlocalmemory/storage/deferred_writes.py +67 -11
- package/src/superlocalmemory/storage/memory_write.py +8 -12
- package/src/superlocalmemory/storage/migration_runner.py +37 -0
- package/src/superlocalmemory/storage/migrations/M032_write_coordinator_admission.py +188 -0
- package/src/superlocalmemory/storage/read_connection.py +115 -0
- package/src/superlocalmemory/storage/write_coordinator.py +756 -0
- package/src/superlocalmemory/ui/index.html +1 -1
- package/src/superlocalmemory/ui/js/auto-settings.js +14 -1
- package/src/superlocalmemory/ui/js/od-settings.js +9 -3
|
@@ -218,9 +218,62 @@ class DatabaseManager:
|
|
|
218
218
|
conn.execute("PRAGMA foreign_keys=ON")
|
|
219
219
|
return conn
|
|
220
220
|
|
|
221
|
+
@contextmanager
|
|
222
|
+
def _bind_coordinator_connection(
|
|
223
|
+
self,
|
|
224
|
+
conn: sqlite3.Connection,
|
|
225
|
+
capability: Any,
|
|
226
|
+
) -> Generator[None, None, None]:
|
|
227
|
+
"""Reuse the coordinator's sole writable connection for one handler.
|
|
228
|
+
|
|
229
|
+
This deliberately stays internal: only ``WriteCoordinator`` can issue
|
|
230
|
+
a capability, and that capability is valid only for its worker thread
|
|
231
|
+
and the exact resolved database path. While bound, ``transaction``
|
|
232
|
+
and ``raw_connection`` become no-op ownership scopes: they may yield
|
|
233
|
+
the connection, but they must never commit, rollback, or close it.
|
|
234
|
+
The coordinator owns the enclosing ``BEGIN IMMEDIATE`` and final
|
|
235
|
+
commit/rollback together with the command receipt.
|
|
236
|
+
"""
|
|
237
|
+
from superlocalmemory.storage.write_coordinator import WriteCoordinatorError
|
|
238
|
+
|
|
239
|
+
if not isinstance(conn, sqlite3.Connection):
|
|
240
|
+
raise WriteCoordinatorError("coordinator binding requires a sqlite3 connection")
|
|
241
|
+
validate = getattr(capability, "_validate", None)
|
|
242
|
+
if not callable(validate):
|
|
243
|
+
raise WriteCoordinatorError("untrusted coordinator capability")
|
|
244
|
+
try:
|
|
245
|
+
validate(self.db_path.expanduser().resolve())
|
|
246
|
+
except Exception as exc:
|
|
247
|
+
# Import lazily so the storage manager retains its legacy import
|
|
248
|
+
# surface when the coordinator is not used.
|
|
249
|
+
if isinstance(exc, WriteCoordinatorError):
|
|
250
|
+
raise
|
|
251
|
+
raise WriteCoordinatorError("untrusted coordinator capability") from exc
|
|
252
|
+
|
|
253
|
+
attached = conn.execute("PRAGMA database_list").fetchall()
|
|
254
|
+
main_path = next((row[2] for row in attached if row[1] == "main"), "")
|
|
255
|
+
expected_path = self.db_path.expanduser().resolve()
|
|
256
|
+
if not main_path or Path(main_path).expanduser().resolve() != expected_path:
|
|
257
|
+
raise WriteCoordinatorError("coordinator connection targets a different database")
|
|
258
|
+
if getattr(self._txn_state, "conn", None) is not None:
|
|
259
|
+
raise WriteCoordinatorError("database manager is already bound to a transaction")
|
|
260
|
+
|
|
261
|
+
self._txn_state.conn = conn
|
|
262
|
+
self._txn_state.coordinator_bound = True
|
|
263
|
+
try:
|
|
264
|
+
yield
|
|
265
|
+
finally:
|
|
266
|
+
self._txn_state.conn = None
|
|
267
|
+
self._txn_state.coordinator_bound = False
|
|
268
|
+
|
|
221
269
|
@contextmanager
|
|
222
270
|
def transaction(self) -> Generator[None, None, None]:
|
|
223
271
|
"""Atomic transaction. All writes commit or rollback together."""
|
|
272
|
+
if getattr(self._txn_state, "coordinator_bound", False):
|
|
273
|
+
# The coordinator has already issued BEGIN IMMEDIATE. Do not
|
|
274
|
+
# create a nested transaction or steal its commit/close lifecycle.
|
|
275
|
+
yield
|
|
276
|
+
return
|
|
224
277
|
with self._lock:
|
|
225
278
|
conn = self._connect()
|
|
226
279
|
self._txn_state.conn = conn
|
|
@@ -243,6 +296,12 @@ class DatabaseManager:
|
|
|
243
296
|
error, and always closes — mirroring transaction(). This is the public
|
|
244
297
|
way to obtain a connection; there is no `.conn` attribute.
|
|
245
298
|
"""
|
|
299
|
+
coordinator_conn = getattr(self._txn_state, "conn", None)
|
|
300
|
+
if getattr(self._txn_state, "coordinator_bound", False):
|
|
301
|
+
if coordinator_conn is None: # pragma: no cover - binding invariant
|
|
302
|
+
raise RuntimeError("coordinator binding has no active connection")
|
|
303
|
+
yield coordinator_conn
|
|
304
|
+
return
|
|
246
305
|
with self._lock:
|
|
247
306
|
conn = self._connect()
|
|
248
307
|
self._txn_state.conn = conn
|
|
@@ -21,6 +21,7 @@ single-writer queue (see WRITE-QUEUE-PLAN.md).
|
|
|
21
21
|
"""
|
|
22
22
|
from __future__ import annotations
|
|
23
23
|
|
|
24
|
+
import atexit
|
|
24
25
|
import queue
|
|
25
26
|
import threading
|
|
26
27
|
|
|
@@ -40,11 +41,16 @@ _BG_MAXSIZE = 20000
|
|
|
40
41
|
_bg_queue: "queue.Queue" = queue.Queue(maxsize=_BG_MAXSIZE)
|
|
41
42
|
_bg_started = False
|
|
42
43
|
_bg_start_lock = threading.Lock()
|
|
44
|
+
_bg_stop = threading.Event()
|
|
45
|
+
_bg_thread: threading.Thread | None = None
|
|
43
46
|
|
|
44
47
|
|
|
45
|
-
def _bg_run() -> None:
|
|
46
|
-
while
|
|
47
|
-
|
|
48
|
+
def _bg_run(work_queue: "queue.Queue", stop: threading.Event) -> None:
|
|
49
|
+
while not stop.is_set():
|
|
50
|
+
try:
|
|
51
|
+
fn = work_queue.get(timeout=0.1)
|
|
52
|
+
except queue.Empty:
|
|
53
|
+
continue
|
|
48
54
|
try:
|
|
49
55
|
fn()
|
|
50
56
|
except Exception:
|
|
@@ -55,15 +61,21 @@ def _bg_run() -> None:
|
|
|
55
61
|
|
|
56
62
|
|
|
57
63
|
def _ensure_bg_thread() -> None:
|
|
58
|
-
global _bg_started
|
|
59
|
-
if
|
|
64
|
+
global _bg_queue, _bg_started, _bg_stop, _bg_thread
|
|
65
|
+
if _bg_thread is not None and _bg_thread.is_alive():
|
|
60
66
|
return
|
|
61
67
|
with _bg_start_lock:
|
|
62
|
-
if
|
|
68
|
+
if _bg_thread is not None and _bg_thread.is_alive():
|
|
63
69
|
return
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
70
|
+
_bg_queue = queue.Queue(maxsize=_BG_MAXSIZE)
|
|
71
|
+
_bg_stop = threading.Event()
|
|
72
|
+
_bg_thread = threading.Thread(
|
|
73
|
+
target=_bg_run,
|
|
74
|
+
args=(_bg_queue, _bg_stop),
|
|
75
|
+
name="slm-bg-writer",
|
|
76
|
+
daemon=True,
|
|
77
|
+
)
|
|
78
|
+
_bg_thread.start()
|
|
67
79
|
_bg_started = True
|
|
68
80
|
|
|
69
81
|
|
|
@@ -81,6 +93,23 @@ def submit_background(fn) -> None:
|
|
|
81
93
|
pass
|
|
82
94
|
|
|
83
95
|
|
|
96
|
+
def _shutdown_background_writer(timeout: float) -> None:
|
|
97
|
+
"""Stop the shared bookkeeping worker and wait for its thread to exit."""
|
|
98
|
+
global _bg_started, _bg_thread
|
|
99
|
+
with _bg_start_lock:
|
|
100
|
+
thread = _bg_thread
|
|
101
|
+
if thread is None:
|
|
102
|
+
return
|
|
103
|
+
_bg_stop.set()
|
|
104
|
+
if thread is not threading.current_thread():
|
|
105
|
+
thread.join(timeout=timeout)
|
|
106
|
+
if not thread.is_alive():
|
|
107
|
+
with _bg_start_lock:
|
|
108
|
+
if _bg_thread is thread:
|
|
109
|
+
_bg_thread = None
|
|
110
|
+
_bg_started = False
|
|
111
|
+
|
|
112
|
+
|
|
84
113
|
class DeferredLastSeen:
|
|
85
114
|
"""Coalescing background flusher for canonical_entities.last_seen."""
|
|
86
115
|
|
|
@@ -90,6 +119,7 @@ class DeferredLastSeen:
|
|
|
90
119
|
self._pending: dict[tuple[str, str], str] = {}
|
|
91
120
|
self._lock = threading.Lock()
|
|
92
121
|
self._stop = threading.Event()
|
|
122
|
+
self._stopped = False
|
|
93
123
|
self._thread = threading.Thread(
|
|
94
124
|
target=self._run, name="slm-deferred-lastseen", daemon=True
|
|
95
125
|
)
|
|
@@ -126,9 +156,17 @@ class DeferredLastSeen:
|
|
|
126
156
|
while not self._stop.wait(self._interval):
|
|
127
157
|
self.flush()
|
|
128
158
|
|
|
129
|
-
|
|
159
|
+
@property
|
|
160
|
+
def is_stopped(self) -> bool:
|
|
161
|
+
return self._stopped
|
|
162
|
+
|
|
163
|
+
def stop(self, timeout: float = 3.0) -> None:
|
|
164
|
+
"""Flush then join the owner thread so it cannot outlive its database."""
|
|
130
165
|
self._stop.set()
|
|
131
166
|
self.flush()
|
|
167
|
+
if self._thread is not threading.current_thread():
|
|
168
|
+
self._thread.join(timeout=timeout)
|
|
169
|
+
self._stopped = not self._thread.is_alive()
|
|
132
170
|
|
|
133
171
|
|
|
134
172
|
_registry: dict[int, DeferredLastSeen] = {}
|
|
@@ -140,14 +178,32 @@ def get_deferred_last_seen(db) -> DeferredLastSeen:
|
|
|
140
178
|
key = id(db)
|
|
141
179
|
with _registry_lock:
|
|
142
180
|
writer = _registry.get(key)
|
|
143
|
-
if writer is None:
|
|
181
|
+
if writer is None or writer.is_stopped:
|
|
144
182
|
writer = DeferredLastSeen(db)
|
|
145
183
|
_registry[key] = writer
|
|
146
184
|
return writer
|
|
147
185
|
|
|
148
186
|
|
|
187
|
+
def shutdown_deferred_writes(timeout: float = 3.0) -> None:
|
|
188
|
+
"""Stop every deferred-write worker owned by this process.
|
|
189
|
+
|
|
190
|
+
This is intentionally idempotent: service shutdown and pytest cleanup can
|
|
191
|
+
both call it, and a later caller can lazily create fresh workers.
|
|
192
|
+
"""
|
|
193
|
+
with _registry_lock:
|
|
194
|
+
writers = list(_registry.values())
|
|
195
|
+
_registry.clear()
|
|
196
|
+
for writer in writers:
|
|
197
|
+
writer.stop(timeout=timeout)
|
|
198
|
+
_shutdown_background_writer(timeout)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
atexit.register(shutdown_deferred_writes)
|
|
202
|
+
|
|
203
|
+
|
|
149
204
|
__all__ = [
|
|
150
205
|
"DeferredLastSeen",
|
|
151
206
|
"get_deferred_last_seen",
|
|
207
|
+
"shutdown_deferred_writes",
|
|
152
208
|
"submit_background",
|
|
153
209
|
]
|
|
@@ -40,8 +40,9 @@ each commit quickly.
|
|
|
40
40
|
Reads
|
|
41
41
|
-----
|
|
42
42
|
Use :func:`memory_read` for read-only access. WAL allows concurrent readers
|
|
43
|
-
without blocking the writer, so reads do NOT take the write lock
|
|
44
|
-
|
|
43
|
+
without blocking the writer, so reads do NOT take the write lock. The helper
|
|
44
|
+
opens SQLite with ``mode=ro`` and ``PRAGMA query_only=ON``; it uses the
|
|
45
|
+
3.8.6 read-path budget of at most 250ms during a brief checkpoint window.
|
|
45
46
|
"""
|
|
46
47
|
from __future__ import annotations
|
|
47
48
|
|
|
@@ -51,6 +52,7 @@ from contextlib import contextmanager
|
|
|
51
52
|
from pathlib import Path
|
|
52
53
|
from typing import Generator
|
|
53
54
|
|
|
55
|
+
from superlocalmemory.storage.read_connection import ReadConnectionFactory
|
|
54
56
|
from superlocalmemory.storage.write_lock import get_write_lock
|
|
55
57
|
|
|
56
58
|
|
|
@@ -100,20 +102,14 @@ def memory_write(db_path: str | Path) -> Generator[sqlite3.Connection, None, Non
|
|
|
100
102
|
|
|
101
103
|
@contextmanager
|
|
102
104
|
def memory_read(db_path: str | Path) -> Generator[sqlite3.Connection, None, None]:
|
|
103
|
-
"""Yield a read-only
|
|
105
|
+
"""Yield a physically read-only connection with no write lock.
|
|
104
106
|
|
|
105
107
|
WAL permits concurrent readers, so this deliberately does NOT take the
|
|
106
|
-
write lock.
|
|
107
|
-
|
|
108
|
+
write lock. A 250ms bounded wait protects the query path from a brief
|
|
109
|
+
checkpoint without converting a dashboard recall into a long hang.
|
|
108
110
|
"""
|
|
109
|
-
|
|
110
|
-
conn = sqlite3.connect(str(db_path), timeout=ms / 1000.0)
|
|
111
|
-
try:
|
|
112
|
-
conn.execute(f"PRAGMA busy_timeout={ms}")
|
|
113
|
-
conn.row_factory = sqlite3.Row
|
|
111
|
+
with ReadConnectionFactory(db_path, timeout_ms=min(_busy_timeout_ms(), 250)).snapshot() as conn:
|
|
114
112
|
yield conn
|
|
115
|
-
finally:
|
|
116
|
-
conn.close()
|
|
117
113
|
|
|
118
114
|
|
|
119
115
|
__all__ = ["memory_write", "memory_read"]
|
|
@@ -125,6 +125,9 @@ from superlocalmemory.storage.migrations import (
|
|
|
125
125
|
from superlocalmemory.storage.migrations import (
|
|
126
126
|
M031_dead_letter_operations as _M031,
|
|
127
127
|
)
|
|
128
|
+
from superlocalmemory.storage.migrations import (
|
|
129
|
+
M032_write_coordinator_admission as _M032,
|
|
130
|
+
)
|
|
128
131
|
|
|
129
132
|
# Map migration name → module (used for the optional ``verify(conn)`` hook
|
|
130
133
|
# that lets the runner detect "already applied" state when an idempotent
|
|
@@ -160,6 +163,7 @@ _MODULES = {
|
|
|
160
163
|
_M029.NAME: _M029,
|
|
161
164
|
_M030.NAME: _M030,
|
|
162
165
|
_M031.NAME: _M031,
|
|
166
|
+
_M032.NAME: _M032,
|
|
163
167
|
}
|
|
164
168
|
|
|
165
169
|
logger = logging.getLogger(__name__)
|
|
@@ -173,6 +177,12 @@ _KNOWN_EQUIVALENT_DDL_HASHES: dict[str, frozenset[str]] = {
|
|
|
173
177
|
# v3.4.22 model_version-default variant shipped through 3.6.x.
|
|
174
178
|
"d28666fa1dfa66e6514efd288e6748363513da2255a4cee95d80f233e6728ae7",
|
|
175
179
|
}),
|
|
180
|
+
_M032.NAME: frozenset({
|
|
181
|
+
# Provisional 3.8.6 development ledger: global idempotency_key and
|
|
182
|
+
# operation_id uniqueness. Its standalone table is safely rebuilt by
|
|
183
|
+
# M032.repair() into the profile-scoped receipt contract.
|
|
184
|
+
"e45df41becba3d0c3342eca5ec3bd83aa899eef76943c819d2da73b4ca1625a7",
|
|
185
|
+
}),
|
|
176
186
|
}
|
|
177
187
|
|
|
178
188
|
|
|
@@ -223,6 +233,9 @@ MIGRATIONS: list[Migration] = [
|
|
|
223
233
|
# M031 creates dead_letter_operations — standalone table, no FK to engine-
|
|
224
234
|
# bootstrapped tables, so it can run during apply_all (before engine init).
|
|
225
235
|
Migration(name=_M031.NAME, db_target="memory", ddl=_M031.DDL),
|
|
236
|
+
# M032 is standalone and must precede daemon readiness: typed writes use
|
|
237
|
+
# this append-only receipt ledger for durable idempotency.
|
|
238
|
+
Migration(name=_M032.NAME, db_target="memory", ddl=_M032.DDL),
|
|
226
239
|
# M006 + M011 are deliberately NOT here — see DEFERRED_MIGRATIONS below.
|
|
227
240
|
]
|
|
228
241
|
|
|
@@ -422,6 +435,30 @@ def _apply_single(
|
|
|
422
435
|
)
|
|
423
436
|
except sqlite3.Error: # pragma: no cover
|
|
424
437
|
pass
|
|
438
|
+
if dry_run:
|
|
439
|
+
return (
|
|
440
|
+
"skipped",
|
|
441
|
+
"dry-run: would repair allowlisted historical schema",
|
|
442
|
+
)
|
|
443
|
+
repair_fn = getattr(mod, "repair", None) if mod is not None else None
|
|
444
|
+
if callable(repair_fn):
|
|
445
|
+
try:
|
|
446
|
+
repair_fn(conn)
|
|
447
|
+
if not bool(verify_fn(conn)):
|
|
448
|
+
return (
|
|
449
|
+
"failed",
|
|
450
|
+
f"safe repair did not restore {migration.name}",
|
|
451
|
+
)
|
|
452
|
+
_upsert_log(conn, migration.name, ddl_hash, "complete")
|
|
453
|
+
return (
|
|
454
|
+
"applied",
|
|
455
|
+
"allowlisted historical schema repaired safely",
|
|
456
|
+
)
|
|
457
|
+
except sqlite3.Error as exc:
|
|
458
|
+
return (
|
|
459
|
+
"failed",
|
|
460
|
+
f"safe repair failed for {migration.name}: {exc}",
|
|
461
|
+
)
|
|
425
462
|
detail = (
|
|
426
463
|
f"DDL drift detected for {migration.name}: "
|
|
427
464
|
f"logged={logged_hash[:8]}... current={ddl_hash[:8]}..."
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
|
|
4
|
+
"""M032 — durable, append-only receipts for canonical write commands.
|
|
5
|
+
|
|
6
|
+
The receipt ledger is profile-isolated at the client idempotency boundary.
|
|
7
|
+
``command_id`` and ``journal_id`` remain globally durable identifiers; a
|
|
8
|
+
client-supplied idempotency key is unique only with its target profile. An
|
|
9
|
+
operation id is a projection label (for example, ``update:<fact_id>``), not a
|
|
10
|
+
durable command identifier, so it is deliberately indexed but not unique.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import sqlite3
|
|
16
|
+
|
|
17
|
+
NAME = "M032_write_coordinator_admission"
|
|
18
|
+
DB_TARGET = "memory"
|
|
19
|
+
|
|
20
|
+
DDL = """
|
|
21
|
+
CREATE TABLE IF NOT EXISTS write_commits (
|
|
22
|
+
commit_sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
23
|
+
command_id TEXT NOT NULL UNIQUE,
|
|
24
|
+
journal_id TEXT NOT NULL UNIQUE,
|
|
25
|
+
command_kind TEXT NOT NULL,
|
|
26
|
+
request_hash TEXT NOT NULL,
|
|
27
|
+
profile_id TEXT NOT NULL,
|
|
28
|
+
idempotency_key TEXT NOT NULL,
|
|
29
|
+
operation_id TEXT NOT NULL,
|
|
30
|
+
receipt_json TEXT NOT NULL,
|
|
31
|
+
committed_at REAL NOT NULL,
|
|
32
|
+
UNIQUE(profile_id, idempotency_key)
|
|
33
|
+
);
|
|
34
|
+
CREATE INDEX IF NOT EXISTS idx_write_commits_committed_at
|
|
35
|
+
ON write_commits (committed_at);
|
|
36
|
+
CREATE INDEX IF NOT EXISTS idx_write_commits_operation_id
|
|
37
|
+
ON write_commits (operation_id);
|
|
38
|
+
CREATE TRIGGER IF NOT EXISTS trg_write_commits_immutable_update
|
|
39
|
+
BEFORE UPDATE ON write_commits
|
|
40
|
+
BEGIN
|
|
41
|
+
SELECT RAISE(ABORT, 'write_commits receipts are immutable');
|
|
42
|
+
END;
|
|
43
|
+
CREATE TRIGGER IF NOT EXISTS trg_write_commits_immutable_delete
|
|
44
|
+
BEFORE DELETE ON write_commits
|
|
45
|
+
BEGIN
|
|
46
|
+
SELECT RAISE(ABORT, 'write_commits receipts are immutable');
|
|
47
|
+
END;
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
_CREATE_TABLE = DDL.split(";", 1)[0]
|
|
51
|
+
_CREATE_COMMITTED_AT_INDEX = (
|
|
52
|
+
"CREATE INDEX IF NOT EXISTS idx_write_commits_committed_at "
|
|
53
|
+
"ON write_commits (committed_at)"
|
|
54
|
+
)
|
|
55
|
+
_CREATE_OPERATION_ID_INDEX = (
|
|
56
|
+
"CREATE INDEX IF NOT EXISTS idx_write_commits_operation_id "
|
|
57
|
+
"ON write_commits (operation_id)"
|
|
58
|
+
)
|
|
59
|
+
_CREATE_UPDATE_TRIGGER = """
|
|
60
|
+
CREATE TRIGGER IF NOT EXISTS trg_write_commits_immutable_update
|
|
61
|
+
BEFORE UPDATE ON write_commits
|
|
62
|
+
BEGIN
|
|
63
|
+
SELECT RAISE(ABORT, 'write_commits receipts are immutable');
|
|
64
|
+
END
|
|
65
|
+
"""
|
|
66
|
+
_CREATE_DELETE_TRIGGER = """
|
|
67
|
+
CREATE TRIGGER IF NOT EXISTS trg_write_commits_immutable_delete
|
|
68
|
+
BEFORE DELETE ON write_commits
|
|
69
|
+
BEGIN
|
|
70
|
+
SELECT RAISE(ABORT, 'write_commits receipts are immutable');
|
|
71
|
+
END
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def apply(conn: sqlite3.Connection) -> None:
|
|
76
|
+
"""Create or safely upgrade the profile-scoped append-only receipt ledger."""
|
|
77
|
+
if verify(conn):
|
|
78
|
+
return
|
|
79
|
+
if not _table_exists(conn):
|
|
80
|
+
_create_current_schema(conn)
|
|
81
|
+
return
|
|
82
|
+
_rebuild_legacy_schema(conn)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def repair(conn: sqlite3.Connection) -> None:
|
|
86
|
+
"""Repair a completed provisional M032 in developer/test databases."""
|
|
87
|
+
apply(conn)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def verify(conn: sqlite3.Connection) -> bool:
|
|
91
|
+
"""Return true only when the full profile-safe ledger contract exists."""
|
|
92
|
+
if not _table_exists(conn):
|
|
93
|
+
return False
|
|
94
|
+
columns = {row[1] for row in conn.execute("PRAGMA table_info(write_commits)").fetchall()}
|
|
95
|
+
required = {
|
|
96
|
+
"commit_sequence",
|
|
97
|
+
"command_id",
|
|
98
|
+
"journal_id",
|
|
99
|
+
"command_kind",
|
|
100
|
+
"request_hash",
|
|
101
|
+
"profile_id",
|
|
102
|
+
"idempotency_key",
|
|
103
|
+
"operation_id",
|
|
104
|
+
"receipt_json",
|
|
105
|
+
"committed_at",
|
|
106
|
+
}
|
|
107
|
+
if not required <= columns:
|
|
108
|
+
return False
|
|
109
|
+
if not _has_unique_index(conn, ("command_id",)):
|
|
110
|
+
return False
|
|
111
|
+
if not _has_unique_index(conn, ("journal_id",)):
|
|
112
|
+
return False
|
|
113
|
+
if not _has_unique_index(conn, ("profile_id", "idempotency_key")):
|
|
114
|
+
return False
|
|
115
|
+
if _has_unique_index(conn, ("idempotency_key",)):
|
|
116
|
+
return False
|
|
117
|
+
if _has_unique_index(conn, ("operation_id",)):
|
|
118
|
+
return False
|
|
119
|
+
object_names = {
|
|
120
|
+
row[0]
|
|
121
|
+
for row in conn.execute(
|
|
122
|
+
"SELECT name FROM sqlite_master WHERE name IN (?, ?, ?, ?)",
|
|
123
|
+
(
|
|
124
|
+
"idx_write_commits_committed_at",
|
|
125
|
+
"idx_write_commits_operation_id",
|
|
126
|
+
"trg_write_commits_immutable_update",
|
|
127
|
+
"trg_write_commits_immutable_delete",
|
|
128
|
+
),
|
|
129
|
+
).fetchall()
|
|
130
|
+
}
|
|
131
|
+
return object_names == {
|
|
132
|
+
"idx_write_commits_committed_at",
|
|
133
|
+
"idx_write_commits_operation_id",
|
|
134
|
+
"trg_write_commits_immutable_update",
|
|
135
|
+
"trg_write_commits_immutable_delete",
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _table_exists(conn: sqlite3.Connection) -> bool:
|
|
140
|
+
return conn.execute(
|
|
141
|
+
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='write_commits'"
|
|
142
|
+
).fetchone() is not None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _has_unique_index(conn: sqlite3.Connection, columns: tuple[str, ...]) -> bool:
|
|
146
|
+
for index in conn.execute("PRAGMA index_list(write_commits)").fetchall():
|
|
147
|
+
if not index[2]:
|
|
148
|
+
continue
|
|
149
|
+
index_columns = tuple(
|
|
150
|
+
row[2] for row in conn.execute(f"PRAGMA index_info({index[1]})").fetchall()
|
|
151
|
+
)
|
|
152
|
+
if index_columns == columns:
|
|
153
|
+
return True
|
|
154
|
+
return False
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _create_current_schema(conn: sqlite3.Connection) -> None:
|
|
158
|
+
conn.execute(_CREATE_TABLE)
|
|
159
|
+
conn.execute(_CREATE_COMMITTED_AT_INDEX)
|
|
160
|
+
conn.execute(_CREATE_OPERATION_ID_INDEX)
|
|
161
|
+
conn.execute(_CREATE_UPDATE_TRIGGER)
|
|
162
|
+
conn.execute(_CREATE_DELETE_TRIGGER)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _rebuild_legacy_schema(conn: sqlite3.Connection) -> None:
|
|
166
|
+
"""Rebuild only M032's own standalone table under one savepoint."""
|
|
167
|
+
conn.execute("SAVEPOINT m032_profile_scoped_idempotency")
|
|
168
|
+
try:
|
|
169
|
+
conn.execute("DROP TRIGGER IF EXISTS trg_write_commits_immutable_update")
|
|
170
|
+
conn.execute("DROP TRIGGER IF EXISTS trg_write_commits_immutable_delete")
|
|
171
|
+
conn.execute("ALTER TABLE write_commits RENAME TO write_commits_legacy")
|
|
172
|
+
conn.execute("DROP INDEX IF EXISTS idx_write_commits_committed_at")
|
|
173
|
+
conn.execute("DROP INDEX IF EXISTS idx_write_commits_operation_id")
|
|
174
|
+
_create_current_schema(conn)
|
|
175
|
+
conn.execute(
|
|
176
|
+
"INSERT INTO write_commits("
|
|
177
|
+
"commit_sequence, command_id, journal_id, command_kind, request_hash, "
|
|
178
|
+
"profile_id, idempotency_key, operation_id, receipt_json, committed_at"
|
|
179
|
+
") SELECT commit_sequence, command_id, journal_id, command_kind, request_hash, "
|
|
180
|
+
"profile_id, idempotency_key, operation_id, receipt_json, committed_at "
|
|
181
|
+
"FROM write_commits_legacy"
|
|
182
|
+
)
|
|
183
|
+
conn.execute("DROP TABLE write_commits_legacy")
|
|
184
|
+
except BaseException:
|
|
185
|
+
conn.execute("ROLLBACK TO m032_profile_scoped_idempotency")
|
|
186
|
+
conn.execute("RELEASE m032_profile_scoped_idempotency")
|
|
187
|
+
raise
|
|
188
|
+
conn.execute("RELEASE m032_profile_scoped_idempotency")
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com
|
|
4
|
+
|
|
5
|
+
"""Physically read-only SQLite snapshots for canonical ``memory.db``.
|
|
6
|
+
|
|
7
|
+
The factory is deliberately small: a query path must receive a connection
|
|
8
|
+
which SQLite itself refuses to mutate. This is the storage half of the
|
|
9
|
+
Command-Query Separation contract; higher layers must not use it to perform
|
|
10
|
+
telemetry, access tracking, or any other side effect.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import sqlite3
|
|
15
|
+
from contextlib import AbstractContextManager, contextmanager
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Iterator
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ReadConnectionError(RuntimeError):
|
|
21
|
+
"""Raised when a read snapshot cannot be opened safely."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ReadConnectionFactory:
|
|
25
|
+
"""Open short-lived query-only snapshots of one canonical database path."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, memory_db: str | Path, timeout_ms: int = 250) -> None:
|
|
28
|
+
if timeout_ms < 0:
|
|
29
|
+
raise ValueError("timeout_ms must be greater than or equal to zero")
|
|
30
|
+
self._memory_db = Path(memory_db).expanduser().resolve()
|
|
31
|
+
self._timeout_ms = timeout_ms
|
|
32
|
+
|
|
33
|
+
@property
|
|
34
|
+
def memory_db(self) -> Path:
|
|
35
|
+
"""Return the resolved canonical database path."""
|
|
36
|
+
return self._memory_db
|
|
37
|
+
|
|
38
|
+
@contextmanager
|
|
39
|
+
def snapshot(self) -> Iterator[sqlite3.Connection]:
|
|
40
|
+
"""Yield a SQLite connection that cannot issue a write statement.
|
|
41
|
+
|
|
42
|
+
``immutable=1`` is intentionally not used: a live WAL database may
|
|
43
|
+
have valid state in its WAL file. ``mode=ro`` retains normal WAL
|
|
44
|
+
visibility while prohibiting a writable open, and ``query_only`` is a
|
|
45
|
+
second SQLite-enforced guard against accidental DML or DDL.
|
|
46
|
+
"""
|
|
47
|
+
if not self._memory_db.exists():
|
|
48
|
+
raise ReadConnectionError(f"canonical database does not exist: {self._memory_db}")
|
|
49
|
+
|
|
50
|
+
uri = f"{self._memory_db.as_uri()}?mode=ro"
|
|
51
|
+
try:
|
|
52
|
+
conn = sqlite3.connect(
|
|
53
|
+
uri,
|
|
54
|
+
uri=True,
|
|
55
|
+
timeout=self._timeout_ms / 1000.0,
|
|
56
|
+
)
|
|
57
|
+
except sqlite3.Error as exc:
|
|
58
|
+
raise ReadConnectionError(
|
|
59
|
+
f"could not open read-only snapshot for {self._memory_db}"
|
|
60
|
+
) from exc
|
|
61
|
+
|
|
62
|
+
try:
|
|
63
|
+
conn.row_factory = sqlite3.Row
|
|
64
|
+
conn.execute("PRAGMA query_only=ON")
|
|
65
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
66
|
+
conn.execute(f"PRAGMA busy_timeout={self._timeout_ms}")
|
|
67
|
+
yield conn
|
|
68
|
+
finally:
|
|
69
|
+
conn.close()
|
|
70
|
+
|
|
71
|
+
def open(self) -> "ReadConnectionLease":
|
|
72
|
+
"""Return a manually-closeable read-only lease for legacy query code.
|
|
73
|
+
|
|
74
|
+
New code should use :meth:`snapshot`; the lease is a small compatibility
|
|
75
|
+
bridge for client paths that still use ``try/finally: conn.close()``.
|
|
76
|
+
Closing it exits this same context manager, preserving both ``mode=ro``
|
|
77
|
+
and ``PRAGMA query_only`` rather than reopening SQLite directly.
|
|
78
|
+
"""
|
|
79
|
+
return ReadConnectionLease(self.snapshot())
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class ReadConnectionLease:
|
|
83
|
+
"""A legacy-compatible facade around one read-only snapshot."""
|
|
84
|
+
|
|
85
|
+
def __init__(self, snapshot: AbstractContextManager[sqlite3.Connection]) -> None:
|
|
86
|
+
object.__setattr__(self, "_snapshot", snapshot)
|
|
87
|
+
object.__setattr__(self, "_connection", snapshot.__enter__())
|
|
88
|
+
object.__setattr__(self, "_closed", False)
|
|
89
|
+
|
|
90
|
+
def __getattr__(self, name: str):
|
|
91
|
+
return getattr(self._connection, name)
|
|
92
|
+
|
|
93
|
+
def __setattr__(self, name: str, value: object) -> None:
|
|
94
|
+
if name.startswith("_"):
|
|
95
|
+
object.__setattr__(self, name, value)
|
|
96
|
+
else:
|
|
97
|
+
setattr(self._connection, name, value)
|
|
98
|
+
|
|
99
|
+
def __enter__(self) -> sqlite3.Connection:
|
|
100
|
+
return self._connection
|
|
101
|
+
|
|
102
|
+
def __exit__(self, exc_type, exc, traceback) -> None:
|
|
103
|
+
self._close(exc_type, exc, traceback)
|
|
104
|
+
|
|
105
|
+
def close(self) -> None:
|
|
106
|
+
"""Close this lease once, including its context-manager cleanup."""
|
|
107
|
+
self._close(None, None, None)
|
|
108
|
+
|
|
109
|
+
def _close(self, exc_type, exc, traceback) -> None:
|
|
110
|
+
if not self._closed:
|
|
111
|
+
object.__setattr__(self, "_closed", True)
|
|
112
|
+
self._snapshot.__exit__(exc_type, exc, traceback)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
__all__ = ["ReadConnectionError", "ReadConnectionFactory", "ReadConnectionLease"]
|