superlocalmemory 3.8.9 → 3.8.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +3 -3
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/daemon.py +89 -16
- package/src/superlocalmemory/core/embeddings.py +35 -4
- package/src/superlocalmemory/core/ollama_embedder.py +11 -2
- package/src/superlocalmemory/core/remember_admission.py +14 -5
- package/src/superlocalmemory/hooks/adapter_base.py +10 -3
- package/src/superlocalmemory/mcp/tools_core.py +16 -5
- package/src/superlocalmemory/optimize/proxy/capture.py +148 -30
- package/src/superlocalmemory/optimize/storage/db.py +6 -2
- package/src/superlocalmemory/server/unified_daemon.py +8 -1
- package/src/superlocalmemory/storage/admission_codec.py +10 -0
- package/src/superlocalmemory/storage/admission_journal.py +182 -67
- package/src/superlocalmemory/storage/embedding_migrator.py +27 -13
- package/src/superlocalmemory/storage/write_coordinator.py +68 -21
|
@@ -105,6 +105,7 @@ def register_core_tools(server, get_engine: Callable) -> None:
|
|
|
105
105
|
# recall window so a parallel/next agent finds memories saved seconds ago.
|
|
106
106
|
# Falls back to the capability-owned worker only if the daemon is
|
|
107
107
|
# unreachable. Raw pending.db writes are legacy replay input only.
|
|
108
|
+
daemon_owned = False
|
|
108
109
|
try:
|
|
109
110
|
import asyncio as _asyncio
|
|
110
111
|
from superlocalmemory.cli.daemon import daemon_request, is_daemon_running
|
|
@@ -159,6 +160,15 @@ def register_core_tools(server, get_engine: Callable) -> None:
|
|
|
159
160
|
}
|
|
160
161
|
except Exception as dexc:
|
|
161
162
|
logger.debug("MCP remember via daemon failed, pending fallback: %s", dexc)
|
|
163
|
+
if daemon_owned:
|
|
164
|
+
return {
|
|
165
|
+
"success": False,
|
|
166
|
+
"code": "DAEMON_UNAVAILABLE",
|
|
167
|
+
"retryable": True,
|
|
168
|
+
"error": (
|
|
169
|
+
"DAEMON_UNAVAILABLE: owned daemon is unavailable; retry later."
|
|
170
|
+
),
|
|
171
|
+
}
|
|
162
172
|
|
|
163
173
|
try:
|
|
164
174
|
import asyncio as _asyncio
|
|
@@ -174,11 +184,12 @@ def register_core_tools(server, get_engine: Callable) -> None:
|
|
|
174
184
|
or "mcp:" + hashlib.sha256(content.encode("utf-8")).hexdigest()
|
|
175
185
|
),
|
|
176
186
|
}
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
worker_meta
|
|
181
|
-
|
|
187
|
+
|
|
188
|
+
def _store_via_daemon_pool():
|
|
189
|
+
pool = choose_pool()
|
|
190
|
+
return pool.store(content, worker_meta)
|
|
191
|
+
|
|
192
|
+
stored = await _asyncio.to_thread(_store_via_daemon_pool)
|
|
182
193
|
if not isinstance(stored, dict) or not stored.get("ok"):
|
|
183
194
|
if isinstance(stored, dict) and stored.get("code") == "DAEMON_UNAVAILABLE":
|
|
184
195
|
return {
|
|
@@ -74,7 +74,7 @@ def _windows_owner_dacl(
|
|
|
74
74
|
win32api: Any,
|
|
75
75
|
win32con: Any,
|
|
76
76
|
win32security: Any,
|
|
77
|
-
) -> Any:
|
|
77
|
+
) -> tuple[Any, Any]:
|
|
78
78
|
"""Build one protected owner-only DACL for a Windows capture file."""
|
|
79
79
|
import ntsecuritycon
|
|
80
80
|
|
|
@@ -95,7 +95,53 @@ def _windows_owner_dacl(
|
|
|
95
95
|
ntsecuritycon.FILE_ALL_ACCESS,
|
|
96
96
|
owner_sid,
|
|
97
97
|
)
|
|
98
|
-
return dacl
|
|
98
|
+
return owner_sid, dacl
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _windows_dacl_is_owner_only(
|
|
102
|
+
security_descriptor: Any,
|
|
103
|
+
owner_sid: Any,
|
|
104
|
+
ntsecuritycon: Any,
|
|
105
|
+
win32security: Any,
|
|
106
|
+
) -> bool:
|
|
107
|
+
"""Return whether a live descriptor already matches the capture policy."""
|
|
108
|
+
def _same_sid(left: Any, right: Any) -> bool:
|
|
109
|
+
try:
|
|
110
|
+
return (
|
|
111
|
+
win32security.ConvertSidToStringSid(left)
|
|
112
|
+
== win32security.ConvertSidToStringSid(right)
|
|
113
|
+
)
|
|
114
|
+
except Exception:
|
|
115
|
+
return False
|
|
116
|
+
|
|
117
|
+
control, _revision = security_descriptor.GetSecurityDescriptorControl()
|
|
118
|
+
if not control & win32security.SE_DACL_PROTECTED:
|
|
119
|
+
return False
|
|
120
|
+
|
|
121
|
+
descriptor_owner = security_descriptor.GetSecurityDescriptorOwner()
|
|
122
|
+
if descriptor_owner is None or not _same_sid(
|
|
123
|
+
descriptor_owner,
|
|
124
|
+
owner_sid,
|
|
125
|
+
):
|
|
126
|
+
return False
|
|
127
|
+
|
|
128
|
+
dacl = security_descriptor.GetSecurityDescriptorDacl()
|
|
129
|
+
if dacl is None or dacl.GetAceCount() != 1:
|
|
130
|
+
return False
|
|
131
|
+
|
|
132
|
+
ace = dacl.GetAce(0)
|
|
133
|
+
if not isinstance(ace, tuple) or len(ace) != 3:
|
|
134
|
+
return False
|
|
135
|
+
ace_header, access_mask, ace_sid = ace
|
|
136
|
+
if (
|
|
137
|
+
not isinstance(ace_header, tuple)
|
|
138
|
+
or not ace_header
|
|
139
|
+
or ace_header[0] != win32security.ACCESS_ALLOWED_ACE_TYPE
|
|
140
|
+
):
|
|
141
|
+
return False
|
|
142
|
+
if access_mask & ntsecuritycon.FILE_ALL_ACCESS != ntsecuritycon.FILE_ALL_ACCESS:
|
|
143
|
+
return False
|
|
144
|
+
return _same_sid(ace_sid, owner_sid)
|
|
99
145
|
|
|
100
146
|
|
|
101
147
|
def _open_windows_capture_append(path: Path) -> int:
|
|
@@ -117,7 +163,11 @@ def _open_windows_capture_append(path: Path) -> int:
|
|
|
117
163
|
# ever inheriting a broader parent DACL. For an existing file Windows
|
|
118
164
|
# ignores this descriptor; _enforce_owner_only_permissions replaces
|
|
119
165
|
# that DACL through the same WRITE_DAC-capable handle before writing.
|
|
120
|
-
dacl = _windows_owner_dacl(
|
|
166
|
+
owner_sid, dacl = _windows_owner_dacl(
|
|
167
|
+
win32api,
|
|
168
|
+
win32con,
|
|
169
|
+
win32security,
|
|
170
|
+
)
|
|
121
171
|
security_attributes = win32security.SECURITY_ATTRIBUTES()
|
|
122
172
|
security_attributes.bInheritHandle = False
|
|
123
173
|
security_attributes.SECURITY_DESCRIPTOR.SetSecurityDescriptorDacl(
|
|
@@ -129,21 +179,57 @@ def _open_windows_capture_append(path: Path) -> int:
|
|
|
129
179
|
win32security.SE_DACL_PROTECTED,
|
|
130
180
|
win32security.SE_DACL_PROTECTED,
|
|
131
181
|
)
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
182
|
+
desired_access = (
|
|
183
|
+
ntsecuritycon.FILE_APPEND_DATA
|
|
184
|
+
| ntsecuritycon.WRITE_DAC
|
|
185
|
+
| ntsecuritycon.READ_CONTROL
|
|
186
|
+
)
|
|
187
|
+
share_mode = (
|
|
135
188
|
win32con.FILE_SHARE_READ
|
|
136
189
|
| win32con.FILE_SHARE_WRITE
|
|
137
|
-
| win32con.FILE_SHARE_DELETE
|
|
138
|
-
|
|
139
|
-
|
|
190
|
+
| win32con.FILE_SHARE_DELETE
|
|
191
|
+
)
|
|
192
|
+
file_flags = (
|
|
140
193
|
win32con.FILE_ATTRIBUTE_NORMAL
|
|
141
194
|
# pywin32 does not export this SDK constant from win32con on
|
|
142
195
|
# every supported Python build. Keep the Microsoft-defined value
|
|
143
196
|
# as a named fallback rather than silently following a reparse.
|
|
144
|
-
| getattr(win32con, "FILE_FLAG_OPEN_REPARSE_POINT", 0x00200000)
|
|
145
|
-
None,
|
|
197
|
+
| getattr(win32con, "FILE_FLAG_OPEN_REPARSE_POINT", 0x00200000)
|
|
146
198
|
)
|
|
199
|
+
try:
|
|
200
|
+
# CREATE_NEW is the only race-safe proof that the protected
|
|
201
|
+
# SECURITY_ATTRIBUTES were applied to this exact file. OPEN_ALWAYS
|
|
202
|
+
# would require trusting GetLastError after the pywin32 wrapper has
|
|
203
|
+
# returned, which is not a documented preservation boundary.
|
|
204
|
+
handle = win32file.CreateFile(
|
|
205
|
+
os.fspath(path),
|
|
206
|
+
desired_access,
|
|
207
|
+
share_mode,
|
|
208
|
+
security_attributes,
|
|
209
|
+
getattr(win32con, "CREATE_NEW", 1),
|
|
210
|
+
file_flags,
|
|
211
|
+
None,
|
|
212
|
+
)
|
|
213
|
+
created_new = True
|
|
214
|
+
except Exception as exc:
|
|
215
|
+
winerror = getattr(exc, "winerror", None)
|
|
216
|
+
if winerror is None and exc.args:
|
|
217
|
+
winerror = exc.args[0]
|
|
218
|
+
if winerror != getattr(win32con, "ERROR_FILE_EXISTS", 80):
|
|
219
|
+
raise
|
|
220
|
+
# The creation descriptor is ignored for existing files. Reopen
|
|
221
|
+
# the exact path without following a reparse point, then replace
|
|
222
|
+
# its DACL through this WRITE_DAC-capable handle before appending.
|
|
223
|
+
handle = win32file.CreateFile(
|
|
224
|
+
os.fspath(path),
|
|
225
|
+
desired_access,
|
|
226
|
+
share_mode,
|
|
227
|
+
None,
|
|
228
|
+
getattr(win32con, "OPEN_EXISTING", 3),
|
|
229
|
+
file_flags,
|
|
230
|
+
None,
|
|
231
|
+
)
|
|
232
|
+
created_new = False
|
|
147
233
|
file_info = win32file.GetFileInformationByHandle(handle)
|
|
148
234
|
if file_info[0] & win32con.FILE_ATTRIBUTE_REPARSE_POINT:
|
|
149
235
|
raise OSError(
|
|
@@ -151,25 +237,57 @@ def _open_windows_capture_append(path: Path) -> int:
|
|
|
151
237
|
"capture path is a Windows reparse point",
|
|
152
238
|
path,
|
|
153
239
|
)
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
win32security.
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
240
|
+
if not created_new:
|
|
241
|
+
try:
|
|
242
|
+
query_flags = (
|
|
243
|
+
win32security.OWNER_SECURITY_INFORMATION
|
|
244
|
+
| win32security.DACL_SECURITY_INFORMATION
|
|
245
|
+
)
|
|
246
|
+
descriptor = win32security.GetSecurityInfo(
|
|
247
|
+
handle,
|
|
248
|
+
win32security.SE_FILE_OBJECT,
|
|
249
|
+
query_flags,
|
|
250
|
+
)
|
|
251
|
+
if not _windows_dacl_is_owner_only(
|
|
252
|
+
descriptor,
|
|
253
|
+
owner_sid,
|
|
254
|
+
ntsecuritycon,
|
|
255
|
+
win32security,
|
|
256
|
+
):
|
|
257
|
+
# Existing files ignore the creation security descriptor,
|
|
258
|
+
# so repair an unsafe DACL while this is still the original
|
|
259
|
+
# CreateFile handle carrying WRITE_DAC. Avoid rewriting an
|
|
260
|
+
# already-protected DACL: Windows may correctly deny that
|
|
261
|
+
# redundant mutation even though append access is allowed.
|
|
262
|
+
win32security.SetSecurityInfo(
|
|
263
|
+
handle,
|
|
264
|
+
win32security.SE_FILE_OBJECT,
|
|
265
|
+
win32security.DACL_SECURITY_INFORMATION
|
|
266
|
+
| win32security.PROTECTED_DACL_SECURITY_INFORMATION,
|
|
267
|
+
None,
|
|
268
|
+
None,
|
|
269
|
+
dacl,
|
|
270
|
+
None,
|
|
271
|
+
)
|
|
272
|
+
descriptor = win32security.GetSecurityInfo(
|
|
273
|
+
handle,
|
|
274
|
+
win32security.SE_FILE_OBJECT,
|
|
275
|
+
query_flags,
|
|
276
|
+
)
|
|
277
|
+
if not _windows_dacl_is_owner_only(
|
|
278
|
+
descriptor,
|
|
279
|
+
owner_sid,
|
|
280
|
+
ntsecuritycon,
|
|
281
|
+
win32security,
|
|
282
|
+
):
|
|
283
|
+
raise OSError(
|
|
284
|
+
"Windows capture ACL verification failed after repair"
|
|
285
|
+
)
|
|
286
|
+
except Exception as exc:
|
|
287
|
+
raise OSError(
|
|
288
|
+
"Windows capture ACL could not be enforced "
|
|
289
|
+
f"({type(exc).__name__}: {exc})"
|
|
290
|
+
) from exc
|
|
173
291
|
|
|
174
292
|
# Transfer the native handle to Python's CRT descriptor exactly once.
|
|
175
293
|
raw_handle = handle.Detach()
|
|
@@ -212,8 +212,12 @@ class CacheDB:
|
|
|
212
212
|
try:
|
|
213
213
|
import sqlite3 as _sq
|
|
214
214
|
test_conn = _sq.connect(str(self._db_path))
|
|
215
|
-
|
|
216
|
-
|
|
215
|
+
try:
|
|
216
|
+
test_conn.execute("PRAGMA schema_version")
|
|
217
|
+
finally:
|
|
218
|
+
# Windows will not rename an open SQLite file. Always
|
|
219
|
+
# release the probe before corrupt-file recovery runs.
|
|
220
|
+
test_conn.close()
|
|
217
221
|
except Exception as exc:
|
|
218
222
|
corrupt_sidecar = self._db_path.with_suffix(
|
|
219
223
|
self._db_path.suffix + ".corrupt"
|
|
@@ -1244,6 +1244,7 @@ async def lifespan(application: FastAPI):
|
|
|
1244
1244
|
engine = None
|
|
1245
1245
|
config = None
|
|
1246
1246
|
canonical_remember_runtime = None
|
|
1247
|
+
profile_runtime = None
|
|
1247
1248
|
|
|
1248
1249
|
# The local dashboard obtains its short-lived browser credential from
|
|
1249
1250
|
# ``/internal/token`` before its first write or token-gated read. A
|
|
@@ -1916,11 +1917,17 @@ async def lifespan(application: FastAPI):
|
|
|
1916
1917
|
|
|
1917
1918
|
except Exception:
|
|
1918
1919
|
logger.exception("Engine init failed") # auto-includes traceback
|
|
1919
|
-
_release_canonical_remember_runtime(
|
|
1920
|
+
writer_released = _release_canonical_remember_runtime(
|
|
1920
1921
|
application, canonical_remember_runtime,
|
|
1921
1922
|
)
|
|
1922
1923
|
application.state.engine = None
|
|
1923
1924
|
application.state.config = None
|
|
1925
|
+
if engine is not None and writer_released:
|
|
1926
|
+
try:
|
|
1927
|
+
engine.close()
|
|
1928
|
+
except Exception:
|
|
1929
|
+
logger.debug("partially initialized engine cleanup failed", exc_info=True)
|
|
1930
|
+
raise
|
|
1924
1931
|
|
|
1925
1932
|
application.state.observe_buffer = _observe_buffer
|
|
1926
1933
|
|
|
@@ -74,6 +74,16 @@ def _load_or_create_key(path: Path) -> bytes:
|
|
|
74
74
|
)
|
|
75
75
|
except FileExistsError:
|
|
76
76
|
fd = -1
|
|
77
|
+
except OSError as exc:
|
|
78
|
+
try:
|
|
79
|
+
info = path.lstat()
|
|
80
|
+
except OSError:
|
|
81
|
+
raise AdmissionKeyError("admission key cannot be created") from exc
|
|
82
|
+
if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode):
|
|
83
|
+
raise AdmissionKeyError(
|
|
84
|
+
"admission key path must be a regular file"
|
|
85
|
+
) from exc
|
|
86
|
+
raise AdmissionKeyError("admission key cannot be created") from exc
|
|
77
87
|
else:
|
|
78
88
|
try:
|
|
79
89
|
key = os.urandom(_KEY_BYTES)
|
|
@@ -2,10 +2,12 @@
|
|
|
2
2
|
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
3
|
# Part of SuperLocalMemory V3 | https://qualixar.com
|
|
4
4
|
|
|
5
|
-
"""Separate
|
|
5
|
+
"""Separate journal for durable remember admission.
|
|
6
6
|
|
|
7
7
|
The journal is intentionally not a memory store. Its only mutable state is a
|
|
8
8
|
small encrypted replay command and the canonical receipt associated with it.
|
|
9
|
+
Durable prepare and terminal transitions are FULL-synchronous; the advisory
|
|
10
|
+
dispatched marker may use NORMAL because both replay states are equivalent.
|
|
9
11
|
Canonical facts, FTS, graph, vectors, and model work stay outside this module.
|
|
10
12
|
"""
|
|
11
13
|
|
|
@@ -31,6 +33,7 @@ from cryptography.exceptions import InvalidTag
|
|
|
31
33
|
_MAX_COMMAND_BYTES = 256 * 1024
|
|
32
34
|
_MAX_RECEIPT_BYTES = 16 * 1024
|
|
33
35
|
_MAX_METADATA_DEPTH = 8
|
|
36
|
+
_MAX_PREOPENED_WRITE_CONNECTIONS = 8
|
|
34
37
|
_IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{1,256}$")
|
|
35
38
|
_STATES = frozenset({"prepared", "dispatched", "committed", "rejected"})
|
|
36
39
|
|
|
@@ -213,10 +216,13 @@ class AdmissionJournal:
|
|
|
213
216
|
self._codec = codec
|
|
214
217
|
# The daemon is the sole journal owner, but many HTTP/MCP request
|
|
215
218
|
# threads can prepare and transition entries concurrently. SQLite has
|
|
216
|
-
# one writer, so admit those tiny
|
|
217
|
-
#
|
|
218
|
-
#
|
|
219
|
+
# one writer, so admit those tiny journal transactions through one
|
|
220
|
+
# process-local lock instead of letting BEGIN IMMEDIATE race and leak
|
|
221
|
+
# SQLITE_BUSY to remember callers.
|
|
219
222
|
self._write_lock = threading.RLock()
|
|
223
|
+
self._write_connection_slots = threading.BoundedSemaphore(
|
|
224
|
+
_MAX_PREOPENED_WRITE_CONNECTIONS
|
|
225
|
+
)
|
|
220
226
|
self._initialize()
|
|
221
227
|
|
|
222
228
|
def prepare(
|
|
@@ -248,36 +254,59 @@ class AdmissionJournal:
|
|
|
248
254
|
now = _now_ms()
|
|
249
255
|
journal_id = uuid.uuid4().hex
|
|
250
256
|
|
|
251
|
-
|
|
257
|
+
# Keep existing retries read-only and outside mutation admission. A
|
|
258
|
+
# miss closes this short-lived reader before entering the bounded
|
|
259
|
+
# write lane, where the key is rechecked transactionally.
|
|
260
|
+
with self._read_connection(deadline=deadline) as conn:
|
|
252
261
|
existing = conn.execute(
|
|
253
262
|
"SELECT * FROM admission_journal "
|
|
254
263
|
"WHERE profile_id=? AND idempotency_key=?",
|
|
255
264
|
(request.profile_id, request.idempotency_key),
|
|
256
265
|
).fetchone()
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
(
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
266
|
+
if existing is not None:
|
|
267
|
+
entry = self._entry_from_row(existing)
|
|
268
|
+
if entry.request_hash != request_hash:
|
|
269
|
+
raise IdempotencyConflict(
|
|
270
|
+
"idempotency key belongs to a different immutable request"
|
|
271
|
+
)
|
|
272
|
+
return entry
|
|
273
|
+
|
|
274
|
+
with self._write_connection_slot(deadline=deadline):
|
|
275
|
+
remaining = _remaining_seconds(deadline)
|
|
276
|
+
with self._connection(timeout=remaining) as conn:
|
|
277
|
+
with self._write_slot(deadline=deadline):
|
|
278
|
+
with self._sqlite_transaction(conn, deadline=deadline):
|
|
279
|
+
existing = conn.execute(
|
|
280
|
+
"SELECT * FROM admission_journal "
|
|
281
|
+
"WHERE profile_id=? AND idempotency_key=?",
|
|
282
|
+
(request.profile_id, request.idempotency_key),
|
|
283
|
+
).fetchone()
|
|
284
|
+
if existing is not None:
|
|
285
|
+
entry = self._entry_from_row(existing)
|
|
286
|
+
if entry.request_hash != request_hash:
|
|
287
|
+
raise IdempotencyConflict(
|
|
288
|
+
"idempotency key belongs to a different immutable request"
|
|
289
|
+
)
|
|
290
|
+
return entry
|
|
291
|
+
conn.execute(
|
|
292
|
+
"INSERT INTO admission_journal "
|
|
293
|
+
"(journal_id, idempotency_key, request_hash, profile_id, "
|
|
294
|
+
"command_json, state, created_at_ms, updated_at_ms) "
|
|
295
|
+
"VALUES (?, ?, ?, ?, ?, 'prepared', ?, ?)",
|
|
296
|
+
(
|
|
297
|
+
journal_id,
|
|
298
|
+
request.idempotency_key,
|
|
299
|
+
request_hash,
|
|
300
|
+
request.profile_id,
|
|
301
|
+
command_json,
|
|
302
|
+
now,
|
|
303
|
+
now,
|
|
304
|
+
),
|
|
305
|
+
)
|
|
306
|
+
row = conn.execute(
|
|
307
|
+
"SELECT * FROM admission_journal WHERE journal_id=?",
|
|
308
|
+
(journal_id,),
|
|
309
|
+
).fetchone()
|
|
281
310
|
assert row is not None
|
|
282
311
|
return self._entry_from_row(row)
|
|
283
312
|
|
|
@@ -319,16 +348,22 @@ class AdmissionJournal:
|
|
|
319
348
|
journal_id: str,
|
|
320
349
|
*,
|
|
321
350
|
deadline: float | None = None,
|
|
351
|
+
known_prepared: bool = False,
|
|
322
352
|
) -> AdmissionEntry:
|
|
323
353
|
# A concurrent retry may observe ``prepared`` and then lose the race to
|
|
324
354
|
# another caller that commits the same idempotent command. Treat that
|
|
325
355
|
# terminal state as a successful no-op so the retry can return the
|
|
326
356
|
# canonical receipt instead of surfacing a false transition failure.
|
|
357
|
+
if not known_prepared:
|
|
358
|
+
existing = self._get_entry(journal_id, deadline=deadline)
|
|
359
|
+
if existing.state in {"dispatched", "committed"}:
|
|
360
|
+
return existing
|
|
327
361
|
return self._transition(
|
|
328
362
|
journal_id,
|
|
329
363
|
target="dispatched",
|
|
330
364
|
allowed={"prepared", "dispatched", "committed"},
|
|
331
365
|
deadline=deadline,
|
|
366
|
+
full_sync=False,
|
|
332
367
|
)
|
|
333
368
|
|
|
334
369
|
def mark_rejected(
|
|
@@ -363,6 +398,9 @@ class AdmissionJournal:
|
|
|
363
398
|
raise ValueError("receipt operation_id must be a string")
|
|
364
399
|
if commit_sequence is not None and not isinstance(commit_sequence, int):
|
|
365
400
|
raise ValueError("receipt commit_sequence must be an integer")
|
|
401
|
+
existing = self._get_entry(journal_id, deadline=deadline)
|
|
402
|
+
if existing.state == "committed":
|
|
403
|
+
return existing
|
|
366
404
|
return self._transition(
|
|
367
405
|
journal_id,
|
|
368
406
|
target="committed",
|
|
@@ -374,24 +412,13 @@ class AdmissionJournal:
|
|
|
374
412
|
)
|
|
375
413
|
|
|
376
414
|
def get(self, journal_id: str) -> AdmissionEntry:
|
|
377
|
-
|
|
378
|
-
row = conn.execute(
|
|
379
|
-
"SELECT * FROM admission_journal WHERE journal_id=?", (journal_id,)
|
|
380
|
-
).fetchone()
|
|
381
|
-
if row is None:
|
|
382
|
-
raise KeyError(journal_id)
|
|
383
|
-
return self._entry_from_row(row)
|
|
415
|
+
return self._get_entry(journal_id)
|
|
384
416
|
|
|
385
417
|
def get_by_idempotency_key(
|
|
386
418
|
self, profile_id: str, idempotency_key: str
|
|
387
419
|
) -> AdmissionEntry | None:
|
|
388
420
|
"""Return a profile-scoped retry record, never a cross-profile match."""
|
|
389
|
-
|
|
390
|
-
row = conn.execute(
|
|
391
|
-
"SELECT * FROM admission_journal WHERE profile_id=? AND idempotency_key=?",
|
|
392
|
-
(profile_id, idempotency_key),
|
|
393
|
-
).fetchone()
|
|
394
|
-
return self._entry_from_row(row) if row is not None else None
|
|
421
|
+
return self._get_by_idempotency_key(profile_id, idempotency_key)
|
|
395
422
|
|
|
396
423
|
def count(self) -> int:
|
|
397
424
|
with self._read_connection() as conn:
|
|
@@ -451,8 +478,9 @@ class AdmissionJournal:
|
|
|
451
478
|
operation_id: str | None = None,
|
|
452
479
|
commit_sequence: int | None = None,
|
|
453
480
|
deadline: float | None = None,
|
|
481
|
+
full_sync: bool = True,
|
|
454
482
|
) -> AdmissionEntry:
|
|
455
|
-
with self._write_transaction(deadline=deadline) as conn:
|
|
483
|
+
with self._write_transaction(deadline=deadline, full_sync=full_sync) as conn:
|
|
456
484
|
row = conn.execute(
|
|
457
485
|
"SELECT * FROM admission_journal WHERE journal_id=?", (journal_id,)
|
|
458
486
|
).fetchone()
|
|
@@ -490,6 +518,34 @@ class AdmissionJournal:
|
|
|
490
518
|
assert updated is not None
|
|
491
519
|
return self._entry_from_row(updated)
|
|
492
520
|
|
|
521
|
+
def _get_entry(
|
|
522
|
+
self,
|
|
523
|
+
journal_id: str,
|
|
524
|
+
*,
|
|
525
|
+
deadline: float | None = None,
|
|
526
|
+
) -> AdmissionEntry:
|
|
527
|
+
with self._read_connection(deadline=deadline) as conn:
|
|
528
|
+
row = conn.execute(
|
|
529
|
+
"SELECT * FROM admission_journal WHERE journal_id=?", (journal_id,)
|
|
530
|
+
).fetchone()
|
|
531
|
+
if row is None:
|
|
532
|
+
raise KeyError(journal_id)
|
|
533
|
+
return self._entry_from_row(row)
|
|
534
|
+
|
|
535
|
+
def _get_by_idempotency_key(
|
|
536
|
+
self,
|
|
537
|
+
profile_id: str,
|
|
538
|
+
idempotency_key: str,
|
|
539
|
+
*,
|
|
540
|
+
deadline: float | None = None,
|
|
541
|
+
) -> AdmissionEntry | None:
|
|
542
|
+
with self._read_connection(deadline=deadline) as conn:
|
|
543
|
+
row = conn.execute(
|
|
544
|
+
"SELECT * FROM admission_journal WHERE profile_id=? AND idempotency_key=?",
|
|
545
|
+
(profile_id, idempotency_key),
|
|
546
|
+
).fetchone()
|
|
547
|
+
return self._entry_from_row(row) if row is not None else None
|
|
548
|
+
|
|
493
549
|
def _initialize(self) -> None:
|
|
494
550
|
with self._connection() as conn:
|
|
495
551
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
@@ -503,8 +559,52 @@ class AdmissionJournal:
|
|
|
503
559
|
self,
|
|
504
560
|
*,
|
|
505
561
|
deadline: float | None = None,
|
|
562
|
+
full_sync: bool = True,
|
|
506
563
|
) -> Generator[sqlite3.Connection, None, None]:
|
|
507
564
|
"""Serialize and atomically commit one deadline-bounded mutation."""
|
|
565
|
+
with self._write_connection_slot(deadline=deadline):
|
|
566
|
+
remaining = _remaining_seconds(deadline)
|
|
567
|
+
with self._connection(timeout=remaining) as conn:
|
|
568
|
+
with self._write_slot(deadline=deadline):
|
|
569
|
+
if not full_sync:
|
|
570
|
+
# ``dispatched`` is advisory: both prepared and dispatched
|
|
571
|
+
# entries replay through the same idempotent coordinator.
|
|
572
|
+
# NORMAL avoids a second foreground fsync after the durable
|
|
573
|
+
# FULL-synchronous prepare; a power loss can at worst
|
|
574
|
+
# restore the safe prepared state.
|
|
575
|
+
conn.execute("PRAGMA synchronous=NORMAL")
|
|
576
|
+
with self._sqlite_transaction(conn, deadline=deadline):
|
|
577
|
+
yield conn
|
|
578
|
+
|
|
579
|
+
@contextmanager
|
|
580
|
+
def _write_connection_slot(
|
|
581
|
+
self,
|
|
582
|
+
*,
|
|
583
|
+
deadline: float | None = None,
|
|
584
|
+
) -> Generator[None, None, None]:
|
|
585
|
+
"""Bound SQLite handles retained by mutations queued for the writer."""
|
|
586
|
+
if deadline is None:
|
|
587
|
+
acquired = self._write_connection_slots.acquire()
|
|
588
|
+
else:
|
|
589
|
+
acquired = self._write_connection_slots.acquire(
|
|
590
|
+
timeout=max(0.0, deadline - time.monotonic())
|
|
591
|
+
)
|
|
592
|
+
if not acquired:
|
|
593
|
+
raise AdmissionJournalUnavailable(
|
|
594
|
+
"admission journal deadline expired waiting for a connection"
|
|
595
|
+
)
|
|
596
|
+
try:
|
|
597
|
+
yield
|
|
598
|
+
finally:
|
|
599
|
+
self._write_connection_slots.release()
|
|
600
|
+
|
|
601
|
+
@contextmanager
|
|
602
|
+
def _write_slot(
|
|
603
|
+
self,
|
|
604
|
+
*,
|
|
605
|
+
deadline: float | None = None,
|
|
606
|
+
) -> Generator[None, None, None]:
|
|
607
|
+
"""Acquire the process-local SQLite writer slot within the caller budget."""
|
|
508
608
|
if deadline is None:
|
|
509
609
|
acquired = self._write_lock.acquire()
|
|
510
610
|
else:
|
|
@@ -516,33 +616,48 @@ class AdmissionJournal:
|
|
|
516
616
|
"admission journal deadline expired waiting for its writer"
|
|
517
617
|
)
|
|
518
618
|
try:
|
|
519
|
-
|
|
520
|
-
with self._connection(timeout=remaining) as conn:
|
|
521
|
-
try:
|
|
522
|
-
conn.execute("BEGIN IMMEDIATE")
|
|
523
|
-
if deadline is not None and time.monotonic() >= deadline:
|
|
524
|
-
raise AdmissionJournalUnavailable(
|
|
525
|
-
"admission journal deadline expired before its mutation"
|
|
526
|
-
)
|
|
527
|
-
yield conn
|
|
528
|
-
if deadline is not None and time.monotonic() >= deadline:
|
|
529
|
-
raise AdmissionJournalUnavailable(
|
|
530
|
-
"admission journal deadline expired during its mutation"
|
|
531
|
-
)
|
|
532
|
-
conn.commit()
|
|
533
|
-
except sqlite3.OperationalError as exc:
|
|
534
|
-
conn.rollback()
|
|
535
|
-
if _is_sqlite_busy(exc):
|
|
536
|
-
raise AdmissionJournalUnavailable(
|
|
537
|
-
"admission journal is busy beyond its caller deadline"
|
|
538
|
-
) from exc
|
|
539
|
-
raise
|
|
540
|
-
except BaseException:
|
|
541
|
-
conn.rollback()
|
|
542
|
-
raise
|
|
619
|
+
yield
|
|
543
620
|
finally:
|
|
544
621
|
self._write_lock.release()
|
|
545
622
|
|
|
623
|
+
@contextmanager
|
|
624
|
+
def _sqlite_transaction(
|
|
625
|
+
self,
|
|
626
|
+
conn: sqlite3.Connection,
|
|
627
|
+
*,
|
|
628
|
+
deadline: float | None = None,
|
|
629
|
+
) -> Generator[None, None, None]:
|
|
630
|
+
"""Commit or roll back one SQLite mutation on an already-open connection."""
|
|
631
|
+
try:
|
|
632
|
+
# A reused prepare connection may have waited for the local writer
|
|
633
|
+
# slot after its optimistic read. Refresh SQLite's own wait budget
|
|
634
|
+
# immediately before BEGIN so combined local and external
|
|
635
|
+
# contention cannot exceed the caller's original deadline.
|
|
636
|
+
remaining = _remaining_seconds(deadline)
|
|
637
|
+
busy_timeout_ms = max(1, int(remaining * 1_000))
|
|
638
|
+
conn.execute(f"PRAGMA busy_timeout={busy_timeout_ms}")
|
|
639
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
640
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
641
|
+
raise AdmissionJournalUnavailable(
|
|
642
|
+
"admission journal deadline expired before its mutation"
|
|
643
|
+
)
|
|
644
|
+
yield
|
|
645
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
646
|
+
raise AdmissionJournalUnavailable(
|
|
647
|
+
"admission journal deadline expired during its mutation"
|
|
648
|
+
)
|
|
649
|
+
conn.commit()
|
|
650
|
+
except sqlite3.OperationalError as exc:
|
|
651
|
+
conn.rollback()
|
|
652
|
+
if _is_sqlite_busy(exc):
|
|
653
|
+
raise AdmissionJournalUnavailable(
|
|
654
|
+
"admission journal is busy beyond its caller deadline"
|
|
655
|
+
) from exc
|
|
656
|
+
raise
|
|
657
|
+
except BaseException:
|
|
658
|
+
conn.rollback()
|
|
659
|
+
raise
|
|
660
|
+
|
|
546
661
|
@contextmanager
|
|
547
662
|
def _read_connection(
|
|
548
663
|
self,
|