superlocalmemory 3.8.5 → 3.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +3 -3
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +9 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +139 -404
- package/src/superlocalmemory/core/backend_orchestrator.py +7 -1
- package/src/superlocalmemory/core/component_registry.py +4 -2
- package/src/superlocalmemory/core/embeddings.py +33 -6
- package/src/superlocalmemory/core/engine.py +94 -49
- package/src/superlocalmemory/core/engine_ingestion.py +150 -63
- package/src/superlocalmemory/core/ingestion_command.py +133 -21
- package/src/superlocalmemory/core/mutations.py +32 -10
- package/src/superlocalmemory/core/recall_pipeline.py +111 -77
- package/src/superlocalmemory/core/remember_admission.py +152 -0
- package/src/superlocalmemory/core/remember_runtime.py +712 -0
- package/src/superlocalmemory/graph/cozo_backend.py +5 -5
- package/src/superlocalmemory/learning/bandit.py +50 -1
- package/src/superlocalmemory/learning/source_quality.py +38 -35
- package/src/superlocalmemory/mcp/_daemon_proxy.py +38 -15
- package/src/superlocalmemory/mcp/tools_active.py +4 -41
- package/src/superlocalmemory/mcp/tools_core.py +26 -87
- package/src/superlocalmemory/mcp/tools_evolution.py +5 -10
- package/src/superlocalmemory/optimize/proxy/capture.py +196 -8
- package/src/superlocalmemory/retrieval/engine.py +8 -3
- package/src/superlocalmemory/retrieval/reranker.py +35 -10
- package/src/superlocalmemory/server/loopback.py +7 -13
- package/src/superlocalmemory/server/profile_runtime.py +14 -0
- package/src/superlocalmemory/server/routes/abstraction.py +2 -4
- package/src/superlocalmemory/server/routes/agents.py +3 -5
- package/src/superlocalmemory/server/routes/behavioral.py +5 -13
- package/src/superlocalmemory/server/routes/brain.py +6 -9
- package/src/superlocalmemory/server/routes/entity.py +3 -7
- package/src/superlocalmemory/server/routes/evolution.py +3 -5
- package/src/superlocalmemory/server/routes/helpers.py +44 -23
- package/src/superlocalmemory/server/routes/insights.py +2 -4
- package/src/superlocalmemory/server/routes/learning.py +2 -5
- package/src/superlocalmemory/server/routes/lifecycle.py +2 -4
- package/src/superlocalmemory/server/routes/memories.py +122 -100
- package/src/superlocalmemory/server/routes/tiers.py +3 -22
- package/src/superlocalmemory/server/routes/timeline.py +2 -4
- package/src/superlocalmemory/server/routes/v3_api.py +18 -16
- package/src/superlocalmemory/server/unified_daemon.py +200 -109
- package/src/superlocalmemory/storage/admission_codec.py +119 -0
- package/src/superlocalmemory/storage/admission_journal.py +728 -0
- package/src/superlocalmemory/storage/database.py +59 -0
- package/src/superlocalmemory/storage/deferred_writes.py +67 -11
- package/src/superlocalmemory/storage/memory_write.py +8 -12
- package/src/superlocalmemory/storage/migration_runner.py +37 -0
- package/src/superlocalmemory/storage/migrations/M032_write_coordinator_admission.py +188 -0
- package/src/superlocalmemory/storage/read_connection.py +115 -0
- package/src/superlocalmemory/storage/write_coordinator.py +756 -0
- package/src/superlocalmemory/ui/index.html +1 -1
- package/src/superlocalmemory/ui/js/auto-settings.js +14 -1
- package/src/superlocalmemory/ui/js/od-settings.js +9 -3
|
@@ -9,7 +9,8 @@ Activation: set ``SLM_OPTIMIZE_CAPTURE=1`` in the daemon's environment. When on:
|
|
|
9
9
|
at load time (see server._load_hooks), so capture never observes a mutated
|
|
10
10
|
request or a cache hit; every line is a genuine upstream exchange.
|
|
11
11
|
* each completed exchange is appended as one JSON line to
|
|
12
|
-
``~/.superlocalmemory/optimize_capture.jsonl`` (0600
|
|
12
|
+
``~/.superlocalmemory/optimize_capture.jsonl`` (owner-only: POSIX 0600 or
|
|
13
|
+
Windows owner DACL, gitignored).
|
|
13
14
|
|
|
14
15
|
ISOLATION GUARANTEE: this module writes ONLY to optimize_capture.jsonl. It never
|
|
15
16
|
opens memory.db, llmcache.db, or any SLM memory store. (Plan §9 hard rule.)
|
|
@@ -21,9 +22,11 @@ swallowed — it MUST NOT break the proxied request the user is waiting on.
|
|
|
21
22
|
from __future__ import annotations
|
|
22
23
|
|
|
23
24
|
import asyncio
|
|
25
|
+
import errno
|
|
24
26
|
import json
|
|
25
27
|
import logging
|
|
26
28
|
import os
|
|
29
|
+
import stat
|
|
27
30
|
import threading
|
|
28
31
|
from pathlib import Path
|
|
29
32
|
from typing import Any
|
|
@@ -55,6 +58,180 @@ def _capture_path() -> Path:
|
|
|
55
58
|
return state_path(_CAPTURE_FILENAME)
|
|
56
59
|
|
|
57
60
|
|
|
61
|
+
def _is_windows() -> bool:
|
|
62
|
+
"""Return whether the running platform uses Windows DACLs."""
|
|
63
|
+
return os.name == "nt"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _is_link_or_reparse(info: os.stat_result) -> bool:
|
|
67
|
+
"""Reject POSIX symlinks and Windows reparse points before capture."""
|
|
68
|
+
reparse = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)
|
|
69
|
+
attributes = getattr(info, "st_file_attributes", 0)
|
|
70
|
+
return stat.S_ISLNK(info.st_mode) or bool(reparse and attributes & reparse)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _windows_owner_dacl(
|
|
74
|
+
win32api: Any,
|
|
75
|
+
win32con: Any,
|
|
76
|
+
win32security: Any,
|
|
77
|
+
) -> Any:
|
|
78
|
+
"""Build one protected owner-only DACL for a Windows capture file."""
|
|
79
|
+
import ntsecuritycon
|
|
80
|
+
|
|
81
|
+
token = win32security.OpenProcessToken(
|
|
82
|
+
win32api.GetCurrentProcess(),
|
|
83
|
+
win32con.TOKEN_QUERY,
|
|
84
|
+
)
|
|
85
|
+
try:
|
|
86
|
+
owner_sid = win32security.GetTokenInformation(
|
|
87
|
+
token,
|
|
88
|
+
win32security.TokenUser,
|
|
89
|
+
)[0]
|
|
90
|
+
finally:
|
|
91
|
+
token.Close()
|
|
92
|
+
dacl = win32security.ACL()
|
|
93
|
+
dacl.AddAccessAllowedAce(
|
|
94
|
+
win32security.ACL_REVISION,
|
|
95
|
+
ntsecuritycon.FILE_ALL_ACCESS,
|
|
96
|
+
owner_sid,
|
|
97
|
+
)
|
|
98
|
+
return dacl
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _open_windows_capture_append(path: Path) -> int:
|
|
102
|
+
"""Create/open a Windows file with append + WRITE_DAC on one handle."""
|
|
103
|
+
try:
|
|
104
|
+
import msvcrt
|
|
105
|
+
|
|
106
|
+
import ntsecuritycon
|
|
107
|
+
import win32api
|
|
108
|
+
import win32con
|
|
109
|
+
import win32file
|
|
110
|
+
import win32security
|
|
111
|
+
except ImportError as exc:
|
|
112
|
+
raise OSError("Windows capture ACL support is unavailable") from exc
|
|
113
|
+
|
|
114
|
+
handle = None
|
|
115
|
+
try:
|
|
116
|
+
# A creation-time descriptor prevents a newly created capture file from
|
|
117
|
+
# ever inheriting a broader parent DACL. For an existing file Windows
|
|
118
|
+
# ignores this descriptor; _enforce_owner_only_permissions replaces
|
|
119
|
+
# that DACL through the same WRITE_DAC-capable handle before writing.
|
|
120
|
+
dacl = _windows_owner_dacl(win32api, win32con, win32security)
|
|
121
|
+
security_attributes = win32security.SECURITY_ATTRIBUTES()
|
|
122
|
+
security_attributes.bInheritHandle = False
|
|
123
|
+
security_attributes.SECURITY_DESCRIPTOR.SetSecurityDescriptorDacl(
|
|
124
|
+
1,
|
|
125
|
+
dacl,
|
|
126
|
+
0,
|
|
127
|
+
)
|
|
128
|
+
security_attributes.SECURITY_DESCRIPTOR.SetSecurityDescriptorControl(
|
|
129
|
+
win32security.SE_DACL_PROTECTED,
|
|
130
|
+
win32security.SE_DACL_PROTECTED,
|
|
131
|
+
)
|
|
132
|
+
handle = win32file.CreateFile(
|
|
133
|
+
os.fspath(path),
|
|
134
|
+
ntsecuritycon.FILE_APPEND_DATA | ntsecuritycon.WRITE_DAC,
|
|
135
|
+
win32con.FILE_SHARE_READ
|
|
136
|
+
| win32con.FILE_SHARE_WRITE
|
|
137
|
+
| win32con.FILE_SHARE_DELETE,
|
|
138
|
+
security_attributes,
|
|
139
|
+
win32con.OPEN_ALWAYS,
|
|
140
|
+
win32con.FILE_ATTRIBUTE_NORMAL
|
|
141
|
+
# pywin32 does not export this SDK constant from win32con on
|
|
142
|
+
# every supported Python build. Keep the Microsoft-defined value
|
|
143
|
+
# as a named fallback rather than silently following a reparse.
|
|
144
|
+
| getattr(win32con, "FILE_FLAG_OPEN_REPARSE_POINT", 0x00200000),
|
|
145
|
+
None,
|
|
146
|
+
)
|
|
147
|
+
file_info = win32file.GetFileInformationByHandle(handle)
|
|
148
|
+
if file_info[0] & win32con.FILE_ATTRIBUTE_REPARSE_POINT:
|
|
149
|
+
raise OSError(
|
|
150
|
+
errno.ELOOP,
|
|
151
|
+
"capture path is a Windows reparse point",
|
|
152
|
+
path,
|
|
153
|
+
)
|
|
154
|
+
try:
|
|
155
|
+
# Enforce the DACL while this is still the original CreateFile
|
|
156
|
+
# handle carrying WRITE_DAC. Transferring it into Python's CRT
|
|
157
|
+
# first can lose the authority SetSecurityInfo needs on Windows.
|
|
158
|
+
win32security.SetSecurityInfo(
|
|
159
|
+
handle,
|
|
160
|
+
win32security.SE_FILE_OBJECT,
|
|
161
|
+
win32security.DACL_SECURITY_INFORMATION
|
|
162
|
+
| win32security.PROTECTED_DACL_SECURITY_INFORMATION,
|
|
163
|
+
None,
|
|
164
|
+
None,
|
|
165
|
+
dacl,
|
|
166
|
+
None,
|
|
167
|
+
)
|
|
168
|
+
except Exception as exc:
|
|
169
|
+
raise OSError(
|
|
170
|
+
"Windows capture ACL could not be enforced "
|
|
171
|
+
f"({type(exc).__name__}: {exc})"
|
|
172
|
+
) from exc
|
|
173
|
+
|
|
174
|
+
# Transfer the native handle to Python's CRT descriptor exactly once.
|
|
175
|
+
raw_handle = handle.Detach()
|
|
176
|
+
handle = None
|
|
177
|
+
try:
|
|
178
|
+
return msvcrt.open_osfhandle(
|
|
179
|
+
raw_handle,
|
|
180
|
+
os.O_WRONLY | os.O_APPEND | getattr(os, "O_BINARY", 0),
|
|
181
|
+
)
|
|
182
|
+
except BaseException:
|
|
183
|
+
win32api.CloseHandle(raw_handle)
|
|
184
|
+
raise
|
|
185
|
+
except OSError:
|
|
186
|
+
raise
|
|
187
|
+
except Exception as exc:
|
|
188
|
+
raise OSError(f"Windows secure capture open failed: {exc}") from exc
|
|
189
|
+
finally:
|
|
190
|
+
if handle is not None:
|
|
191
|
+
handle.Close()
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _open_capture_append(path: Path) -> int:
|
|
195
|
+
"""Open one append descriptor without following or racing a link target."""
|
|
196
|
+
try:
|
|
197
|
+
before = path.lstat()
|
|
198
|
+
except FileNotFoundError:
|
|
199
|
+
before = None
|
|
200
|
+
if before is not None and _is_link_or_reparse(before):
|
|
201
|
+
raise OSError(errno.ELOOP, "capture path is a link or reparse point", path)
|
|
202
|
+
|
|
203
|
+
if _is_windows():
|
|
204
|
+
fd = _open_windows_capture_append(path)
|
|
205
|
+
else:
|
|
206
|
+
flags = os.O_CREAT | os.O_WRONLY | os.O_APPEND | getattr(os, "O_NOFOLLOW", 0)
|
|
207
|
+
fd = os.open(path, flags, 0o600)
|
|
208
|
+
try:
|
|
209
|
+
opened = os.fstat(fd)
|
|
210
|
+
current = path.lstat()
|
|
211
|
+
if (
|
|
212
|
+
_is_link_or_reparse(current)
|
|
213
|
+
or (opened.st_dev, opened.st_ino) != (current.st_dev, current.st_ino)
|
|
214
|
+
):
|
|
215
|
+
raise OSError(
|
|
216
|
+
errno.ELOOP,
|
|
217
|
+
"capture path changed during secure open",
|
|
218
|
+
path,
|
|
219
|
+
)
|
|
220
|
+
except BaseException:
|
|
221
|
+
os.close(fd)
|
|
222
|
+
raise
|
|
223
|
+
return fd
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _enforce_owner_only_permissions(fd: int) -> None:
|
|
227
|
+
"""Apply owner-only access control through the already-verified handle."""
|
|
228
|
+
if not _is_windows():
|
|
229
|
+
os.fchmod(fd, 0o600)
|
|
230
|
+
return
|
|
231
|
+
# _open_windows_capture_append applies the protected DACL on the original
|
|
232
|
+
# WRITE_DAC-capable CreateFile handle before CRT descriptor transfer.
|
|
233
|
+
|
|
234
|
+
|
|
58
235
|
class ShadowCapture:
|
|
59
236
|
"""Thread-safe append-only JSONL writer for proxy exchanges (singleton)."""
|
|
60
237
|
|
|
@@ -95,10 +272,10 @@ class ShadowCapture:
|
|
|
95
272
|
Fail-open: any error is logged and False is returned; never raised.
|
|
96
273
|
|
|
97
274
|
Security: opens with a single ``os.open`` carrying ``O_CREAT |
|
|
98
|
-
O_APPEND
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
275
|
+
O_APPEND`` and mode ``0o600`` on every write. POSIX adds O_NOFOLLOW;
|
|
276
|
+
all platforms reject link/reparse metadata and verify that the opened
|
|
277
|
+
descriptor still identifies the current path. Permissions are then
|
|
278
|
+
enforced through that verified descriptor before any data is written.
|
|
102
279
|
"""
|
|
103
280
|
try:
|
|
104
281
|
line = json.dumps(entry, ensure_ascii=False, separators=(",", ":"))
|
|
@@ -109,9 +286,20 @@ class ShadowCapture:
|
|
|
109
286
|
try:
|
|
110
287
|
with self._write_lock:
|
|
111
288
|
self._path.parent.mkdir(parents=True, exist_ok=True)
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
289
|
+
fd = _open_capture_append(self._path)
|
|
290
|
+
try:
|
|
291
|
+
# Enforce the ACL through the verified descriptor so a
|
|
292
|
+
# pathname swap cannot redirect chmod/icacls elsewhere.
|
|
293
|
+
_enforce_owner_only_permissions(fd)
|
|
294
|
+
# fdopen transfers descriptor ownership only after it
|
|
295
|
+
# returns successfully. Keep this conversion inside the
|
|
296
|
+
# explicit-close boundary so an allocation/codec failure
|
|
297
|
+
# cannot leak one descriptor per captured request.
|
|
298
|
+
fh = os.fdopen(fd, "a", encoding="utf-8")
|
|
299
|
+
except BaseException:
|
|
300
|
+
os.close(fd)
|
|
301
|
+
raise
|
|
302
|
+
with fh:
|
|
115
303
|
fh.write(line + "\n")
|
|
116
304
|
self._count += 1
|
|
117
305
|
return True
|
|
@@ -904,13 +904,18 @@ class RetrievalEngine:
|
|
|
904
904
|
|
|
905
905
|
return out
|
|
906
906
|
|
|
907
|
-
def close(self) -> None:
|
|
908
|
-
"""Release
|
|
907
|
+
def close(self, *, wait: bool = False) -> None:
|
|
908
|
+
"""Release owned channel workers without blocking daemon shutdown.
|
|
909
|
+
|
|
910
|
+
Active channel calls have their own response deadline. Waiting here
|
|
911
|
+
can still deadlock shutdown when an extension ignores that deadline,
|
|
912
|
+
so the daemon uses the executor's non-blocking cancellation path.
|
|
913
|
+
"""
|
|
909
914
|
with self._close_lock:
|
|
910
915
|
if self._closed:
|
|
911
916
|
return
|
|
912
917
|
self._closed = True
|
|
913
|
-
self._channel_executor.shutdown(wait=
|
|
918
|
+
self._channel_executor.shutdown(wait=wait, cancel_futures=True)
|
|
914
919
|
|
|
915
920
|
# -- Fact loading -------------------------------------------------------
|
|
916
921
|
|
|
@@ -111,6 +111,8 @@ class CrossEncoderReranker:
|
|
|
111
111
|
self._model_loaded = False # True once worker confirms model is ready
|
|
112
112
|
self._worker_loading = False # True while background warmup in progress
|
|
113
113
|
self._lock = threading.Lock()
|
|
114
|
+
self._shutdown_event = threading.Event()
|
|
115
|
+
self._warmup_thread: threading.Thread | None = None
|
|
114
116
|
self._idle_timer: threading.Timer | None = None
|
|
115
117
|
self._request_count: int = 0
|
|
116
118
|
|
|
@@ -126,7 +128,7 @@ class CrossEncoderReranker:
|
|
|
126
128
|
def __del__(self) -> None:
|
|
127
129
|
"""Kill worker subprocess when reranker is garbage-collected."""
|
|
128
130
|
try:
|
|
129
|
-
self.
|
|
131
|
+
self.shutdown(timeout=0.1)
|
|
130
132
|
except Exception:
|
|
131
133
|
pass
|
|
132
134
|
|
|
@@ -142,14 +144,14 @@ class CrossEncoderReranker:
|
|
|
142
144
|
lock, creating a race where the warmup's readline thread could
|
|
143
145
|
steal responses meant for _send_request → deadlock → timeout.
|
|
144
146
|
"""
|
|
145
|
-
if self._worker_loading or self._model_loaded:
|
|
147
|
+
if self._shutdown_event.is_set() or self._worker_loading or self._model_loaded:
|
|
146
148
|
return
|
|
147
149
|
self._worker_loading = True
|
|
148
150
|
|
|
149
151
|
def _warmup() -> None:
|
|
150
152
|
try:
|
|
151
153
|
for attempt in range(1, _WARMUP_MAX_ATTEMPTS + 1):
|
|
152
|
-
if self._model_loaded:
|
|
154
|
+
if self._shutdown_event.is_set() or self._model_loaded:
|
|
153
155
|
return
|
|
154
156
|
try:
|
|
155
157
|
self._ensure_worker()
|
|
@@ -206,7 +208,10 @@ class CrossEncoderReranker:
|
|
|
206
208
|
)
|
|
207
209
|
|
|
208
210
|
if attempt < _WARMUP_MAX_ATTEMPTS and not self._model_loaded:
|
|
209
|
-
|
|
211
|
+
if self._shutdown_event.wait(
|
|
212
|
+
min(_WARMUP_RETRY_BACKOFF_S * attempt, 15.0),
|
|
213
|
+
):
|
|
214
|
+
return
|
|
210
215
|
|
|
211
216
|
if not self._model_loaded:
|
|
212
217
|
logger.warning(
|
|
@@ -231,7 +236,11 @@ class CrossEncoderReranker:
|
|
|
231
236
|
"""
|
|
232
237
|
if self._model_loaded:
|
|
233
238
|
return True
|
|
234
|
-
if
|
|
239
|
+
if (
|
|
240
|
+
not self._shutdown_event.is_set()
|
|
241
|
+
and not self._worker_loading
|
|
242
|
+
and not self._model_loaded
|
|
243
|
+
):
|
|
235
244
|
self._start_background_warmup()
|
|
236
245
|
t = getattr(self, '_warmup_thread', None)
|
|
237
246
|
if t is not None:
|
|
@@ -248,6 +257,8 @@ class CrossEncoderReranker:
|
|
|
248
257
|
v3.4.13: Checks PID file before spawning — only ONE reranker worker
|
|
249
258
|
can exist at a time on the machine.
|
|
250
259
|
"""
|
|
260
|
+
if self._shutdown_event.is_set():
|
|
261
|
+
return
|
|
251
262
|
if self._worker_proc is not None and self._worker_proc.poll() is None:
|
|
252
263
|
return
|
|
253
264
|
self._worker_proc = None
|
|
@@ -327,6 +338,8 @@ class CrossEncoderReranker:
|
|
|
327
338
|
falls back to fusion scores without reranking). This prevents
|
|
328
339
|
concurrent recall requests from serialising on the lock.
|
|
329
340
|
"""
|
|
341
|
+
if self._shutdown_event.is_set():
|
|
342
|
+
return None
|
|
330
343
|
effective_timeout = timeout or _SUBPROCESS_RESPONSE_TIMEOUT
|
|
331
344
|
|
|
332
345
|
acquired = self._lock.acquire(blocking=block)
|
|
@@ -393,7 +406,7 @@ class CrossEncoderReranker:
|
|
|
393
406
|
raise error_container[0]
|
|
394
407
|
return result_container[0] if result_container else ""
|
|
395
408
|
|
|
396
|
-
def _kill_worker(self) -> None:
|
|
409
|
+
def _kill_worker(self, timeout: float = 3.0) -> None:
|
|
397
410
|
"""Terminate the worker and close every owned pipe exactly once."""
|
|
398
411
|
if self._idle_timer is not None:
|
|
399
412
|
self._idle_timer.cancel()
|
|
@@ -413,7 +426,7 @@ class CrossEncoderReranker:
|
|
|
413
426
|
try:
|
|
414
427
|
proc.stdin.write('{"cmd":"quit"}\n')
|
|
415
428
|
proc.stdin.flush()
|
|
416
|
-
proc.wait(timeout=
|
|
429
|
+
proc.wait(timeout=max(0.0, timeout))
|
|
417
430
|
except Exception:
|
|
418
431
|
try:
|
|
419
432
|
returncode = proc.poll()
|
|
@@ -422,7 +435,7 @@ class CrossEncoderReranker:
|
|
|
422
435
|
if returncode is None or not isinstance(returncode, int):
|
|
423
436
|
try:
|
|
424
437
|
proc.kill()
|
|
425
|
-
proc.wait(timeout=
|
|
438
|
+
proc.wait(timeout=max(0.0, timeout))
|
|
426
439
|
except Exception:
|
|
427
440
|
pass
|
|
428
441
|
finally:
|
|
@@ -438,6 +451,8 @@ class CrossEncoderReranker:
|
|
|
438
451
|
|
|
439
452
|
def _reset_idle_timer(self) -> None:
|
|
440
453
|
"""Reset idle timer — kills worker after 2 min inactivity."""
|
|
454
|
+
if self._shutdown_event.is_set():
|
|
455
|
+
return
|
|
441
456
|
if self._idle_timer is not None:
|
|
442
457
|
self._idle_timer.cancel()
|
|
443
458
|
self._idle_timer = threading.Timer(
|
|
@@ -452,6 +467,16 @@ class CrossEncoderReranker:
|
|
|
452
467
|
self._kill_worker()
|
|
453
468
|
logger.info("CrossEncoderReranker: worker killed (idle timeout)")
|
|
454
469
|
|
|
470
|
+
def shutdown(self, timeout: float = 3.0) -> None:
|
|
471
|
+
"""Cancel warmup, terminate the child, and join owned background work."""
|
|
472
|
+
shutdown_event = getattr(self, "_shutdown_event", None)
|
|
473
|
+
if shutdown_event is not None:
|
|
474
|
+
shutdown_event.set()
|
|
475
|
+
self._kill_worker(timeout=min(max(0.0, timeout), 1.0))
|
|
476
|
+
warmup_thread = getattr(self, "_warmup_thread", None)
|
|
477
|
+
if warmup_thread is not None and warmup_thread is not threading.current_thread():
|
|
478
|
+
warmup_thread.join(timeout=timeout)
|
|
479
|
+
|
|
455
480
|
# ------------------------------------------------------------------
|
|
456
481
|
# Public API
|
|
457
482
|
# ------------------------------------------------------------------
|
|
@@ -494,7 +519,7 @@ class CrossEncoderReranker:
|
|
|
494
519
|
# this recall — it returns fallback now and full quality resumes within
|
|
495
520
|
# seconds once the model is warm again.
|
|
496
521
|
if not self._model_loaded:
|
|
497
|
-
if not self._worker_loading:
|
|
522
|
+
if not self._shutdown_event.is_set() and not self._worker_loading:
|
|
498
523
|
self._start_background_warmup()
|
|
499
524
|
sorted_cands = sorted(candidates, key=lambda x: x[1], reverse=True)
|
|
500
525
|
return sorted_cands[:top_k], False, "fallback_not_ready"
|
|
@@ -562,7 +587,7 @@ def _cleanup_all_rerankers() -> None:
|
|
|
562
587
|
reranker = ref()
|
|
563
588
|
if reranker is not None:
|
|
564
589
|
try:
|
|
565
|
-
reranker.
|
|
590
|
+
reranker.shutdown()
|
|
566
591
|
except Exception:
|
|
567
592
|
pass
|
|
568
593
|
_live_rerankers.clear()
|
|
@@ -39,15 +39,6 @@ from __future__ import annotations
|
|
|
39
39
|
import ipaddress as _ipa
|
|
40
40
|
|
|
41
41
|
|
|
42
|
-
# CRIT-2 guard: detect a future CPython regression where IPv4-mapped loopback
|
|
43
|
-
# is no longer reported as .is_loopback. Fails at daemon startup (import time),
|
|
44
|
-
# not silently at request time.
|
|
45
|
-
assert _ipa.ip_address("::ffff:127.0.0.1").is_loopback, (
|
|
46
|
-
"Python ipaddress regression: ::ffff:127.0.0.1 no longer reports as "
|
|
47
|
-
"loopback. Update superlocalmemory/server/loopback.py. (Issue #90)"
|
|
48
|
-
)
|
|
49
|
-
|
|
50
|
-
|
|
51
42
|
def is_loopback(host: str) -> bool:
|
|
52
43
|
"""Return ``True`` iff ``host`` is a loopback address in any standard form.
|
|
53
44
|
|
|
@@ -82,10 +73,13 @@ def is_loopback(host: str) -> bool:
|
|
|
82
73
|
ip = _ipa.ip_address(host)
|
|
83
74
|
except ValueError:
|
|
84
75
|
return False
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
#
|
|
88
|
-
|
|
76
|
+
if ip.is_loopback:
|
|
77
|
+
return True
|
|
78
|
+
# CPython's handling of IPv4-mapped IPv6 changed across supported
|
|
79
|
+
# runtimes. Normalize through the embedded IPv4 address so a dual-stack
|
|
80
|
+
# socket reporting ::ffff:127.x.x.x retains the equivalent IPv4 decision.
|
|
81
|
+
mapped_ipv4 = getattr(ip, "ipv4_mapped", None)
|
|
82
|
+
return bool(mapped_ipv4 is not None and mapped_ipv4.is_loopback)
|
|
89
83
|
|
|
90
84
|
|
|
91
85
|
__all__ = ("is_loopback",)
|
|
@@ -361,6 +361,13 @@ def commit_daemon_profile_switch(
|
|
|
361
361
|
# Rebind the in-memory engine first, then make compatibility files the
|
|
362
362
|
# final commit step so they can never lead daemon runtime truth.
|
|
363
363
|
engine.profile_id = target_profile
|
|
364
|
+
canonical_remember = getattr(
|
|
365
|
+
app_state,
|
|
366
|
+
"canonical_remember_runtime",
|
|
367
|
+
None,
|
|
368
|
+
)
|
|
369
|
+
if canonical_remember is not None:
|
|
370
|
+
canonical_remember.rebind_engine(engine)
|
|
364
371
|
if app_config is not None:
|
|
365
372
|
app_config.active_profile = target_profile
|
|
366
373
|
if engine_config is not None:
|
|
@@ -368,6 +375,13 @@ def commit_daemon_profile_switch(
|
|
|
368
375
|
persistence = persist_active_profile(target_profile)
|
|
369
376
|
except BaseException:
|
|
370
377
|
engine.profile_id = previous.profile_id
|
|
378
|
+
canonical_remember = getattr(
|
|
379
|
+
app_state,
|
|
380
|
+
"canonical_remember_runtime",
|
|
381
|
+
None,
|
|
382
|
+
)
|
|
383
|
+
if canonical_remember is not None:
|
|
384
|
+
canonical_remember.rebind_engine(engine)
|
|
371
385
|
if app_config is not None:
|
|
372
386
|
app_config.active_profile = previous.profile_id
|
|
373
387
|
if engine_config is not None:
|
|
@@ -24,7 +24,7 @@ from typing import Any
|
|
|
24
24
|
from fastapi import APIRouter, Query
|
|
25
25
|
from fastapi.responses import JSONResponse
|
|
26
26
|
|
|
27
|
-
from superlocalmemory.server.routes.helpers import DB_PATH, get_active_profile
|
|
27
|
+
from superlocalmemory.server.routes.helpers import DB_PATH, get_active_profile, get_read_connection
|
|
28
28
|
|
|
29
29
|
logger = logging.getLogger(__name__)
|
|
30
30
|
|
|
@@ -45,9 +45,7 @@ class _ReadDB:
|
|
|
45
45
|
def _conn() -> sqlite3.Connection | None:
|
|
46
46
|
if not DB_PATH.exists():
|
|
47
47
|
return None
|
|
48
|
-
|
|
49
|
-
conn.row_factory = sqlite3.Row
|
|
50
|
-
return conn
|
|
48
|
+
return get_read_connection(DB_PATH)
|
|
51
49
|
|
|
52
50
|
|
|
53
51
|
@router.get("/persona")
|
|
@@ -14,7 +14,7 @@ from fastapi import APIRouter, HTTPException, Query, Request
|
|
|
14
14
|
|
|
15
15
|
from superlocalmemory.infra.data_root import state_path
|
|
16
16
|
|
|
17
|
-
from .helpers import DB_PATH
|
|
17
|
+
from .helpers import DB_PATH, get_read_connection
|
|
18
18
|
|
|
19
19
|
logger = logging.getLogger("superlocalmemory.routes.agents")
|
|
20
20
|
router = APIRouter()
|
|
@@ -103,8 +103,7 @@ async def get_agent_memory_activity(
|
|
|
103
103
|
total = 0
|
|
104
104
|
|
|
105
105
|
if DB_PATH.exists():
|
|
106
|
-
conn =
|
|
107
|
-
conn.row_factory = sqlite3.Row
|
|
106
|
+
conn = get_read_connection(DB_PATH)
|
|
108
107
|
try:
|
|
109
108
|
try:
|
|
110
109
|
rows = conn.execute(
|
|
@@ -188,8 +187,7 @@ async def get_trust_stats(request: Request):
|
|
|
188
187
|
by_signal_type = {}
|
|
189
188
|
|
|
190
189
|
if DB_PATH.exists():
|
|
191
|
-
conn =
|
|
192
|
-
conn.row_factory = sqlite3.Row
|
|
190
|
+
conn = get_read_connection(DB_PATH)
|
|
193
191
|
try:
|
|
194
192
|
try:
|
|
195
193
|
# Count trust signals
|
|
@@ -18,6 +18,7 @@ from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator
|
|
|
18
18
|
|
|
19
19
|
from .helpers import MEMORY_DIR, get_active_profile
|
|
20
20
|
from superlocalmemory.storage.memory_write import memory_write
|
|
21
|
+
from superlocalmemory.storage.read_connection import ReadConnectionFactory
|
|
21
22
|
|
|
22
23
|
logger = logging.getLogger("superlocalmemory.routes.behavioral")
|
|
23
24
|
router = APIRouter()
|
|
@@ -146,8 +147,7 @@ def _load_action_outcomes(profile_id: str) -> dict:
|
|
|
146
147
|
if not db_path.exists():
|
|
147
148
|
return empty
|
|
148
149
|
try:
|
|
149
|
-
conn =
|
|
150
|
-
conn.row_factory = sqlite3.Row
|
|
150
|
+
conn = ReadConnectionFactory(db_path).open()
|
|
151
151
|
try:
|
|
152
152
|
columns = {
|
|
153
153
|
str(row["name"])
|
|
@@ -369,7 +369,6 @@ def report_outcome(request: Request, data: ReportOutcomeRequest):
|
|
|
369
369
|
action_type = data.action_type
|
|
370
370
|
context_note = data.context
|
|
371
371
|
|
|
372
|
-
import sqlite3
|
|
373
372
|
import uuid
|
|
374
373
|
from datetime import datetime, timezone
|
|
375
374
|
|
|
@@ -451,10 +450,8 @@ def get_assertions(
|
|
|
451
450
|
):
|
|
452
451
|
"""Get learned behavioral assertions for dashboard display."""
|
|
453
452
|
try:
|
|
454
|
-
import sqlite3 as _sqlite3
|
|
455
453
|
profile = get_active_profile()
|
|
456
|
-
conn =
|
|
457
|
-
conn.row_factory = _sqlite3.Row
|
|
454
|
+
conn = ReadConnectionFactory(MEMORY_DIR / "memory.db").open()
|
|
458
455
|
|
|
459
456
|
query = (
|
|
460
457
|
"SELECT id, trigger_condition, action, category, confidence, "
|
|
@@ -495,10 +492,8 @@ def get_tool_events(
|
|
|
495
492
|
):
|
|
496
493
|
"""Get recent tool events for dashboard display."""
|
|
497
494
|
try:
|
|
498
|
-
import sqlite3 as _sqlite3
|
|
499
495
|
profile = get_active_profile()
|
|
500
|
-
conn =
|
|
501
|
-
conn.row_factory = _sqlite3.Row
|
|
496
|
+
conn = ReadConnectionFactory(MEMORY_DIR / "memory.db").open()
|
|
502
497
|
|
|
503
498
|
query = (
|
|
504
499
|
"SELECT id, tool_name, event_type, input_summary, output_summary, "
|
|
@@ -528,10 +523,8 @@ def get_soft_prompts(request: Request):
|
|
|
528
523
|
"""Get active soft prompt templates for dashboard display."""
|
|
529
524
|
_require_read(request)
|
|
530
525
|
try:
|
|
531
|
-
import sqlite3 as _sqlite3
|
|
532
526
|
profile = get_active_profile()
|
|
533
|
-
conn =
|
|
534
|
-
conn.row_factory = _sqlite3.Row
|
|
527
|
+
conn = ReadConnectionFactory(MEMORY_DIR / "memory.db").open()
|
|
535
528
|
rows = conn.execute(
|
|
536
529
|
"SELECT prompt_id, category, content, confidence, effectiveness, "
|
|
537
530
|
"token_count, active, version, created_at "
|
|
@@ -570,7 +563,6 @@ def log_tool_event_api(request: Request, data: dict):
|
|
|
570
563
|
_require_write(request)
|
|
571
564
|
try:
|
|
572
565
|
import os
|
|
573
|
-
import sqlite3 as _sqlite3
|
|
574
566
|
from datetime import datetime, timezone
|
|
575
567
|
|
|
576
568
|
tool_name = data.get("tool_name", "unknown")
|
|
@@ -60,6 +60,7 @@ from superlocalmemory.core.security_primitives import (
|
|
|
60
60
|
from superlocalmemory.learning.database import LearningDatabase
|
|
61
61
|
from superlocalmemory.learning.features import FEATURE_DIM
|
|
62
62
|
from superlocalmemory.infra.data_root import canonical_data_root
|
|
63
|
+
from superlocalmemory.storage.read_connection import ReadConnectionFactory
|
|
63
64
|
from .helpers import get_active_profile
|
|
64
65
|
|
|
65
66
|
logger = logging.getLogger("superlocalmemory.routes.brain")
|
|
@@ -554,7 +555,7 @@ def _compute_cache_stats() -> dict:
|
|
|
554
555
|
size = db.stat().st_size
|
|
555
556
|
entry_count = 0
|
|
556
557
|
try:
|
|
557
|
-
conn =
|
|
558
|
+
conn = ReadConnectionFactory(db).open()
|
|
558
559
|
try:
|
|
559
560
|
row = conn.execute(
|
|
560
561
|
"SELECT COUNT(*) AS cnt FROM atomic_facts",
|
|
@@ -579,14 +580,11 @@ def _adapter_last_sync_ago(adapter_name: str) -> int | None:
|
|
|
579
580
|
honest empty rather than a fabricated number.
|
|
580
581
|
"""
|
|
581
582
|
try:
|
|
582
|
-
import sqlite3 as _sqlite3
|
|
583
583
|
from datetime import datetime as _dt, timezone as _tz
|
|
584
584
|
memory_db = _memory_dir() / "memory.db"
|
|
585
585
|
if not memory_db.exists():
|
|
586
586
|
return None
|
|
587
|
-
conn =
|
|
588
|
-
f"file:{memory_db}?mode=ro", uri=True, timeout=1.0,
|
|
589
|
-
)
|
|
587
|
+
conn = ReadConnectionFactory(memory_db).open()
|
|
590
588
|
try:
|
|
591
589
|
cur = conn.execute(
|
|
592
590
|
"SELECT last_sync_at FROM cross_platform_sync_log "
|
|
@@ -914,7 +912,7 @@ def _compute_action_outcomes_preview(profile_id: str) -> dict:
|
|
|
914
912
|
if not db_path.exists():
|
|
915
913
|
return empty
|
|
916
914
|
try:
|
|
917
|
-
conn =
|
|
915
|
+
conn = ReadConnectionFactory(db_path).open()
|
|
918
916
|
try:
|
|
919
917
|
row = conn.execute(
|
|
920
918
|
"SELECT COUNT(*) FROM action_outcomes WHERE profile_id = ?",
|
|
@@ -1036,7 +1034,6 @@ def _compute_outcome_queue_stats(profile_id: str) -> dict:
|
|
|
1036
1034
|
``pending_outcomes`` so the operator can see the closed loop is
|
|
1037
1035
|
actually flowing: recall → enqueue → persist → finalize.
|
|
1038
1036
|
"""
|
|
1039
|
-
import sqlite3
|
|
1040
1037
|
try:
|
|
1041
1038
|
from superlocalmemory.learning.outcome_queue import (
|
|
1042
1039
|
get_counters, queue_size,
|
|
@@ -1050,7 +1047,7 @@ def _compute_outcome_queue_stats(profile_id: str) -> dict:
|
|
|
1050
1047
|
pending_now = 0
|
|
1051
1048
|
if db.exists():
|
|
1052
1049
|
try:
|
|
1053
|
-
conn =
|
|
1050
|
+
conn = ReadConnectionFactory(db).open()
|
|
1054
1051
|
try:
|
|
1055
1052
|
row = conn.execute(
|
|
1056
1053
|
"SELECT COUNT(*) FROM pending_outcomes "
|
|
@@ -1089,7 +1086,7 @@ def _compute_reward_preview(profile_id: str) -> dict:
|
|
|
1089
1086
|
if not db.exists():
|
|
1090
1087
|
return default
|
|
1091
1088
|
try:
|
|
1092
|
-
conn =
|
|
1089
|
+
conn = ReadConnectionFactory(db).open()
|
|
1093
1090
|
try:
|
|
1094
1091
|
row = conn.execute(
|
|
1095
1092
|
"SELECT COUNT(*) AS c, AVG(reward) AS m "
|