superlocalmemory 3.8.2 → 3.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/README.md +3 -2
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/access/rbac.py +68 -76
- package/src/superlocalmemory/cli/commands.py +19 -0
- package/src/superlocalmemory/cli/ingest_cmd.py +11 -1
- package/src/superlocalmemory/cli/main.py +30 -0
- package/src/superlocalmemory/cli/pending_store.py +39 -14
- package/src/superlocalmemory/core/backend_orchestrator.py +93 -0
- package/src/superlocalmemory/core/config.py +78 -0
- package/src/superlocalmemory/core/consolidation_engine.py +79 -73
- package/src/superlocalmemory/core/engine.py +92 -11
- package/src/superlocalmemory/core/fact_consolidator.py +148 -30
- package/src/superlocalmemory/core/graph_pruner.py +436 -39
- package/src/superlocalmemory/core/ingestion_command.py +160 -31
- package/src/superlocalmemory/core/maintenance_scheduler.py +61 -1
- package/src/superlocalmemory/core/recall_pipeline.py +3 -0
- package/src/superlocalmemory/core/registry.py +5 -1
- package/src/superlocalmemory/core/remote_mode.py +3 -1
- package/src/superlocalmemory/core/scale_engine.py +41 -18
- package/src/superlocalmemory/core/store_pipeline.py +18 -4
- package/src/superlocalmemory/encoding/entity_resolver.py +18 -11
- package/src/superlocalmemory/hooks/_outcome_common.py +9 -2
- package/src/superlocalmemory/hooks/adapter_base.py +58 -44
- package/src/superlocalmemory/hooks/ide_connector.py +26 -8
- package/src/superlocalmemory/hooks/portable_kit.py +105 -9
- package/src/superlocalmemory/hooks/prewarm_auth.py +21 -2
- package/src/superlocalmemory/infra/auth_middleware.py +3 -1
- package/src/superlocalmemory/infra/cloud_backup.py +26 -27
- package/src/superlocalmemory/infra/event_bus.py +250 -88
- package/src/superlocalmemory/learning/consolidation_cycle.py +33 -16
- package/src/superlocalmemory/learning/entity_compiler.py +148 -132
- package/src/superlocalmemory/learning/memory_merge.py +97 -82
- package/src/superlocalmemory/learning/reward_archive.py +98 -90
- package/src/superlocalmemory/learning/reward_boost.py +40 -30
- package/src/superlocalmemory/mcp/http_transport.py +335 -3
- package/src/superlocalmemory/retrieval/engine.py +7 -1
- package/src/superlocalmemory/retrieval/entity_channel.py +25 -1
- package/src/superlocalmemory/retrieval/reranker.py +98 -15
- package/src/superlocalmemory/retrieval/spreading_activation.py +20 -12
- package/src/superlocalmemory/retrieval/vector_store.py +84 -69
- package/src/superlocalmemory/server/loopback.py +91 -0
- package/src/superlocalmemory/server/origin.py +9 -4
- package/src/superlocalmemory/server/routes/backup.py +6 -2
- package/src/superlocalmemory/server/routes/behavioral.py +6 -12
- package/src/superlocalmemory/server/routes/compliance.py +20 -23
- package/src/superlocalmemory/server/routes/config_api.py +83 -0
- package/src/superlocalmemory/server/routes/helpers.py +24 -13
- package/src/superlocalmemory/server/routes/memories.py +139 -91
- package/src/superlocalmemory/server/routes/mesh.py +7 -2
- package/src/superlocalmemory/server/routes/profiles.py +20 -21
- package/src/superlocalmemory/server/routes/rbac.py +0 -1
- package/src/superlocalmemory/server/routes/tiers.py +42 -30
- package/src/superlocalmemory/server/routes/v3_api.py +67 -77
- package/src/superlocalmemory/server/unified_daemon.py +283 -39
- package/src/superlocalmemory/server/write_identity.py +22 -4
- package/src/superlocalmemory/storage/database.py +109 -19
- package/src/superlocalmemory/storage/deferred_writes.py +153 -0
- package/src/superlocalmemory/storage/embedding_migrator.py +19 -0
- package/src/superlocalmemory/storage/memory_write.py +119 -0
- package/src/superlocalmemory/storage/migration_runner.py +7 -0
- package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +113 -78
- package/src/superlocalmemory/storage/migrations/M031_dead_letter_operations.py +80 -0
- package/src/superlocalmemory/storage/write_lock.py +88 -0
- package/src/superlocalmemory/ui/js/core.js +6 -1
|
@@ -23,6 +23,8 @@ from typing import Any
|
|
|
23
23
|
|
|
24
24
|
import numpy as np
|
|
25
25
|
|
|
26
|
+
from superlocalmemory.storage.write_lock import get_write_lock
|
|
27
|
+
|
|
26
28
|
logger = logging.getLogger(__name__)
|
|
27
29
|
|
|
28
30
|
|
|
@@ -182,51 +184,62 @@ class VectorStore:
|
|
|
182
184
|
)
|
|
183
185
|
return False
|
|
184
186
|
|
|
187
|
+
# Embedding bytes computed OUTSIDE the lock — serialisation must not
|
|
188
|
+
# cover slow computation, only the sqlite3 write transaction itself.
|
|
185
189
|
vec_bytes = self._serialize_f32(embedding)
|
|
186
190
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
conn.execute(
|
|
201
|
-
"
|
|
202
|
-
"WHERE
|
|
203
|
-
(
|
|
204
|
-
)
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
191
|
+
# Acquire the process-level write lock for this db file BEFORE opening
|
|
192
|
+
# the sqlite3 connection. This is the OUTERMOST lock (see write_lock.py
|
|
193
|
+
# ordering rule). self._lock is INNER and acquired inside. The write
|
|
194
|
+
# lock is the same RLock that DatabaseManager._lock references for this
|
|
195
|
+
# db_path, so the self-heal backfill pattern
|
|
196
|
+
# with db._lock: vs.upsert()
|
|
197
|
+
# simply re-enters the RLock (same thread — always safe).
|
|
198
|
+
_wl = get_write_lock(self._db_path)
|
|
199
|
+
with _wl: # OUTER: serialises all memory.db writers
|
|
200
|
+
with self._lock: # INNER: VectorStore per-instance state
|
|
201
|
+
try:
|
|
202
|
+
conn = self._connect()
|
|
203
|
+
# Check if fact_id already exists in metadata
|
|
204
|
+
row = conn.execute(
|
|
205
|
+
"SELECT vec_rowid FROM embedding_metadata "
|
|
206
|
+
"WHERE fact_id = ?",
|
|
207
|
+
(fact_id,),
|
|
208
|
+
).fetchone()
|
|
209
|
+
|
|
210
|
+
if row is not None:
|
|
211
|
+
# UPDATE existing
|
|
212
|
+
rowid = row["vec_rowid"]
|
|
213
|
+
conn.execute(
|
|
214
|
+
"UPDATE fact_embeddings SET embedding = ? "
|
|
215
|
+
"WHERE rowid = ?",
|
|
216
|
+
(vec_bytes, rowid),
|
|
217
|
+
)
|
|
218
|
+
else:
|
|
219
|
+
# INSERT new
|
|
220
|
+
conn.execute(
|
|
221
|
+
"INSERT INTO fact_embeddings(profile_id, embedding) "
|
|
222
|
+
"VALUES (?, ?)",
|
|
223
|
+
(profile_id, vec_bytes),
|
|
224
|
+
)
|
|
225
|
+
rowid = conn.execute(
|
|
226
|
+
"SELECT last_insert_rowid()"
|
|
227
|
+
).fetchone()[0]
|
|
228
|
+
conn.execute(
|
|
229
|
+
"INSERT INTO embedding_metadata "
|
|
230
|
+
"(vec_rowid, fact_id, profile_id, model_name, dimension) "
|
|
231
|
+
"VALUES (?, ?, ?, ?, ?)",
|
|
232
|
+
(rowid, fact_id, profile_id,
|
|
233
|
+
model_name or self._config.model_name,
|
|
234
|
+
self._config.dimension),
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
conn.commit()
|
|
238
|
+
conn.close()
|
|
239
|
+
return True
|
|
240
|
+
except Exception as exc:
|
|
241
|
+
logger.debug("upsert failed for fact_id=%s: %s", fact_id, exc)
|
|
242
|
+
return False
|
|
230
243
|
|
|
231
244
|
def search(
|
|
232
245
|
self,
|
|
@@ -302,41 +315,43 @@ class VectorStore:
|
|
|
302
315
|
def delete(self, fact_id: str) -> bool:
|
|
303
316
|
"""Remove a vector from vec0 and metadata.
|
|
304
317
|
|
|
305
|
-
Thread-safe: acquires self._lock.
|
|
318
|
+
Thread-safe: acquires the process-level write lock then self._lock.
|
|
306
319
|
Returns True if deleted, False if not found or error.
|
|
307
320
|
"""
|
|
308
321
|
if not self._available:
|
|
309
322
|
return False
|
|
310
323
|
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
324
|
+
_wl = get_write_lock(self._db_path)
|
|
325
|
+
with _wl: # OUTER: process-level write serialisation
|
|
326
|
+
with self._lock: # INNER: VectorStore per-instance state
|
|
327
|
+
try:
|
|
328
|
+
conn = self._connect()
|
|
329
|
+
row = conn.execute(
|
|
330
|
+
"SELECT vec_rowid FROM embedding_metadata "
|
|
331
|
+
"WHERE fact_id = ?",
|
|
332
|
+
(fact_id,),
|
|
333
|
+
).fetchone()
|
|
334
|
+
|
|
335
|
+
if row is None:
|
|
336
|
+
conn.close()
|
|
337
|
+
return False
|
|
319
338
|
|
|
320
|
-
|
|
339
|
+
rowid = row["vec_rowid"]
|
|
340
|
+
conn.execute(
|
|
341
|
+
"DELETE FROM fact_embeddings WHERE rowid = ?",
|
|
342
|
+
(rowid,),
|
|
343
|
+
)
|
|
344
|
+
conn.execute(
|
|
345
|
+
"DELETE FROM embedding_metadata WHERE vec_rowid = ?",
|
|
346
|
+
(rowid,),
|
|
347
|
+
)
|
|
348
|
+
conn.commit()
|
|
321
349
|
conn.close()
|
|
350
|
+
return True
|
|
351
|
+
except Exception as exc:
|
|
352
|
+
logger.debug("delete failed for fact_id=%s: %s", fact_id, exc)
|
|
322
353
|
return False
|
|
323
354
|
|
|
324
|
-
rowid = row["vec_rowid"]
|
|
325
|
-
conn.execute(
|
|
326
|
-
"DELETE FROM fact_embeddings WHERE rowid = ?",
|
|
327
|
-
(rowid,),
|
|
328
|
-
)
|
|
329
|
-
conn.execute(
|
|
330
|
-
"DELETE FROM embedding_metadata WHERE vec_rowid = ?",
|
|
331
|
-
(rowid,),
|
|
332
|
-
)
|
|
333
|
-
conn.commit()
|
|
334
|
-
conn.close()
|
|
335
|
-
return True
|
|
336
|
-
except Exception as exc:
|
|
337
|
-
logger.debug("delete failed for fact_id=%s: %s", fact_id, exc)
|
|
338
|
-
return False
|
|
339
|
-
|
|
340
355
|
def count(self, profile_id: str | None = None) -> int:
|
|
341
356
|
"""Count vectors in the store.
|
|
342
357
|
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
"""Centralized loopback-address predicate for the SLM server.
|
|
5
|
+
|
|
6
|
+
Replaces every ``frozenset({"127.0.0.1", "::1", "localhost"})`` auth check
|
|
7
|
+
with a semantic helper that correctly handles IPv4-mapped IPv6 addresses
|
|
8
|
+
(``::ffff:127.0.0.1``) — the root cause of issue #90.
|
|
9
|
+
|
|
10
|
+
Issue #90 root cause
|
|
11
|
+
--------------------
|
|
12
|
+
When the daemon is started with ``SLM_DAEMON_HOST=0.0.0.0`` on a dual-stack
|
|
13
|
+
Linux host (common in LXC/Docker containers), the OS creates an IPv6 socket
|
|
14
|
+
that accepts IPv4 connections via the IPv4-mapped IPv6 address mechanism
|
|
15
|
+
(RFC 4291 §2.5.5.2). A client connecting to ``localhost`` on such a host
|
|
16
|
+
may have its peer address reported as ``::ffff:127.0.0.1`` by
|
|
17
|
+
uvicorn/Starlette. The literal set ``("127.0.0.1", "::1", "localhost")``
|
|
18
|
+
does not include this form, causing a spurious 403 for install-token and
|
|
19
|
+
uncredentialed-loopback callers.
|
|
20
|
+
|
|
21
|
+
``ipaddress.ip_address("::ffff:127.0.0.1").is_loopback`` already returns
|
|
22
|
+
``True`` in CPython. This module uses that fact.
|
|
23
|
+
|
|
24
|
+
SECURITY INVARIANTS (non-negotiable):
|
|
25
|
+
- Empty/None host → False. SEC-L-02: a missing peer is never trusted.
|
|
26
|
+
- No proxy header trust. Callers MUST pass ``request.client.host`` only,
|
|
27
|
+
never X-Forwarded-For or any other spoofable header.
|
|
28
|
+
- All 127.0.0.0/8 is loopback per RFC 5735 (includes 127.0.0.2, etc.).
|
|
29
|
+
- ``"localhost"`` is accepted as a hostname alias. Callers that need the
|
|
30
|
+
stricter no-hostname check (``prewarm_auth``) keep their own predicate.
|
|
31
|
+
- ``"testclient"`` is NOT in the loopback set — that bypass must be wired
|
|
32
|
+
explicitly alongside ``_TEST_ISOLATION_ALLOWED`` so it cannot appear in
|
|
33
|
+
production paths.
|
|
34
|
+
- ``"0.0.0.0"`` is NOT loopback — it is a bind address, not a peer address.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
import ipaddress as _ipa
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# CRIT-2 guard: detect a future CPython regression where IPv4-mapped loopback
|
|
43
|
+
# is no longer reported as .is_loopback. Fails at daemon startup (import time),
|
|
44
|
+
# not silently at request time.
|
|
45
|
+
assert _ipa.ip_address("::ffff:127.0.0.1").is_loopback, (
|
|
46
|
+
"Python ipaddress regression: ::ffff:127.0.0.1 no longer reports as "
|
|
47
|
+
"loopback. Update superlocalmemory/server/loopback.py. (Issue #90)"
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def is_loopback(host: str) -> bool:
|
|
52
|
+
"""Return ``True`` iff ``host`` is a loopback address in any standard form.
|
|
53
|
+
|
|
54
|
+
Handles:
|
|
55
|
+
* ``"127.0.0.1"`` and the full 127.0.0.0/8 range (RFC 5735).
|
|
56
|
+
* ``"::1"`` (IPv6 loopback).
|
|
57
|
+
* ``"::ffff:127.0.0.1"`` and all ``::ffff:127.x.x.x`` (IPv4-mapped IPv6,
|
|
58
|
+
fixes issue #90).
|
|
59
|
+
* ``"localhost"`` (hostname alias, case-insensitive).
|
|
60
|
+
|
|
61
|
+
Returns ``False`` for:
|
|
62
|
+
* Empty string or ``None`` (SEC-L-02: missing peer is never trusted).
|
|
63
|
+
* Any non-loopback IP (192.168.x.x, 10.x.x.x, public IPs, etc.).
|
|
64
|
+
* ``"::ffff:192.168.x.x"`` and other IPv4-mapped non-loopback addresses.
|
|
65
|
+
* ``"testclient"`` — callers that need the test-client exemption must
|
|
66
|
+
wire it explicitly alongside ``_TEST_ISOLATION_ALLOWED``.
|
|
67
|
+
* ``"0.0.0.0"`` — bind address, not a peer address.
|
|
68
|
+
|
|
69
|
+
Args:
|
|
70
|
+
host: The ``request.client.host`` string from an incoming HTTP
|
|
71
|
+
request. Must be the TCP-observed peer address only.
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
``True`` if the host is any recognised loopback form; ``False``
|
|
75
|
+
otherwise.
|
|
76
|
+
"""
|
|
77
|
+
if not isinstance(host, str) or not host:
|
|
78
|
+
return False # SEC-L-02
|
|
79
|
+
if host.lower() == "localhost":
|
|
80
|
+
return True
|
|
81
|
+
try:
|
|
82
|
+
ip = _ipa.ip_address(host)
|
|
83
|
+
except ValueError:
|
|
84
|
+
return False
|
|
85
|
+
# Python's .is_loopback already handles 127.0.0.0/8, ::1, and
|
|
86
|
+
# IPv4-mapped loopback (::ffff:127.x.x.x). No manual ipv4_mapped
|
|
87
|
+
# normalization needed — the stdlib does the right thing.
|
|
88
|
+
return ip.is_loopback
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
__all__ = ("is_loopback",)
|
|
@@ -4,12 +4,17 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
from urllib.parse import urlsplit
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
_LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"})
|
|
7
|
+
from superlocalmemory.server.loopback import is_loopback as _is_loopback_host
|
|
9
8
|
|
|
10
9
|
|
|
11
10
|
def origin_is_loopback(origin: str) -> bool:
|
|
12
|
-
"""Return whether an Origin is absent or an exact HTTP(S) loopback URL.
|
|
11
|
+
"""Return whether an Origin is absent or an exact HTTP(S) loopback URL.
|
|
12
|
+
|
|
13
|
+
Uses the centralized is_loopback helper so that IPv4-mapped loopback
|
|
14
|
+
addresses (::ffff:127.x.x.x) are accepted correctly (issue #90).
|
|
15
|
+
Note: browsers normalize Origin hostnames; the ::ffff: form would only
|
|
16
|
+
appear in synthetic requests. The helper is used for defense-in-depth.
|
|
17
|
+
"""
|
|
13
18
|
if not origin:
|
|
14
19
|
return True
|
|
15
20
|
try:
|
|
@@ -21,7 +26,7 @@ def origin_is_loopback(origin: str) -> bool:
|
|
|
21
26
|
return (
|
|
22
27
|
parsed.scheme in {"http", "https"}
|
|
23
28
|
and parsed.hostname is not None
|
|
24
|
-
and parsed.hostname.lower()
|
|
29
|
+
and _is_loopback_host(parsed.hostname.lower())
|
|
25
30
|
and parsed.username is None
|
|
26
31
|
and parsed.password is None
|
|
27
32
|
and parsed.path in {"", "/"}
|
|
@@ -87,9 +87,13 @@ def _require_oauth_start(request: Request) -> None:
|
|
|
87
87
|
detail="OAuth initiation requires the local dashboard origin.",
|
|
88
88
|
)
|
|
89
89
|
|
|
90
|
+
from superlocalmemory.server.loopback import is_loopback as _is_loopback_host
|
|
91
|
+
|
|
90
92
|
host = request.client.host if request.client else ""
|
|
91
|
-
|
|
92
|
-
|
|
93
|
+
# "testclient" is included for in-process test compatibility (preserved
|
|
94
|
+
# from original behaviour; was in the original frozenset).
|
|
95
|
+
_from_loopback = _is_loopback_host(host) or host == "testclient"
|
|
96
|
+
if not _from_loopback and principal.get("kind") != "user":
|
|
93
97
|
raise HTTPException(
|
|
94
98
|
status_code=403,
|
|
95
99
|
detail=(
|
|
@@ -17,6 +17,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
|
|
17
17
|
from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator
|
|
18
18
|
|
|
19
19
|
from .helpers import MEMORY_DIR, get_active_profile
|
|
20
|
+
from superlocalmemory.storage.memory_write import memory_write
|
|
20
21
|
|
|
21
22
|
logger = logging.getLogger("superlocalmemory.routes.behavioral")
|
|
22
23
|
router = APIRouter()
|
|
@@ -385,10 +386,9 @@ def report_outcome(request: Request, data: ReportOutcomeRequest):
|
|
|
385
386
|
"action_type": action_type,
|
|
386
387
|
"source": "dashboard_report_outcome",
|
|
387
388
|
}
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
conn.execute("BEGIN IMMEDIATE")
|
|
389
|
+
# memory_write acquires the process write lock (serialises in-process)
|
|
390
|
+
# and sets PRAGMA busy_timeout (cross-process writers wait, not error).
|
|
391
|
+
with memory_write(memory_db_path) as conn:
|
|
392
392
|
_validate_profile_fact_ids(
|
|
393
393
|
conn,
|
|
394
394
|
profile_id=profile,
|
|
@@ -407,9 +407,6 @@ def report_outcome(request: Request, data: ReportOutcomeRequest):
|
|
|
407
407
|
now_iso, reward, now_iso,
|
|
408
408
|
),
|
|
409
409
|
)
|
|
410
|
-
conn.commit()
|
|
411
|
-
finally:
|
|
412
|
-
conn.close()
|
|
413
410
|
|
|
414
411
|
try:
|
|
415
412
|
from superlocalmemory.learning.source_quality import (
|
|
@@ -589,8 +586,8 @@ def log_tool_event_api(request: Request, data: dict):
|
|
|
589
586
|
input_summary = str(input_summary)[:500] if input_summary else ""
|
|
590
587
|
output_summary = str(output_summary)[:500] if output_summary else ""
|
|
591
588
|
|
|
592
|
-
|
|
593
|
-
|
|
589
|
+
# memory_write: process write lock + busy_timeout for SQLITE_BUSY safety.
|
|
590
|
+
with memory_write(MEMORY_DIR / "memory.db") as conn:
|
|
594
591
|
conn.execute(
|
|
595
592
|
"INSERT INTO tool_events "
|
|
596
593
|
"(session_id, profile_id, project_path, tool_name, event_type, "
|
|
@@ -599,9 +596,6 @@ def log_tool_event_api(request: Request, data: dict):
|
|
|
599
596
|
(session_id, profile, project_path, tool_name, event_type,
|
|
600
597
|
input_summary, output_summary, now),
|
|
601
598
|
)
|
|
602
|
-
conn.commit()
|
|
603
|
-
finally:
|
|
604
|
-
conn.close()
|
|
605
599
|
return {"ok": True}
|
|
606
600
|
except Exception:
|
|
607
601
|
logger.exception("behavioral route error")
|
|
@@ -10,7 +10,6 @@ Uses V3 compliance modules: ABACEngine, AuditChain, RetentionEngine.
|
|
|
10
10
|
"""
|
|
11
11
|
import json
|
|
12
12
|
import logging
|
|
13
|
-
import sqlite3
|
|
14
13
|
from typing import Optional
|
|
15
14
|
|
|
16
15
|
from fastapi import APIRouter, Query, Request
|
|
@@ -18,6 +17,7 @@ from fastapi.responses import JSONResponse
|
|
|
18
17
|
|
|
19
18
|
from .helpers import get_active_profile, get_engine_lazy, MEMORY_DIR, DB_PATH
|
|
20
19
|
from superlocalmemory.server.route_mutations import authorize_route_mutation
|
|
20
|
+
from superlocalmemory.storage.memory_write import memory_write
|
|
21
21
|
|
|
22
22
|
logger = logging.getLogger("superlocalmemory.routes.compliance")
|
|
23
23
|
router = APIRouter()
|
|
@@ -61,13 +61,14 @@ async def compliance_status():
|
|
|
61
61
|
except Exception as exc:
|
|
62
62
|
logger.debug("audit chain: %s", exc)
|
|
63
63
|
|
|
64
|
-
# Retention policies (scoped to the active profile)
|
|
64
|
+
# Retention policies (scoped to the active profile).
|
|
65
|
+
# RetentionEngine.__init__ runs DDL (CREATE TABLE IF NOT EXISTS) so even
|
|
66
|
+
# list_rules() is a write at the constructor level — use memory_write().
|
|
65
67
|
retention_policies = []
|
|
66
68
|
try:
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
conn.close()
|
|
69
|
+
with memory_write(DB_PATH) as conn:
|
|
70
|
+
engine = RetentionEngine(conn)
|
|
71
|
+
retention_policies = engine.list_rules(profile)
|
|
71
72
|
except Exception as exc:
|
|
72
73
|
logger.debug("retention engine: %s", exc)
|
|
73
74
|
|
|
@@ -163,15 +164,13 @@ async def create_retention_policy(data: dict):
|
|
|
163
164
|
|
|
164
165
|
try:
|
|
165
166
|
profile = get_active_profile()
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
)
|
|
174
|
-
conn.close()
|
|
167
|
+
with memory_write(DB_PATH) as conn:
|
|
168
|
+
engine = RetentionEngine(conn)
|
|
169
|
+
rule_id = engine.create_rule(
|
|
170
|
+
name=name, framework=framework,
|
|
171
|
+
retention_days=retention_days, action=action,
|
|
172
|
+
applies_to=applies_to, profile_id=profile,
|
|
173
|
+
)
|
|
175
174
|
|
|
176
175
|
return {
|
|
177
176
|
"success": True, "rule_id": rule_id,
|
|
@@ -190,10 +189,9 @@ async def delete_retention_policy(name: str = Query(...)):
|
|
|
190
189
|
return {"success": False, "error": "Compliance engine not available"}
|
|
191
190
|
try:
|
|
192
191
|
profile = get_active_profile()
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
conn.close()
|
|
192
|
+
with memory_write(DB_PATH) as conn:
|
|
193
|
+
engine = RetentionEngine(conn)
|
|
194
|
+
removed = engine.delete_rule(profile, name)
|
|
197
195
|
if not removed:
|
|
198
196
|
return {"success": False, "error": f"Policy '{name}' not found"}
|
|
199
197
|
return {"success": True, "active_profile": profile,
|
|
@@ -214,10 +212,9 @@ async def enforce_retention():
|
|
|
214
212
|
return {"success": False, "error": "Compliance engine not available"}
|
|
215
213
|
try:
|
|
216
214
|
profile = get_active_profile()
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
conn.close()
|
|
215
|
+
with memory_write(DB_PATH) as conn:
|
|
216
|
+
engine = RetentionEngine(conn)
|
|
217
|
+
result = engine.enforce(profile)
|
|
221
218
|
return {"success": True, **result}
|
|
222
219
|
except Exception:
|
|
223
220
|
logger.exception("enforce_retention error")
|
|
@@ -436,3 +436,86 @@ def put_forgetting_config(request: Request, body: ForgettingConfigUpdate):
|
|
|
436
436
|
except Exception:
|
|
437
437
|
logger.exception("put_forgetting_config failed")
|
|
438
438
|
return JSONResponse({"error": "Internal server error"}, status_code=500)
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
# ---------------------------------------------------------------------------
|
|
442
|
+
# Graph Pruning config (v3.8.4-G — GitHub #84)
|
|
443
|
+
# Dedicated endpoint so the forgetting config partial-update contract stays
|
|
444
|
+
# clean. Callers that only want graph knobs do not need to know about the
|
|
445
|
+
# Ebbinghaus curve, and vice-versa.
|
|
446
|
+
# ---------------------------------------------------------------------------
|
|
447
|
+
|
|
448
|
+
_GRAPH_PRUNING_DEFAULTS: dict = {
|
|
449
|
+
"max_degree_per_node": 100,
|
|
450
|
+
"min_edge_weight": 0.0,
|
|
451
|
+
"enabled": True,
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
class GraphPruningConfigUpdate(BaseModel):
|
|
456
|
+
"""Partial update model for graph pruning configuration.
|
|
457
|
+
|
|
458
|
+
All fields are optional so a PUT with only ``max_degree_per_node`` does
|
|
459
|
+
NOT reset ``min_edge_weight`` back to the default. Existing values are
|
|
460
|
+
preserved; only the supplied fields are overwritten.
|
|
461
|
+
"""
|
|
462
|
+
|
|
463
|
+
model_config = ConfigDict(extra="forbid")
|
|
464
|
+
|
|
465
|
+
max_degree_per_node: Optional[int] = Field(None, ge=1)
|
|
466
|
+
min_edge_weight: Optional[float] = Field(None, ge=0.0, le=1.0)
|
|
467
|
+
enabled: Optional[StrictBool] = None
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
# ---------------------------------------------------------------------------
|
|
471
|
+
# GET /api/v3/graph/config
|
|
472
|
+
# ---------------------------------------------------------------------------
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
@router.get("/graph/config")
|
|
476
|
+
def get_graph_config():
|
|
477
|
+
"""Return current graph thinning configuration.
|
|
478
|
+
|
|
479
|
+
Returns all three fields with their defaults when config.json has no
|
|
480
|
+
``graph_pruning`` section (old installations).
|
|
481
|
+
"""
|
|
482
|
+
try:
|
|
483
|
+
data = _read_config()
|
|
484
|
+
stored = data.get("graph_pruning", {})
|
|
485
|
+
result = {**_GRAPH_PRUNING_DEFAULTS, **stored}
|
|
486
|
+
# Return only known fields
|
|
487
|
+
return {k: result[k] for k in _GRAPH_PRUNING_DEFAULTS}
|
|
488
|
+
except Exception:
|
|
489
|
+
logger.exception("get_graph_config failed")
|
|
490
|
+
return JSONResponse({"error": "Internal server error"}, status_code=500)
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
# ---------------------------------------------------------------------------
|
|
494
|
+
# PUT /api/v3/graph/config
|
|
495
|
+
# ---------------------------------------------------------------------------
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
@router.put("/graph/config")
|
|
499
|
+
def put_graph_config(request: Request, body: GraphPruningConfigUpdate):
|
|
500
|
+
"""Update graph thinning configuration.
|
|
501
|
+
|
|
502
|
+
Only supplied fields are written; all other graph pruning fields are
|
|
503
|
+
preserved. Changes take effect at the next maintenance cycle without
|
|
504
|
+
requiring a daemon restart (the scheduler reads graph_pruning live from
|
|
505
|
+
the config object, which the daemon reloads from disk on each cycle).
|
|
506
|
+
"""
|
|
507
|
+
_require_admin(request)
|
|
508
|
+
try:
|
|
509
|
+
updates = body.model_dump(exclude_none=True)
|
|
510
|
+
|
|
511
|
+
def mutate(data: dict) -> None:
|
|
512
|
+
stored = data.get("graph_pruning", {})
|
|
513
|
+
merged = {**_GRAPH_PRUNING_DEFAULTS, **stored, **updates}
|
|
514
|
+
data["graph_pruning"] = merged
|
|
515
|
+
|
|
516
|
+
data = _update_config(mutate)
|
|
517
|
+
merged = data["graph_pruning"]
|
|
518
|
+
return {k: merged[k] for k in _GRAPH_PRUNING_DEFAULTS}
|
|
519
|
+
except Exception:
|
|
520
|
+
logger.exception("put_graph_config failed")
|
|
521
|
+
return JSONResponse({"error": "Internal server error"}, status_code=500)
|
|
@@ -22,6 +22,7 @@ from fastapi import HTTPException, Request
|
|
|
22
22
|
from pydantic import BaseModel, Field
|
|
23
23
|
|
|
24
24
|
from superlocalmemory.infra.data_root import DynamicStatePath, canonical_data_root
|
|
25
|
+
from superlocalmemory.storage.memory_write import memory_write
|
|
25
26
|
|
|
26
27
|
|
|
27
28
|
_engine_logger = logging.getLogger("superlocalmemory.engine")
|
|
@@ -217,13 +218,23 @@ def log_mode_change(
|
|
|
217
218
|
|
|
218
219
|
|
|
219
220
|
def get_db_connection() -> sqlite3.Connection:
|
|
220
|
-
"""Get database connection.
|
|
221
|
+
"""Get database connection with busy_timeout set.
|
|
222
|
+
|
|
223
|
+
Used for READ-heavy callers. Writers should use ``memory_write(DB_PATH)``
|
|
224
|
+
instead to also acquire the process write lock. This connection still gets
|
|
225
|
+
``PRAGMA busy_timeout`` so a cross-process writer (hook / CLI) doing a short
|
|
226
|
+
write does not cause an immediate SQLITE_BUSY here.
|
|
227
|
+
"""
|
|
221
228
|
if not DB_PATH.exists():
|
|
222
229
|
raise HTTPException(
|
|
223
230
|
status_code=500,
|
|
224
231
|
detail="Memory database not found. Run 'slm init' to initialize."
|
|
225
232
|
)
|
|
226
|
-
|
|
233
|
+
import os as _os
|
|
234
|
+
_ms = max(0, int(_os.environ.get("SLM_DB_BUSY_TIMEOUT_MS", "10000")))
|
|
235
|
+
conn = sqlite3.connect(str(DB_PATH), timeout=_ms / 1000.0)
|
|
236
|
+
conn.execute(f"PRAGMA busy_timeout={_ms}")
|
|
237
|
+
return conn
|
|
227
238
|
|
|
228
239
|
|
|
229
240
|
def dict_factory(cursor: sqlite3.Cursor, row: tuple) -> dict:
|
|
@@ -261,20 +272,22 @@ def validate_profile_name(name: str) -> bool:
|
|
|
261
272
|
|
|
262
273
|
|
|
263
274
|
def ensure_profile_in_db(name: str, description: str = "") -> None:
|
|
264
|
-
"""Ensure a profile row exists in SQLite (idempotent).
|
|
275
|
+
"""Ensure a profile row exists in SQLite (idempotent).
|
|
276
|
+
|
|
277
|
+
Hot path: called on every authenticated request. Uses ``memory_write()``
|
|
278
|
+
so in-process writers serialise through the write lock and cross-process
|
|
279
|
+
writers (hooks / CLI) wait via PRAGMA busy_timeout instead of getting
|
|
280
|
+
SQLITE_BUSY.
|
|
281
|
+
"""
|
|
265
282
|
if not DB_PATH.exists():
|
|
266
283
|
return
|
|
267
|
-
|
|
268
|
-
try:
|
|
284
|
+
with memory_write(DB_PATH) as conn:
|
|
269
285
|
conn.execute("PRAGMA foreign_keys=ON")
|
|
270
286
|
conn.execute(
|
|
271
287
|
"INSERT OR IGNORE INTO profiles (profile_id, name, description) "
|
|
272
288
|
"VALUES (?, ?, ?)",
|
|
273
289
|
(name, name, description or f"Memory profile: {name}"),
|
|
274
290
|
)
|
|
275
|
-
conn.commit()
|
|
276
|
-
finally:
|
|
277
|
-
conn.close()
|
|
278
291
|
|
|
279
292
|
|
|
280
293
|
def ensure_profile_in_json(name: str, description: str = "") -> None:
|
|
@@ -347,11 +360,12 @@ def delete_profile_from_db(name: str) -> None:
|
|
|
347
360
|
rbac_memberships has no FK to profiles, so CASCADE does not remove role
|
|
348
361
|
grants — they would otherwise survive deletion and silently re-activate if
|
|
349
362
|
a profile of the same name is later recreated. Remove them explicitly.
|
|
363
|
+
Uses ``memory_write()`` so the multi-statement DELETE is atomic and
|
|
364
|
+
serialised against other in-process writers.
|
|
350
365
|
"""
|
|
351
366
|
if not DB_PATH.exists():
|
|
352
367
|
return
|
|
353
|
-
|
|
354
|
-
try:
|
|
368
|
+
with memory_write(DB_PATH) as conn:
|
|
355
369
|
conn.execute("PRAGMA foreign_keys=ON")
|
|
356
370
|
# Purge role grants for this workspace (no FK CASCADE covers these).
|
|
357
371
|
for tbl in ("rbac_memberships",):
|
|
@@ -360,9 +374,6 @@ def delete_profile_from_db(name: str) -> None:
|
|
|
360
374
|
except sqlite3.OperationalError:
|
|
361
375
|
pass # table may not exist on older installs
|
|
362
376
|
conn.execute("DELETE FROM profiles WHERE profile_id = ?", (name,))
|
|
363
|
-
conn.commit()
|
|
364
|
-
finally:
|
|
365
|
-
conn.close()
|
|
366
377
|
|
|
367
378
|
|
|
368
379
|
def _get_db_profiles() -> list[dict]:
|