superlocalmemory 3.8.13 → 3.8.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +3 -3
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/core/engine_wiring.py +4 -4
- package/src/superlocalmemory/encoding/scene_builder.py +115 -13
- package/src/superlocalmemory/storage/migration_runner.py +7 -0
- package/src/superlocalmemory/storage/migrations/M034_scene_fact_members.py +127 -0
- package/src/superlocalmemory/storage/migrations/__init__.py +2 -0
- package/src/superlocalmemory/storage/schema.py +59 -0
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,22 @@ All notable changes to SuperLocalMemory V3 will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [3.8.14] - 2026-08-05 — Bounded scene assignment at mature scale
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
- Scene assignment no longer expands and compares every historical scene for
|
|
12
|
+
every materialized fact. v3.8.13 crossed a severe CPU and latency cliff on
|
|
13
|
+
mature databases because both candidate selection and anchor-embedding
|
|
14
|
+
loading scaled with the full scene population.
|
|
15
|
+
- A forward-only M034 migration adds an indexed, trigger-maintained
|
|
16
|
+
`scene_fact_members` projection while retaining `fact_ids_json` for backward
|
|
17
|
+
compatibility. The existing fact-vector index now selects semantically near
|
|
18
|
+
scenes, with a bounded live-recency fallback when vector search is
|
|
19
|
+
unavailable. Profile isolation and deleted-fact handling remain enforced.
|
|
20
|
+
- On the 11,519-scene production-data repro used for this repair, the complete
|
|
21
|
+
scene-assignment path measured 214 ms median and 231 ms maximum across ten
|
|
22
|
+
post-warmup runs. The migrated copy passes SQLite `quick_check`.
|
|
23
|
+
|
|
8
24
|
## [3.8.13] - 2026-08-03 — Stale-process detection
|
|
9
25
|
|
|
10
26
|
### Fixed
|
package/README.md
CHANGED
|
@@ -5,15 +5,15 @@
|
|
|
5
5
|
</picture>
|
|
6
6
|
</p>
|
|
7
7
|
|
|
8
|
-
<h1 align="center">SuperLocalMemory V3.8.
|
|
8
|
+
<h1 align="center">SuperLocalMemory V3.8.14</h1>
|
|
9
9
|
<p align="center"><strong>Enterprise-grade, local-first memory for AI agents and teams.</strong><br/>
|
|
10
10
|
<em>A persistent, auditable long-term brain for your agents that runs on your own infrastructure — with multi-workspace isolation, role-based access, and GDPR + EU AI Act governance controls built in.</em></p>
|
|
11
|
-
<p align="center"><code>v3.8.
|
|
11
|
+
<p align="center"><code>v3.8.14</code> — one control plane: auditable retrieval · multi-scope memory (personal / shared / global) · Cache · Compress · trusted-peer Mesh · bounded loops — across CLI, MCP, dashboard, the <strong>Claude plugin</strong>, the <strong>Codex add-on</strong>, and documented IDE integrations.<br/>
|
|
12
12
|
Proxy: <code>slm wrap claude</code> · MCP: add <code>slm_compress</code> to your config · Skill: zero-config</p>
|
|
13
13
|
<p align="center"><strong>3 public research preprints</strong> (arXiv + Zenodo archives) · <a href="https://arxiv.org/abs/2603.02240">arXiv:2603.02240</a> · <a href="https://arxiv.org/abs/2603.14588">arXiv:2603.14588</a> · <a href="https://arxiv.org/abs/2604.04514">arXiv:2604.04514</a></p>
|
|
14
14
|
|
|
15
15
|
<p align="center">
|
|
16
|
-
<a href="CHANGELOG.md"><img src="https://img.shields.io/badge/v3.8.
|
|
16
|
+
<a href="CHANGELOG.md"><img src="https://img.shields.io/badge/v3.8.14-Current_Release-2ea44f?style=for-the-badge&logo=checkmarx&logoColor=white" alt="v3.8.14 — Current Release"/></a>
|
|
17
17
|
<a href="https://arxiv.org/abs/2603.14588"><img src="https://img.shields.io/badge/arXiv-2603.14588-b31b1b?style=for-the-badge&logo=arxiv&logoColor=white" alt="arXiv Paper"/></a>
|
|
18
18
|
<a href="#three-surfaces-proxy--mcp-tools--skill"><img src="https://img.shields.io/badge/Proxy_|_MCP_|_Skill-22c55e?style=for-the-badge" alt="Three Surfaces: Proxy, MCP Tools, Skill"/></a>
|
|
19
19
|
<a href="https://pypi.org/project/superlocalmemory/"><img src="https://img.shields.io/pypi/v/superlocalmemory?style=for-the-badge&logo=pypi&logoColor=white" alt="PyPI"/></a>
|
package/package.json
CHANGED
package/plugin/CLAUDE.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
<!-- BEGIN SuperLocalMemory v3.8.
|
|
1
|
+
<!-- BEGIN SuperLocalMemory v3.8.14 -->
|
|
2
2
|
|
|
3
3
|
## SuperLocalMemory (SLM) — Agent Rules
|
|
4
4
|
|
|
@@ -39,6 +39,6 @@ slm-recall · slm-remember · slm-session · slm-status · slm-cache · slm-comp
|
|
|
39
39
|
### Subagents
|
|
40
40
|
slm-memory-advisor (memory decisions, session hygiene, scope/profile guidance) · slm-optimize-advisor (context compression + KV cache) · slm-governance-advisor (scope/roles/compliance/GDPR)
|
|
41
41
|
|
|
42
|
-
<!-- END SuperLocalMemory v3.8.
|
|
42
|
+
<!-- END SuperLocalMemory v3.8.14 -->
|
|
43
43
|
|
|
44
|
-
SuperLocalMemory v3.8.
|
|
44
|
+
SuperLocalMemory v3.8.14 · Qualixar · AGPL-3.0-or-later
|
|
@@ -77,4 +77,4 @@ slm-scope · slm-governance · slm-profile · slm-remember · slm-recall
|
|
|
77
77
|
# What NOT to do
|
|
78
78
|
Never session_init twice; never forget without dry-run preview; never store secrets; never bypass role checks; never claim an erasure succeeded without verifying via recall.
|
|
79
79
|
|
|
80
|
-
SuperLocalMemory v3.8.
|
|
80
|
+
SuperLocalMemory v3.8.14 · Qualixar · AGPL-3.0-or-later
|
|
@@ -46,4 +46,4 @@ slm-recall · slm-remember · slm-session · slm-scope · slm-profile · slm-gov
|
|
|
46
46
|
# What NOT to do
|
|
47
47
|
Never session_init twice; never forget dry_run=False without reporting preview; never dump a whole file into remember; never invent a memory; never claim "saved" without success:true / clean CLI exit; never bypass scope or governance restrictions.
|
|
48
48
|
|
|
49
|
-
SuperLocalMemory v3.8.
|
|
49
|
+
SuperLocalMemory v3.8.14 · Qualixar · AGPL-3.0-or-later
|
|
@@ -41,4 +41,4 @@ slm-compress · slm-cache · slm-status · slm-profile
|
|
|
41
41
|
# What NOT to do
|
|
42
42
|
Never compress code-for-edit/JSON-to-parse/<500 chars; never store secrets/ccr_ids; never let optimize failure block/alter the task; never claim a specific savings %; never carry ccr_ids across profile switches.
|
|
43
43
|
|
|
44
|
-
SuperLocalMemory v3.8.
|
|
44
|
+
SuperLocalMemory v3.8.14 · Qualixar · AGPL-3.0-or-later
|
package/plugin/requirements.txt
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
superlocalmemory==3.8.
|
|
1
|
+
superlocalmemory==3.8.14
|
|
@@ -128,4 +128,4 @@ When the SLM MCP server is unavailable, use these CLI equivalents:
|
|
|
128
128
|
- **slm-optimize-advisor** — context compression and KV cache
|
|
129
129
|
- **slm-governance-advisor** — scope/role compliance, retention policies, GDPR
|
|
130
130
|
|
|
131
|
-
SuperLocalMemory v3.8.
|
|
131
|
+
SuperLocalMemory v3.8.14 · Qualixar · AGPL-3.0-or-later
|
package/pyproject.toml
CHANGED
|
@@ -32,7 +32,7 @@ if "OMP_NUM_THREADS" not in os.environ:
|
|
|
32
32
|
os.environ["OMP_NUM_THREADS"] = "2"
|
|
33
33
|
# ---------------------------------------------------------------------------
|
|
34
34
|
|
|
35
|
-
__version__ = "3.8.
|
|
35
|
+
__version__ = "3.8.14"
|
|
36
36
|
|
|
37
37
|
_REQUIRED_VERSIONS = {
|
|
38
38
|
"sentence_transformers": "5.3.0",
|
|
@@ -264,7 +264,10 @@ def init_encoding(
|
|
|
264
264
|
db, embedder, llm, config.encoding,
|
|
265
265
|
)
|
|
266
266
|
observation_builder = ObservationBuilder(db)
|
|
267
|
-
|
|
267
|
+
# V3.2: VectorStore (Phase 1) -- sqlite-vec KNN. Scene assignment also
|
|
268
|
+
# consumes it, so initialize it before the encoding component is wired.
|
|
269
|
+
vector_store = _init_vector_store(config)
|
|
270
|
+
scene_builder = SceneBuilder(db, embedder, vector_store=vector_store)
|
|
268
271
|
entropy_gate = EntropyGate(
|
|
269
272
|
embedder, config.encoding.entropy_threshold,
|
|
270
273
|
)
|
|
@@ -276,9 +279,6 @@ def init_encoding(
|
|
|
276
279
|
db, config.math.sheaf_contradiction_threshold,
|
|
277
280
|
)
|
|
278
281
|
|
|
279
|
-
# V3.2: VectorStore (Phase 1) -- sqlite-vec KNN
|
|
280
|
-
vector_store = _init_vector_store(config)
|
|
281
|
-
|
|
282
282
|
# V3.2: AccessLog (Phase 1) -- fact access tracking
|
|
283
283
|
access_log = _init_access_log(db)
|
|
284
284
|
|
|
@@ -24,6 +24,7 @@ logger = logging.getLogger(__name__)
|
|
|
24
24
|
|
|
25
25
|
# Similarity threshold for assigning fact to existing scene
|
|
26
26
|
_ASSIGN_THRESHOLD = 0.6
|
|
27
|
+
_MAX_ASSIGNMENT_CANDIDATES = 256
|
|
27
28
|
|
|
28
29
|
|
|
29
30
|
class SceneBuilder:
|
|
@@ -35,9 +36,10 @@ class SceneBuilder:
|
|
|
35
36
|
3. If below threshold: create new scene
|
|
36
37
|
"""
|
|
37
38
|
|
|
38
|
-
def __init__(self, db, embedder=None) -> None:
|
|
39
|
+
def __init__(self, db, embedder=None, vector_store=None) -> None:
|
|
39
40
|
self._db = db
|
|
40
41
|
self._embedder = embedder
|
|
42
|
+
self._vector_store = vector_store
|
|
41
43
|
# Key by scene ID, never theme. Themes are deliberately non-unique,
|
|
42
44
|
# while eligibility and durable anchor membership are scene-specific.
|
|
43
45
|
self._scene_embeddings_cache: dict[str, list[float]] = {}
|
|
@@ -71,11 +73,14 @@ class SceneBuilder:
|
|
|
71
73
|
if fact_emb is None:
|
|
72
74
|
return self._create_scene(new_fact, profile_id)
|
|
73
75
|
|
|
74
|
-
scenes = self.
|
|
76
|
+
scenes = self._get_assignment_scenes(profile_id, fact_emb)
|
|
75
77
|
if not scenes:
|
|
76
78
|
return self._create_scene(new_fact, profile_id)
|
|
77
79
|
|
|
78
|
-
live_scene_embeddings = self._load_live_scene_embeddings(
|
|
80
|
+
live_scene_embeddings = self._load_live_scene_embeddings(
|
|
81
|
+
profile_id,
|
|
82
|
+
tuple(scene.scene_id for scene in scenes),
|
|
83
|
+
)
|
|
79
84
|
live_scene_ids = set(live_scene_embeddings)
|
|
80
85
|
self._scene_embeddings_cache.update({
|
|
81
86
|
scene_id: embedding
|
|
@@ -209,41 +214,138 @@ class SceneBuilder:
|
|
|
209
214
|
)
|
|
210
215
|
return [self._row_to_scene(dict(r)) for r in rows]
|
|
211
216
|
|
|
217
|
+
def _get_assignment_scenes(
|
|
218
|
+
self,
|
|
219
|
+
profile_id: str,
|
|
220
|
+
fact_embedding: list[float],
|
|
221
|
+
) -> list[MemoryScene]:
|
|
222
|
+
"""Load bounded semantic candidates with a recency fallback.
|
|
223
|
+
|
|
224
|
+
Mature stores must not compare every new fact with every historical
|
|
225
|
+
scene. The fact vector index finds nearby members in bounded time; the
|
|
226
|
+
normalized membership projection maps them back to scenes. Recent
|
|
227
|
+
scenes remain a fail-soft fallback for fresh or unavailable indexes.
|
|
228
|
+
Semantic candidates are ordered first so an old relevant scene cannot
|
|
229
|
+
be displaced by the recency cap.
|
|
230
|
+
"""
|
|
231
|
+
recent_rows = self._db.execute(
|
|
232
|
+
"SELECT ms.* FROM memory_scenes AS ms WHERE ms.profile_id = ? "
|
|
233
|
+
"AND EXISTS (SELECT 1 FROM scene_fact_members AS live_member "
|
|
234
|
+
"WHERE live_member.scene_id = ms.scene_id "
|
|
235
|
+
"AND live_member.profile_id = ms.profile_id) "
|
|
236
|
+
"ORDER BY ms.last_updated DESC LIMIT ?",
|
|
237
|
+
(profile_id, _MAX_ASSIGNMENT_CANDIDATES),
|
|
238
|
+
)
|
|
239
|
+
recent = [self._row_to_scene(dict(row)) for row in recent_rows]
|
|
240
|
+
|
|
241
|
+
nearest_fact_ids: list[str] = []
|
|
242
|
+
if self._vector_store is not None:
|
|
243
|
+
try:
|
|
244
|
+
nearest_fact_ids = [
|
|
245
|
+
fact_id
|
|
246
|
+
for fact_id, _score in self._vector_store.search(
|
|
247
|
+
fact_embedding,
|
|
248
|
+
top_k=_MAX_ASSIGNMENT_CANDIDATES,
|
|
249
|
+
profile_id=profile_id,
|
|
250
|
+
)
|
|
251
|
+
]
|
|
252
|
+
except Exception:
|
|
253
|
+
nearest_fact_ids = []
|
|
254
|
+
|
|
255
|
+
semantic: list[MemoryScene] = []
|
|
256
|
+
if nearest_fact_ids:
|
|
257
|
+
placeholders = ",".join("?" for _ in nearest_fact_ids)
|
|
258
|
+
try:
|
|
259
|
+
semantic_rows = self._db.execute(
|
|
260
|
+
f"""
|
|
261
|
+
SELECT ms.*, member.fact_id AS matched_fact_id
|
|
262
|
+
FROM scene_fact_members AS member
|
|
263
|
+
JOIN memory_scenes AS ms
|
|
264
|
+
ON ms.scene_id = member.scene_id
|
|
265
|
+
AND ms.profile_id = member.profile_id
|
|
266
|
+
WHERE member.profile_id = ?
|
|
267
|
+
AND member.fact_id IN ({placeholders})
|
|
268
|
+
""",
|
|
269
|
+
(profile_id, *nearest_fact_ids),
|
|
270
|
+
)
|
|
271
|
+
hit_rank = {
|
|
272
|
+
fact_id: rank
|
|
273
|
+
for rank, fact_id in enumerate(nearest_fact_ids)
|
|
274
|
+
}
|
|
275
|
+
ranked_scenes: dict[str, tuple[int, MemoryScene]] = {}
|
|
276
|
+
for row in semantic_rows:
|
|
277
|
+
data = dict(row)
|
|
278
|
+
rank = hit_rank.get(
|
|
279
|
+
str(data.pop("matched_fact_id", "")),
|
|
280
|
+
len(hit_rank),
|
|
281
|
+
)
|
|
282
|
+
scene = self._row_to_scene(data)
|
|
283
|
+
previous = ranked_scenes.get(scene.scene_id)
|
|
284
|
+
if previous is None or rank < previous[0]:
|
|
285
|
+
ranked_scenes[scene.scene_id] = (rank, scene)
|
|
286
|
+
semantic = [
|
|
287
|
+
scene
|
|
288
|
+
for _rank, scene in sorted(
|
|
289
|
+
ranked_scenes.values(), key=lambda item: item[0]
|
|
290
|
+
)
|
|
291
|
+
]
|
|
292
|
+
except Exception:
|
|
293
|
+
# Migration failure or a disabled vector projection must not
|
|
294
|
+
# make remember fail. The bounded recent set remains valid.
|
|
295
|
+
semantic = []
|
|
296
|
+
|
|
297
|
+
candidates: list[MemoryScene] = []
|
|
298
|
+
seen: set[str] = set()
|
|
299
|
+
for scene in (*semantic, *recent):
|
|
300
|
+
if scene.scene_id in seen:
|
|
301
|
+
continue
|
|
302
|
+
seen.add(scene.scene_id)
|
|
303
|
+
candidates.append(scene)
|
|
304
|
+
if len(candidates) >= _MAX_ASSIGNMENT_CANDIDATES:
|
|
305
|
+
break
|
|
306
|
+
return candidates
|
|
307
|
+
|
|
212
308
|
def _load_live_scene_embeddings(
|
|
213
309
|
self,
|
|
214
310
|
profile_id: str,
|
|
311
|
+
scene_ids: tuple[str, ...],
|
|
215
312
|
) -> dict[str, list[float] | None]:
|
|
216
313
|
"""Load one durable anchor embedding for every live scene.
|
|
217
314
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
315
|
+
The normalized membership projection resolves the first still-existing
|
|
316
|
+
fact in each scene without expanding every scene's JSON array. Scenes
|
|
317
|
+
whose original anchor was consolidated away can still reuse a surviving
|
|
318
|
+
member. Fully stale scene rows are ignored instead of being re-embedded.
|
|
222
319
|
"""
|
|
320
|
+
if not scene_ids:
|
|
321
|
+
return {}
|
|
322
|
+
placeholders = ",".join("?" for _ in scene_ids)
|
|
223
323
|
try:
|
|
224
324
|
rows = self._db.execute(
|
|
225
|
-
"""
|
|
325
|
+
f"""
|
|
226
326
|
WITH live_scene_facts AS (
|
|
227
327
|
SELECT
|
|
228
328
|
ms.scene_id,
|
|
229
|
-
ms.theme,
|
|
230
329
|
af.embedding,
|
|
231
330
|
ROW_NUMBER() OVER (
|
|
232
331
|
PARTITION BY ms.scene_id
|
|
233
|
-
ORDER BY
|
|
332
|
+
ORDER BY member.position
|
|
234
333
|
) AS member_rank
|
|
235
334
|
FROM memory_scenes AS ms
|
|
236
|
-
JOIN
|
|
335
|
+
JOIN scene_fact_members AS member
|
|
336
|
+
ON member.scene_id = ms.scene_id
|
|
337
|
+
AND member.profile_id = ms.profile_id
|
|
237
338
|
JOIN atomic_facts AS af
|
|
238
|
-
ON af.fact_id = member.
|
|
339
|
+
ON af.fact_id = member.fact_id
|
|
239
340
|
AND af.profile_id = ms.profile_id
|
|
240
341
|
WHERE ms.profile_id = ?
|
|
342
|
+
AND ms.scene_id IN ({placeholders})
|
|
241
343
|
)
|
|
242
344
|
SELECT scene_id, embedding
|
|
243
345
|
FROM live_scene_facts
|
|
244
346
|
WHERE member_rank = 1
|
|
245
347
|
""",
|
|
246
|
-
(profile_id,),
|
|
348
|
+
(profile_id, *scene_ids),
|
|
247
349
|
)
|
|
248
350
|
except Exception:
|
|
249
351
|
return {}
|
|
@@ -131,6 +131,9 @@ from superlocalmemory.storage.migrations import (
|
|
|
131
131
|
from superlocalmemory.storage.migrations import (
|
|
132
132
|
M033_learning_feedback_channel as _M033,
|
|
133
133
|
)
|
|
134
|
+
from superlocalmemory.storage.migrations import (
|
|
135
|
+
M034_scene_fact_members as _M034,
|
|
136
|
+
)
|
|
134
137
|
|
|
135
138
|
# Map migration name → module (used for the optional ``verify(conn)`` hook
|
|
136
139
|
# that lets the runner detect "already applied" state when an idempotent
|
|
@@ -168,6 +171,7 @@ _MODULES = {
|
|
|
168
171
|
_M031.NAME: _M031,
|
|
169
172
|
_M032.NAME: _M032,
|
|
170
173
|
_M033.NAME: _M033,
|
|
174
|
+
_M034.NAME: _M034,
|
|
171
175
|
}
|
|
172
176
|
|
|
173
177
|
logger = logging.getLogger(__name__)
|
|
@@ -305,6 +309,9 @@ DEFERRED_MIGRATIONS: list[Migration] = [
|
|
|
305
309
|
Migration(name=_M029.NAME, db_target="memory", ddl=_M029.DDL),
|
|
306
310
|
# M030 bounds Entity Explorer pagination and profile-summary ranking.
|
|
307
311
|
Migration(name=_M030.NAME, db_target="memory", ddl=_M030.DDL),
|
|
312
|
+
# M034 normalizes memory_scenes.fact_ids_json after engine initialization
|
|
313
|
+
# has created memory_scenes and atomic_facts.
|
|
314
|
+
Migration(name=_M034.NAME, db_target="memory", ddl=_M034.DDL),
|
|
308
315
|
]
|
|
309
316
|
|
|
310
317
|
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
"""M034 — normalized scene/fact membership for bounded assignment.
|
|
5
|
+
|
|
6
|
+
``memory_scenes.fact_ids_json`` remains the public compatibility format. This
|
|
7
|
+
additive projection provides the indexed reverse lookup needed to map nearest
|
|
8
|
+
fact-vector hits back to candidate scenes without scanning every scene for
|
|
9
|
+
every ingested fact. Triggers keep all existing scene write paths synchronized.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import sqlite3
|
|
15
|
+
|
|
16
|
+
NAME = "M034_scene_fact_members"
|
|
17
|
+
DB_TARGET = "memory"
|
|
18
|
+
|
|
19
|
+
DDL = """
|
|
20
|
+
BEGIN IMMEDIATE;
|
|
21
|
+
|
|
22
|
+
-- Keep the deferred migration independently safe for partial/legacy installs.
|
|
23
|
+
-- On normal daemon startup MemoryEngine has already created this table.
|
|
24
|
+
CREATE TABLE IF NOT EXISTS memory_scenes (
|
|
25
|
+
scene_id TEXT PRIMARY KEY,
|
|
26
|
+
profile_id TEXT NOT NULL DEFAULT 'default',
|
|
27
|
+
theme TEXT NOT NULL DEFAULT '',
|
|
28
|
+
fact_ids_json TEXT NOT NULL DEFAULT '[]',
|
|
29
|
+
entity_ids_json TEXT NOT NULL DEFAULT '[]',
|
|
30
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
31
|
+
last_updated TEXT NOT NULL DEFAULT (datetime('now')),
|
|
32
|
+
FOREIGN KEY (profile_id) REFERENCES profiles(profile_id) ON DELETE CASCADE
|
|
33
|
+
);
|
|
34
|
+
CREATE INDEX IF NOT EXISTS idx_scenes_profile ON memory_scenes(profile_id);
|
|
35
|
+
|
|
36
|
+
CREATE TABLE IF NOT EXISTS scene_fact_members (
|
|
37
|
+
profile_id TEXT NOT NULL,
|
|
38
|
+
scene_id TEXT NOT NULL,
|
|
39
|
+
fact_id TEXT NOT NULL,
|
|
40
|
+
position INTEGER NOT NULL DEFAULT 0,
|
|
41
|
+
PRIMARY KEY (scene_id, fact_id),
|
|
42
|
+
FOREIGN KEY (scene_id) REFERENCES memory_scenes(scene_id) ON DELETE CASCADE,
|
|
43
|
+
FOREIGN KEY (fact_id) REFERENCES atomic_facts(fact_id) ON DELETE CASCADE,
|
|
44
|
+
FOREIGN KEY (profile_id) REFERENCES profiles(profile_id) ON DELETE CASCADE
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
CREATE INDEX IF NOT EXISTS idx_scene_fact_members_lookup
|
|
48
|
+
ON scene_fact_members (profile_id, fact_id, scene_id);
|
|
49
|
+
CREATE INDEX IF NOT EXISTS idx_scene_fact_members_order
|
|
50
|
+
ON scene_fact_members (scene_id, position);
|
|
51
|
+
|
|
52
|
+
CREATE TRIGGER IF NOT EXISTS trg_scene_fact_members_insert
|
|
53
|
+
AFTER INSERT ON memory_scenes
|
|
54
|
+
BEGIN
|
|
55
|
+
DELETE FROM scene_fact_members WHERE scene_id = NEW.scene_id;
|
|
56
|
+
INSERT OR IGNORE INTO scene_fact_members
|
|
57
|
+
(profile_id, scene_id, fact_id, position)
|
|
58
|
+
SELECT NEW.profile_id, NEW.scene_id, af.fact_id, CAST(member.key AS INTEGER)
|
|
59
|
+
FROM json_each(
|
|
60
|
+
CASE WHEN json_valid(NEW.fact_ids_json)
|
|
61
|
+
THEN NEW.fact_ids_json ELSE '[]' END
|
|
62
|
+
) AS member
|
|
63
|
+
JOIN atomic_facts AS af
|
|
64
|
+
ON af.fact_id = member.value
|
|
65
|
+
AND af.profile_id = NEW.profile_id;
|
|
66
|
+
END;
|
|
67
|
+
|
|
68
|
+
CREATE TRIGGER IF NOT EXISTS trg_scene_fact_members_update
|
|
69
|
+
AFTER UPDATE OF profile_id, fact_ids_json ON memory_scenes
|
|
70
|
+
BEGIN
|
|
71
|
+
DELETE FROM scene_fact_members WHERE scene_id = NEW.scene_id;
|
|
72
|
+
INSERT OR IGNORE INTO scene_fact_members
|
|
73
|
+
(profile_id, scene_id, fact_id, position)
|
|
74
|
+
SELECT NEW.profile_id, NEW.scene_id, af.fact_id, CAST(member.key AS INTEGER)
|
|
75
|
+
FROM json_each(
|
|
76
|
+
CASE WHEN json_valid(NEW.fact_ids_json)
|
|
77
|
+
THEN NEW.fact_ids_json ELSE '[]' END
|
|
78
|
+
) AS member
|
|
79
|
+
JOIN atomic_facts AS af
|
|
80
|
+
ON af.fact_id = member.value
|
|
81
|
+
AND af.profile_id = NEW.profile_id;
|
|
82
|
+
END;
|
|
83
|
+
|
|
84
|
+
INSERT OR IGNORE INTO scene_fact_members
|
|
85
|
+
(profile_id, scene_id, fact_id, position)
|
|
86
|
+
SELECT ms.profile_id, ms.scene_id, af.fact_id, CAST(member.key AS INTEGER)
|
|
87
|
+
FROM memory_scenes AS ms
|
|
88
|
+
JOIN json_each(
|
|
89
|
+
CASE WHEN json_valid(ms.fact_ids_json) THEN ms.fact_ids_json ELSE '[]' END
|
|
90
|
+
) AS member
|
|
91
|
+
JOIN atomic_facts AS af
|
|
92
|
+
ON af.fact_id = member.value
|
|
93
|
+
AND af.profile_id = ms.profile_id;
|
|
94
|
+
|
|
95
|
+
COMMIT;
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def verify(conn: sqlite3.Connection) -> bool:
|
|
100
|
+
"""Verify the table, covering indexes, and synchronization triggers."""
|
|
101
|
+
objects = {
|
|
102
|
+
(str(row[0]), str(row[1]))
|
|
103
|
+
for row in conn.execute(
|
|
104
|
+
"SELECT name, type FROM sqlite_master "
|
|
105
|
+
"WHERE name IN (?, ?, ?, ?, ?)"
|
|
106
|
+
,
|
|
107
|
+
(
|
|
108
|
+
"scene_fact_members",
|
|
109
|
+
"idx_scene_fact_members_lookup",
|
|
110
|
+
"idx_scene_fact_members_order",
|
|
111
|
+
"trg_scene_fact_members_insert",
|
|
112
|
+
"trg_scene_fact_members_update",
|
|
113
|
+
),
|
|
114
|
+
).fetchall()
|
|
115
|
+
}
|
|
116
|
+
return objects == {
|
|
117
|
+
("scene_fact_members", "table"),
|
|
118
|
+
("idx_scene_fact_members_lookup", "index"),
|
|
119
|
+
("idx_scene_fact_members_order", "index"),
|
|
120
|
+
("trg_scene_fact_members_insert", "trigger"),
|
|
121
|
+
("trg_scene_fact_members_update", "trigger"),
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def repair(conn: sqlite3.Connection) -> None:
|
|
126
|
+
"""Restore an accidentally dropped projection and re-backfill it."""
|
|
127
|
+
conn.executescript(DDL)
|
|
@@ -29,6 +29,7 @@ from . import (
|
|
|
29
29
|
M029_behavioral_history_indexes,
|
|
30
30
|
M030_entity_explorer_indexes,
|
|
31
31
|
M033_learning_feedback_channel,
|
|
32
|
+
M034_scene_fact_members,
|
|
32
33
|
)
|
|
33
34
|
|
|
34
35
|
# ---------------------------------------------------------------------------
|
|
@@ -83,6 +84,7 @@ __all__ = (
|
|
|
83
84
|
"M029_behavioral_history_indexes",
|
|
84
85
|
"M030_entity_explorer_indexes",
|
|
85
86
|
"M033_learning_feedback_channel",
|
|
87
|
+
"M034_scene_fact_members",
|
|
86
88
|
# Legacy re-exports (backward compat):
|
|
87
89
|
"CURRENT_SCHEMA_VERSION",
|
|
88
90
|
"get_schema_version",
|
|
@@ -43,6 +43,7 @@ _TABLES: Final[tuple[str, ...]] = (
|
|
|
43
43
|
"entity_aliases",
|
|
44
44
|
"entity_profiles",
|
|
45
45
|
"memory_scenes",
|
|
46
|
+
"scene_fact_members",
|
|
46
47
|
"temporal_events",
|
|
47
48
|
"graph_edges",
|
|
48
49
|
"consolidation_log",
|
|
@@ -455,6 +456,61 @@ CREATE INDEX IF NOT EXISTS idx_scenes_profile
|
|
|
455
456
|
"""
|
|
456
457
|
|
|
457
458
|
|
|
459
|
+
# ---------------------------------------------------------------------------
|
|
460
|
+
# Normalized scene/fact membership projection (bounded scene assignment)
|
|
461
|
+
# ---------------------------------------------------------------------------
|
|
462
|
+
|
|
463
|
+
_SQL_SCENE_FACT_MEMBERS: Final[str] = """
|
|
464
|
+
CREATE TABLE IF NOT EXISTS scene_fact_members (
|
|
465
|
+
profile_id TEXT NOT NULL,
|
|
466
|
+
scene_id TEXT NOT NULL,
|
|
467
|
+
fact_id TEXT NOT NULL,
|
|
468
|
+
position INTEGER NOT NULL DEFAULT 0,
|
|
469
|
+
PRIMARY KEY (scene_id, fact_id),
|
|
470
|
+
FOREIGN KEY (scene_id) REFERENCES memory_scenes(scene_id) ON DELETE CASCADE,
|
|
471
|
+
FOREIGN KEY (fact_id) REFERENCES atomic_facts(fact_id) ON DELETE CASCADE,
|
|
472
|
+
FOREIGN KEY (profile_id) REFERENCES profiles(profile_id) ON DELETE CASCADE
|
|
473
|
+
);
|
|
474
|
+
|
|
475
|
+
CREATE INDEX IF NOT EXISTS idx_scene_fact_members_lookup
|
|
476
|
+
ON scene_fact_members (profile_id, fact_id, scene_id);
|
|
477
|
+
CREATE INDEX IF NOT EXISTS idx_scene_fact_members_order
|
|
478
|
+
ON scene_fact_members (scene_id, position);
|
|
479
|
+
|
|
480
|
+
CREATE TRIGGER IF NOT EXISTS trg_scene_fact_members_insert
|
|
481
|
+
AFTER INSERT ON memory_scenes
|
|
482
|
+
BEGIN
|
|
483
|
+
DELETE FROM scene_fact_members WHERE scene_id = NEW.scene_id;
|
|
484
|
+
INSERT OR IGNORE INTO scene_fact_members
|
|
485
|
+
(profile_id, scene_id, fact_id, position)
|
|
486
|
+
SELECT NEW.profile_id, NEW.scene_id, af.fact_id, CAST(member.key AS INTEGER)
|
|
487
|
+
FROM json_each(
|
|
488
|
+
CASE WHEN json_valid(NEW.fact_ids_json)
|
|
489
|
+
THEN NEW.fact_ids_json ELSE '[]' END
|
|
490
|
+
) AS member
|
|
491
|
+
JOIN atomic_facts AS af
|
|
492
|
+
ON af.fact_id = member.value
|
|
493
|
+
AND af.profile_id = NEW.profile_id;
|
|
494
|
+
END;
|
|
495
|
+
|
|
496
|
+
CREATE TRIGGER IF NOT EXISTS trg_scene_fact_members_update
|
|
497
|
+
AFTER UPDATE OF profile_id, fact_ids_json ON memory_scenes
|
|
498
|
+
BEGIN
|
|
499
|
+
DELETE FROM scene_fact_members WHERE scene_id = NEW.scene_id;
|
|
500
|
+
INSERT OR IGNORE INTO scene_fact_members
|
|
501
|
+
(profile_id, scene_id, fact_id, position)
|
|
502
|
+
SELECT NEW.profile_id, NEW.scene_id, af.fact_id, CAST(member.key AS INTEGER)
|
|
503
|
+
FROM json_each(
|
|
504
|
+
CASE WHEN json_valid(NEW.fact_ids_json)
|
|
505
|
+
THEN NEW.fact_ids_json ELSE '[]' END
|
|
506
|
+
) AS member
|
|
507
|
+
JOIN atomic_facts AS af
|
|
508
|
+
ON af.fact_id = member.value
|
|
509
|
+
AND af.profile_id = NEW.profile_id;
|
|
510
|
+
END;
|
|
511
|
+
"""
|
|
512
|
+
|
|
513
|
+
|
|
458
514
|
# ---------------------------------------------------------------------------
|
|
459
515
|
# Temporal events (per-entity timeline entries)
|
|
460
516
|
# ---------------------------------------------------------------------------
|
|
@@ -803,6 +859,7 @@ _DDL_ORDERED: Final[tuple[str, ...]] = (
|
|
|
803
859
|
_SQL_ENTITY_ALIASES,
|
|
804
860
|
_SQL_ENTITY_PROFILES,
|
|
805
861
|
_SQL_MEMORY_SCENES,
|
|
862
|
+
_SQL_SCENE_FACT_MEMBERS,
|
|
806
863
|
_SQL_TEMPORAL_EVENTS,
|
|
807
864
|
_SQL_GRAPH_EDGES,
|
|
808
865
|
_SQL_CONSOLIDATION_LOG,
|
|
@@ -887,6 +944,8 @@ def drop_all_tables(conn: sqlite3.Connection) -> None:
|
|
|
887
944
|
"atomic_facts_fts_insert",
|
|
888
945
|
"atomic_facts_fts_delete",
|
|
889
946
|
"atomic_facts_fts_update",
|
|
947
|
+
"trg_scene_fact_members_insert",
|
|
948
|
+
"trg_scene_fact_members_update",
|
|
890
949
|
):
|
|
891
950
|
conn.execute(f"DROP TRIGGER IF EXISTS {trigger}")
|
|
892
951
|
|