superlocalmemory 3.7.4 → 3.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +2 -2
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-recall/SKILL.md +4 -3
- package/plugin-src/manifest.json +1 -1
- package/plugin-src/requirements.txt +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +4 -3
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/scale_engine_cmd.py +2 -1
- package/src/superlocalmemory/cli/setup_wizard.py +35 -1
- package/src/superlocalmemory/core/backend_orchestrator.py +6 -1
- package/src/superlocalmemory/core/config.py +1 -1
- package/src/superlocalmemory/core/engine.py +3 -2
- package/src/superlocalmemory/core/engine_wiring.py +3 -1
- package/src/superlocalmemory/core/scale_engine.py +58 -29
- package/src/superlocalmemory/core/store_pipeline.py +1 -1
- package/src/superlocalmemory/hooks/before_web_hook.py +1 -1
- package/src/superlocalmemory/hooks/claude_code_hooks.py +1 -1
- package/src/superlocalmemory/mcp/tools_active.py +11 -7
- package/src/superlocalmemory/retrieval/engine.py +16 -8
- package/src/superlocalmemory/server/recall_health.py +1 -1
- package/src/superlocalmemory/server/routes/chat.py +2 -2
- package/src/superlocalmemory/server/unified_daemon.py +18 -5
- package/src/superlocalmemory/storage/migration_runner.py +30 -0
- package/src/superlocalmemory/storage/sqlite_vectors.py +169 -0
- package/src/superlocalmemory/ui/js/memory-chat.js +2 -2
- package/src/superlocalmemory/vector/lancedb_backend.py +81 -74
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,27 @@ All notable changes to SuperLocalMemory V3 will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [3.7.6] - 2026-07-19 — Auth, upgrade, and embedding-dimension fixes
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
|
|
12
|
+
- Writes authenticated by the daemon capability or the dashboard install token are no longer rejected once an `api_key` file is configured. The write path ran a redundant second gate that only understood `X-SLM-API-Key`, so capability-authenticated MCP `remember` write-throughs and install-token dashboard writes / config tests returned 401 "Invalid or missing API key" whenever opt-in API-key auth was enabled. The mutation-actor gate — which already accepts the daemon capability, the install token, a matching API key, or an uncredentialed loopback caller — is now the single authoritative write boundary. (#71, #73, #74)
|
|
13
|
+
- Upgrading an install across a benign migration DDL change no longer leaves the daemon permanently `not_ready`. On a `ddl_sha256` mismatch for a migration already marked complete, the runner now consults the migration's `verify()`; when the schema end-state is present it reconciles the log to the current hash instead of failing readiness. Real drift with an absent schema is still surfaced as a failure. (#70)
|
|
14
|
+
- The LanceDB vector backend now follows the configured embedding dimension instead of a hardcoded 768, so custom OpenAI-compatible endpoints (for example 1024-d Qwen3-Embedding) no longer hit a vector-dimension mismatch on initialization. An existing store's on-disk width is always honored, keeping already-materialized data readable after a configuration change. (#72)
|
|
15
|
+
|
|
16
|
+
### Improved
|
|
17
|
+
|
|
18
|
+
- `slm setup` skips the local 768-d embedding-model download when a remote/OpenAI-compatible embedding endpoint is configured, instead of forcing an unnecessary model fetch. (#72)
|
|
19
|
+
- Aligned retrieval documentation and docstrings with the shipped architecture: five parallel candidate producers (semantic, BM25, temporal, spreading-activation, Hopfield) feed single-pass RRF fusion, followed by optional cross-encoder rerank and an entity-graph post-fusion score enhancement. The entity graph is not a sixth parallel candidate producer.
|
|
20
|
+
|
|
21
|
+
## [3.7.5] - 2026-07-18 — Complete Scale Engine projection parity
|
|
22
|
+
|
|
23
|
+
### Improved
|
|
24
|
+
|
|
25
|
+
- Scale Engine vector preparation now reads the supported sqlite-vec virtual-table contract and joins vectors through canonical embedding metadata.
|
|
26
|
+
- LanceDB projection imports are profile-scoped, lifecycle-aware, and processed in bounded batches to keep migration memory stable.
|
|
27
|
+
- Scale verification and fingerprints now cover the same canonical vector rows that are written to LanceDB, so a partial vector projection cannot pass parity.
|
|
28
|
+
|
|
8
29
|
## [3.7.4] - 2026-07-18 — Scale Engine projection-parity release
|
|
9
30
|
|
|
10
31
|
### Fixed
|
package/README.md
CHANGED
|
@@ -5,10 +5,10 @@
|
|
|
5
5
|
</picture>
|
|
6
6
|
</p>
|
|
7
7
|
|
|
8
|
-
<h1 align="center">SuperLocalMemory V3.7.
|
|
8
|
+
<h1 align="center">SuperLocalMemory V3.7.5</h1>
|
|
9
9
|
<p align="center"><strong>Cache. Compress. Remember. Three surfaces — proxy, MCP tools, or skill. Every setup covered.</strong><br/>
|
|
10
10
|
<em>Local-first agent memory with explicit operating modes, auditable retrieval, and optional Optimize tools.</em></p>
|
|
11
|
-
<p align="center"><code>v3.7.
|
|
11
|
+
<p align="center"><code>v3.7.5</code> — <strong>Scale Engine graph and vector projection parity, durable promotion recovery, and honest runtime status.</strong><br/>
|
|
12
12
|
Proxy: <code>slm wrap claude</code> · MCP: add <code>slm_compress</code> to your config · Skill: zero-config</p>
|
|
13
13
|
<p align="center"><strong>3 public research preprints</strong> (arXiv + Zenodo archives) · <a href="https://arxiv.org/abs/2603.02240">arXiv:2603.02240</a> · <a href="https://arxiv.org/abs/2603.14588">arXiv:2603.14588</a> · <a href="https://arxiv.org/abs/2604.04514">arXiv:2604.04514</a></p>
|
|
14
14
|
|
package/package.json
CHANGED
package/plugin/requirements.txt
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
superlocalmemory==3.7.
|
|
1
|
+
superlocalmemory==3.7.6
|
|
@@ -142,9 +142,10 @@ once you have the `fact_id` for full content.
|
|
|
142
142
|
|
|
143
143
|
## How multi-channel retrieval works
|
|
144
144
|
|
|
145
|
-
`recall` runs
|
|
146
|
-
BM25, temporal recency, and
|
|
147
|
-
Rank Fusion (RRF)
|
|
145
|
+
`recall` runs five candidate producers in parallel — semantic vector similarity,
|
|
146
|
+
lexical BM25, temporal recency, spreading activation, and Hopfield — then fuses
|
|
147
|
+
them with Reciprocal Rank Fusion (RRF), applies a reranker, and layers an
|
|
148
|
+
optional entity-graph post-fusion score enhancement. The `channel_weights` field in the
|
|
148
149
|
response shows how each channel contributed for that query. Weights adapt over
|
|
149
150
|
time based on engagement signals attributed via `session_id`.
|
|
150
151
|
|
package/plugin-src/manifest.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
superlocalmemory==3.7.
|
|
1
|
+
superlocalmemory==3.7.6
|
|
@@ -142,9 +142,10 @@ once you have the `fact_id` for full content.
|
|
|
142
142
|
|
|
143
143
|
## How multi-channel retrieval works
|
|
144
144
|
|
|
145
|
-
`recall` runs
|
|
146
|
-
BM25, temporal recency, and
|
|
147
|
-
Rank Fusion (RRF)
|
|
145
|
+
`recall` runs five candidate producers in parallel — semantic vector similarity,
|
|
146
|
+
lexical BM25, temporal recency, spreading activation, and Hopfield — then fuses
|
|
147
|
+
them with Reciprocal Rank Fusion (RRF), applies a reranker, and layers an
|
|
148
|
+
optional entity-graph post-fusion score enhancement. The `channel_weights` field in the
|
|
148
149
|
response shows how each channel contributed for that query. Weights adapt over
|
|
149
150
|
time based on engagement signals attributed via `session_id`.
|
|
150
151
|
|
package/pyproject.toml
CHANGED
|
@@ -32,7 +32,7 @@ if "OMP_NUM_THREADS" not in os.environ:
|
|
|
32
32
|
os.environ["OMP_NUM_THREADS"] = "2"
|
|
33
33
|
# ---------------------------------------------------------------------------
|
|
34
34
|
|
|
35
|
-
__version__ = "3.7.
|
|
35
|
+
__version__ = "3.7.6"
|
|
36
36
|
|
|
37
37
|
_REQUIRED_VERSIONS = {
|
|
38
38
|
"sentence_transformers": "5.3.0",
|
|
@@ -8,6 +8,7 @@ from argparse import Namespace
|
|
|
8
8
|
def cmd_db_scale(args: Namespace) -> int:
|
|
9
9
|
from superlocalmemory.core.config import SLMConfig
|
|
10
10
|
from superlocalmemory.core.scale_engine import ScaleEngineError, ScaleEngineManager
|
|
11
|
+
from superlocalmemory.storage.sqlite_vectors import CanonicalVectorError
|
|
11
12
|
|
|
12
13
|
action = args.scale_action
|
|
13
14
|
# Scale Engine projections are currently canonical-default-profile data.
|
|
@@ -42,7 +43,7 @@ def cmd_db_scale(args: Namespace) -> int:
|
|
|
42
43
|
result = manager.rollback(args.backup_id)
|
|
43
44
|
else:
|
|
44
45
|
raise ScaleEngineError(f"unknown Scale Engine action: {action}")
|
|
45
|
-
except ScaleEngineError as exc:
|
|
46
|
+
except (ScaleEngineError, CanonicalVectorError) as exc:
|
|
46
47
|
print(f"Scale Engine: {exc}")
|
|
47
48
|
return 1
|
|
48
49
|
print(json.dumps(result, indent=2, sort_keys=True))
|
|
@@ -22,6 +22,7 @@ import subprocess
|
|
|
22
22
|
import sys
|
|
23
23
|
import time
|
|
24
24
|
from pathlib import Path
|
|
25
|
+
from typing import Any
|
|
25
26
|
|
|
26
27
|
|
|
27
28
|
# ---------------------------------------------------------------------------
|
|
@@ -227,6 +228,29 @@ def _download_compressor(model_name: str) -> bool:
|
|
|
227
228
|
return False
|
|
228
229
|
|
|
229
230
|
|
|
231
|
+
# ---------------------------------------------------------------------------
|
|
232
|
+
# Embedding-provider detection (v3.7.6 #72)
|
|
233
|
+
# ---------------------------------------------------------------------------
|
|
234
|
+
|
|
235
|
+
def _embedding_is_remote(config: Any) -> bool:
|
|
236
|
+
"""True when embeddings come from a remote/OpenAI-compatible endpoint.
|
|
237
|
+
|
|
238
|
+
In that case the local sentence-transformers download (768d nomic) is
|
|
239
|
+
unnecessary and would only waste bandwidth/disk (#72). Detection is by
|
|
240
|
+
provider name or the presence of a configured HTTP endpoint.
|
|
241
|
+
"""
|
|
242
|
+
emb = getattr(config, "embedding", None)
|
|
243
|
+
if emb is None:
|
|
244
|
+
return False
|
|
245
|
+
provider = (getattr(emb, "provider", "") or "").strip().lower()
|
|
246
|
+
endpoint = (
|
|
247
|
+
getattr(emb, "api_endpoint", "")
|
|
248
|
+
or getattr(emb, "base_url", "")
|
|
249
|
+
or ""
|
|
250
|
+
)
|
|
251
|
+
return provider in ("openai", "openai-compatible", "remote") or bool(endpoint)
|
|
252
|
+
|
|
253
|
+
|
|
230
254
|
# ---------------------------------------------------------------------------
|
|
231
255
|
# Verification
|
|
232
256
|
# ---------------------------------------------------------------------------
|
|
@@ -455,7 +479,17 @@ def run_wizard(auto: bool = False) -> None:
|
|
|
455
479
|
print()
|
|
456
480
|
print("─── Step 4/10: Download Embedding Model ───")
|
|
457
481
|
|
|
458
|
-
if
|
|
482
|
+
if _embedding_is_remote(config):
|
|
483
|
+
# v3.7.6 (#72): a remote/OpenAI-compatible embedding endpoint supplies
|
|
484
|
+
# its own vectors (any width, e.g. 1024d Qwen3-Embedding), so pulling the
|
|
485
|
+
# local 768d nomic model would be wasted bandwidth and could imply a
|
|
486
|
+
# dimension that does not match the endpoint.
|
|
487
|
+
emb = config.embedding
|
|
488
|
+
print(" ✓ Skipped — remote/OpenAI-compatible embedding endpoint configured")
|
|
489
|
+
print(f" provider={getattr(emb, 'provider', '?')}, "
|
|
490
|
+
f"dimension={getattr(emb, 'dimension', '?')}")
|
|
491
|
+
print(" No local embedding model needed.")
|
|
492
|
+
elif not st_ok:
|
|
459
493
|
print(" ⚠ Skipped (sentence-transformers not installed)")
|
|
460
494
|
print(" Models will download on first use.")
|
|
461
495
|
else:
|
|
@@ -354,7 +354,12 @@ class BackendOrchestrator:
|
|
|
354
354
|
try:
|
|
355
355
|
from superlocalmemory.vector.lancedb_backend import LanceDBVectorBackend
|
|
356
356
|
lance_path = self._data_dir / "lance"
|
|
357
|
-
|
|
357
|
+
# v3.7.6 (#72): honor the configured embedding width instead of the
|
|
358
|
+
# hardcoded 768d, so custom endpoints (e.g. 1024d Qwen3-Embedding) work.
|
|
359
|
+
dimension = getattr(
|
|
360
|
+
getattr(self._config, "embedding", None), "dimension", None
|
|
361
|
+
)
|
|
362
|
+
self._lancedb = LanceDBVectorBackend(str(lance_path), dimension=dimension)
|
|
358
363
|
self._update_status("lancedb", "not_initialized")
|
|
359
364
|
logger.info("LanceDB initialized at %s", lance_path)
|
|
360
365
|
except Exception as exc:
|
|
@@ -258,7 +258,7 @@ class RetrievalConfig:
|
|
|
258
258
|
# Reranking (V3.3.2: ONNX backend enabled for all modes)
|
|
259
259
|
# V3.4.2: Tested gte-reranker-modernbert-base (8K context) — REGRESSED
|
|
260
260
|
# LoCoMo from 68.4% to 64.1%. Reverted to MiniLM-L-12-v2. The 512-token
|
|
261
|
-
# limit is acceptable because SLM's
|
|
261
|
+
# limit is acceptable because SLM's multi-producer retrieval pre-filters
|
|
262
262
|
# relevant facts before reranking. See bench-v342-locomo.md.
|
|
263
263
|
use_cross_encoder: bool = True
|
|
264
264
|
cross_encoder_model: str = "cross-encoder/ms-marco-MiniLM-L-12-v2"
|
|
@@ -51,8 +51,9 @@ class MemoryEngine:
|
|
|
51
51
|
"""Main orchestrator for the SuperLocalMemory V3 memory system.
|
|
52
52
|
|
|
53
53
|
Wires encoding (fact extraction, entity resolution, graph building,
|
|
54
|
-
consolidation) with retrieval (
|
|
55
|
-
reranking) and all supporting
|
|
54
|
+
consolidation) with retrieval (five candidate producers -> RRF fusion,
|
|
55
|
+
reranking, entity-graph post-fusion enhancement) and all supporting
|
|
56
|
+
layers (trust, learning, compliance).
|
|
56
57
|
|
|
57
58
|
Usage::
|
|
58
59
|
|
|
@@ -500,7 +500,9 @@ def init_retrieval(
|
|
|
500
500
|
trust_scorer: Any,
|
|
501
501
|
vector_store: Any = None,
|
|
502
502
|
) -> Any:
|
|
503
|
-
"""Create the RetrievalEngine
|
|
503
|
+
"""Create the RetrievalEngine — five candidate producers (semantic, BM25,
|
|
504
|
+
temporal, spreading_activation, hopfield) plus the entity graph used for
|
|
505
|
+
post-fusion score enhancement. Returns it."""
|
|
504
506
|
from superlocalmemory.retrieval.engine import RetrievalEngine
|
|
505
507
|
from superlocalmemory.retrieval.semantic_channel import SemanticChannel
|
|
506
508
|
from superlocalmemory.retrieval.bm25_channel import BM25Channel
|
|
@@ -23,6 +23,12 @@ from pathlib import Path
|
|
|
23
23
|
from typing import Any, Callable
|
|
24
24
|
|
|
25
25
|
from superlocalmemory.storage.logical_edges import count_logical_edges, iter_logical_edges
|
|
26
|
+
from superlocalmemory.storage.sqlite_vectors import (
|
|
27
|
+
CanonicalVectorError,
|
|
28
|
+
count_canonical_vectors,
|
|
29
|
+
iter_canonical_vectors,
|
|
30
|
+
load_sqlite_vec_extension,
|
|
31
|
+
)
|
|
26
32
|
|
|
27
33
|
|
|
28
34
|
class ScaleEngineError(RuntimeError):
|
|
@@ -272,6 +278,9 @@ class ScaleEngineManager:
|
|
|
272
278
|
self.config.scale_engine_state = "prepared"
|
|
273
279
|
self._save_config()
|
|
274
280
|
return manifest
|
|
281
|
+
except CanonicalVectorError as exc:
|
|
282
|
+
shutil.rmtree(stage_dir, ignore_errors=True)
|
|
283
|
+
raise ScaleEngineError(f"canonical vector projection failed: {exc}") from exc
|
|
275
284
|
except Exception:
|
|
276
285
|
shutil.rmtree(stage_dir, ignore_errors=True)
|
|
277
286
|
raise
|
|
@@ -285,6 +294,8 @@ class ScaleEngineManager:
|
|
|
285
294
|
try:
|
|
286
295
|
self._recover_interrupted_promotion()
|
|
287
296
|
return self._verify(stage_id)
|
|
297
|
+
except CanonicalVectorError as exc:
|
|
298
|
+
raise ScaleEngineError(f"canonical vector projection failed: {exc}") from exc
|
|
288
299
|
finally:
|
|
289
300
|
self._release_lifecycle_lock(lock_path)
|
|
290
301
|
|
|
@@ -301,7 +312,9 @@ class ScaleEngineManager:
|
|
|
301
312
|
source_fingerprint = self._projection_fingerprint(conn, canonical)
|
|
302
313
|
observed = self._observed_counts(cozo, lance)
|
|
303
314
|
if manifest["source_fingerprint"] != source_fingerprint:
|
|
304
|
-
raise ScaleEngineError(
|
|
315
|
+
raise ScaleEngineError(
|
|
316
|
+
"canonical SQLite changed after preparation; prepare a new stage"
|
|
317
|
+
)
|
|
305
318
|
if canonical != manifest["canonical"] or observed != canonical:
|
|
306
319
|
raise ScaleEngineError(
|
|
307
320
|
f"projection parity failed: canonical={canonical}, observed={observed}"
|
|
@@ -321,6 +334,8 @@ class ScaleEngineManager:
|
|
|
321
334
|
try:
|
|
322
335
|
self._recover_interrupted_promotion()
|
|
323
336
|
return self._promote(stage_id)
|
|
337
|
+
except CanonicalVectorError as exc:
|
|
338
|
+
raise ScaleEngineError(f"canonical vector projection failed: {exc}") from exc
|
|
324
339
|
finally:
|
|
325
340
|
self._release_lifecycle_lock(lock_path)
|
|
326
341
|
|
|
@@ -341,7 +356,9 @@ class ScaleEngineManager:
|
|
|
341
356
|
gate.execute("BEGIN IMMEDIATE")
|
|
342
357
|
canonical = self._canonical_counts(gate)
|
|
343
358
|
if manifest["source_fingerprint"] != self._projection_fingerprint(gate, canonical):
|
|
344
|
-
raise ScaleEngineError(
|
|
359
|
+
raise ScaleEngineError(
|
|
360
|
+
"canonical SQLite changed after verification; prepare a new stage"
|
|
361
|
+
)
|
|
345
362
|
self._mkdir_durable(self.backup_root)
|
|
346
363
|
journal = {
|
|
347
364
|
"schema_version": self.SCHEMA_VERSION,
|
|
@@ -368,7 +385,13 @@ class ScaleEngineManager:
|
|
|
368
385
|
self._replace_durable(destination, source)
|
|
369
386
|
move["state"] = "complete"
|
|
370
387
|
self._write_promotion_journal(journal)
|
|
371
|
-
manifest.update(
|
|
388
|
+
manifest.update(
|
|
389
|
+
{
|
|
390
|
+
"state": "promoted",
|
|
391
|
+
"promoted_at": _utc_now(),
|
|
392
|
+
"backup_id": backup_dir.name,
|
|
393
|
+
}
|
|
394
|
+
)
|
|
372
395
|
self._write_manifest(stage_dir, manifest)
|
|
373
396
|
self.config.scale_engine_state = "promoted"
|
|
374
397
|
self.config.graph_backend = "cozo"
|
|
@@ -447,7 +470,11 @@ class ScaleEngineManager:
|
|
|
447
470
|
journal["state"] = "committed"
|
|
448
471
|
self._write_promotion_journal(journal)
|
|
449
472
|
self.promotion_journal_path.unlink(missing_ok=True)
|
|
450
|
-
return {
|
|
473
|
+
return {
|
|
474
|
+
"state": "local_core",
|
|
475
|
+
"restored_backup": backup_id,
|
|
476
|
+
"displaced": displaced.name,
|
|
477
|
+
}
|
|
451
478
|
except Exception as exc:
|
|
452
479
|
try:
|
|
453
480
|
self._recover_interrupted_promotion()
|
|
@@ -460,11 +487,20 @@ class ScaleEngineManager:
|
|
|
460
487
|
def _real_backend_factory(self, cozo_dir: Path, lance_dir: Path) -> tuple[Any, Any]:
|
|
461
488
|
from superlocalmemory.graph.cozo_backend import CozoDBGraphBackend
|
|
462
489
|
from superlocalmemory.vector.lancedb_backend import LanceDBVectorBackend
|
|
463
|
-
|
|
490
|
+
# v3.7.6 (#72): promote the configured embedding width into the new
|
|
491
|
+
# LanceDB store so a 1024d (or other) custom endpoint survives promotion.
|
|
492
|
+
dimension = getattr(
|
|
493
|
+
getattr(self.config, "embedding", None), "dimension", None
|
|
494
|
+
)
|
|
495
|
+
return (
|
|
496
|
+
CozoDBGraphBackend(str(cozo_dir / "graph")),
|
|
497
|
+
LanceDBVectorBackend(str(lance_dir), dimension=dimension),
|
|
498
|
+
)
|
|
464
499
|
|
|
465
500
|
def _readonly_connection(self) -> sqlite3.Connection:
|
|
466
501
|
conn = sqlite3.connect(f"file:{self.db_path}?mode=ro", uri=True)
|
|
467
502
|
conn.execute("PRAGMA query_only=ON")
|
|
503
|
+
load_sqlite_vec_extension(conn)
|
|
468
504
|
return conn
|
|
469
505
|
|
|
470
506
|
def _canonical_counts(self, conn: sqlite3.Connection) -> dict[str, int]:
|
|
@@ -473,15 +509,7 @@ class ScaleEngineManager:
|
|
|
473
509
|
(self.profile_id,),
|
|
474
510
|
).fetchone()[0]
|
|
475
511
|
edges = count_logical_edges(conn, self.profile_id)
|
|
476
|
-
|
|
477
|
-
vectors = conn.execute(
|
|
478
|
-
"SELECT COUNT(*) FROM fact_embeddings_rowids fer "
|
|
479
|
-
"JOIN atomic_facts af ON af.fact_id = fer.fact_id "
|
|
480
|
-
"WHERE af.profile_id = ?",
|
|
481
|
-
(self.profile_id,),
|
|
482
|
-
).fetchone()[0]
|
|
483
|
-
except sqlite3.OperationalError:
|
|
484
|
-
vectors = 0
|
|
512
|
+
vectors = count_canonical_vectors(conn, self.profile_id)
|
|
485
513
|
return {"entities": int(nodes), "edges": int(edges), "vectors": int(vectors)}
|
|
486
514
|
|
|
487
515
|
def _observed_counts(self, cozo: Any, lance: Any) -> dict[str, int]:
|
|
@@ -489,7 +517,11 @@ class ScaleEngineManager:
|
|
|
489
517
|
vector = lance.health_check()
|
|
490
518
|
if graph.get("status") != "active" or vector.get("status") != "active":
|
|
491
519
|
raise ScaleEngineError(f"projection health failed: cozo={graph}, lancedb={vector}")
|
|
492
|
-
return {
|
|
520
|
+
return {
|
|
521
|
+
"entities": int(graph["entities"]),
|
|
522
|
+
"edges": int(graph["edges"]),
|
|
523
|
+
"vectors": int(vector["vectors"]),
|
|
524
|
+
}
|
|
493
525
|
|
|
494
526
|
def _projection_fingerprint(
|
|
495
527
|
self, conn: sqlite3.Connection, counts: dict[str, int]
|
|
@@ -498,7 +530,12 @@ class ScaleEngineManager:
|
|
|
498
530
|
digest = hashlib.sha256()
|
|
499
531
|
digest.update(json.dumps(counts, sort_keys=True).encode())
|
|
500
532
|
tables = (
|
|
501
|
-
(
|
|
533
|
+
(
|
|
534
|
+
"canonical_entities",
|
|
535
|
+
"entity_id, canonical_name, entity_type, first_seen, last_seen, "
|
|
536
|
+
"fact_count, profile_id",
|
|
537
|
+
"entity_id",
|
|
538
|
+
),
|
|
502
539
|
("atomic_facts", "fact_id, canonical_entities_json, lifecycle, profile_id", "fact_id"),
|
|
503
540
|
)
|
|
504
541
|
for table, columns, ordering in tables:
|
|
@@ -516,18 +553,8 @@ class ScaleEngineManager:
|
|
|
516
553
|
self._digest_row(digest, "graph_edges", row)
|
|
517
554
|
except sqlite3.OperationalError as exc:
|
|
518
555
|
raise ScaleEngineError("canonical SQLite missing required graph_edges table") from exc
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
"SELECT fer.rowid, fer.fact_id, vec.vector FROM fact_embeddings_rowids fer "
|
|
522
|
-
"JOIN atomic_facts af ON af.fact_id = fer.fact_id "
|
|
523
|
-
"LEFT JOIN fact_embeddings_vector_chunks00 vec ON vec.rowid = fer.rowid "
|
|
524
|
-
"WHERE af.profile_id=? ORDER BY fer.rowid",
|
|
525
|
-
(self.profile_id,),
|
|
526
|
-
)
|
|
527
|
-
for row in rows:
|
|
528
|
-
self._digest_row(digest, "fact_embeddings_rowids", row)
|
|
529
|
-
except sqlite3.OperationalError:
|
|
530
|
-
pass
|
|
556
|
+
for row in iter_canonical_vectors(conn, self.profile_id):
|
|
557
|
+
self._digest_row(digest, "fact_embeddings", row)
|
|
531
558
|
return digest.hexdigest()
|
|
532
559
|
|
|
533
560
|
@staticmethod
|
|
@@ -576,7 +603,9 @@ class ScaleEngineManager:
|
|
|
576
603
|
|
|
577
604
|
def _require_default_profile(self) -> None:
|
|
578
605
|
if self.profile_id != "default":
|
|
579
|
-
raise ScaleEngineError(
|
|
606
|
+
raise ScaleEngineError(
|
|
607
|
+
"Scale Engine promotion currently supports the default profile only"
|
|
608
|
+
)
|
|
580
609
|
|
|
581
610
|
def _require_canonical_db(self) -> None:
|
|
582
611
|
if not self.db_path.exists():
|
|
@@ -310,7 +310,7 @@ def run_store(
|
|
|
310
310
|
# that fact extraction may abstract away (dates, names, specifics).
|
|
311
311
|
# This ensures BM25 and semantic search can always find the original text.
|
|
312
312
|
# V3.3.12: Extract entities from verbatim content so entity channel + temporal
|
|
313
|
-
# channel can find it (was entities=[] which
|
|
313
|
+
# channel can find it (was entities=[] which blinded the entity-graph and temporal signals).
|
|
314
314
|
# V3.3.20: Stronger verbatim filter — skip greetings, filler, short phrases.
|
|
315
315
|
# Verbatim facts with just "Hey! How are you?" dilute embeddings and add noise.
|
|
316
316
|
_MIN_VERBATIM_WORDS = 8
|
|
@@ -17,7 +17,7 @@ the local memories before committing to the external call.
|
|
|
17
17
|
|
|
18
18
|
PERFORMANCE
|
|
19
19
|
===========
|
|
20
|
-
Cost: ~500-800ms warm (full
|
|
20
|
+
Cost: ~500-800ms warm (full recall via SLM daemon). Fires only on
|
|
21
21
|
WebSearch and WebFetch (5-20× per typical session), so per-session overhead
|
|
22
22
|
is ~5-15s in exchange for grounded answers. NOT suitable for UserPromptSubmit
|
|
23
23
|
(too frequent — would be a perf disaster).
|
|
@@ -267,7 +267,7 @@ def _hook_definitions(include_gate: bool = False) -> dict[str, list]:
|
|
|
267
267
|
}
|
|
268
268
|
|
|
269
269
|
# v3.4.43 — default PreToolUse entry: pre-web recall on WebSearch/WebFetch.
|
|
270
|
-
# Fires `slm hook before_web` which runs a
|
|
270
|
+
# Fires `slm hook before_web` which runs a recall on the search
|
|
271
271
|
# query/URL and injects results as a system-reminder BEFORE the web call.
|
|
272
272
|
# Encourages Claude to consider local memories before paying for new web
|
|
273
273
|
# research. Independent of `include_gate` — this is value-add, not gating.
|
|
@@ -42,12 +42,13 @@ def _sqlite_emergency_recall(
|
|
|
42
42
|
native BM25 ranking via ``ORDER BY fts.rank``. This is the Mem0 / Letta
|
|
43
43
|
industry pattern — multi-process safe via SQLite WAL mode.
|
|
44
44
|
|
|
45
|
-
Quality degraded vs full
|
|
45
|
+
Quality degraded vs the full recall path (no semantic, no entity graph, no
|
|
46
46
|
temporal/spreading-activation/Hopfield) but still provides real BM25
|
|
47
47
|
math + age gate. Returns ``degraded_mode=True`` via the caller's flag.
|
|
48
48
|
|
|
49
49
|
Used ONLY when Tier-1 (full daemon recall) fails completely. Normal
|
|
50
|
-
path is full
|
|
50
|
+
path is the full five-producer fusion + entity-graph enhancement;
|
|
51
|
+
this is the fire-alarm.
|
|
51
52
|
"""
|
|
52
53
|
import re
|
|
53
54
|
|
|
@@ -186,9 +187,11 @@ def register_active_tools(server, get_engine: Callable) -> None:
|
|
|
186
187
|
permanently relevant still surface). Default: 30.
|
|
187
188
|
Set to 0 to disable the age gate entirely.
|
|
188
189
|
|
|
189
|
-
Scoring:
|
|
190
|
-
spreading_activation + hopfield)
|
|
191
|
-
|
|
190
|
+
Scoring: five candidate producers (semantic + BM25 + temporal +
|
|
191
|
+
spreading_activation + hopfield) feed RRF fusion; the entity graph then
|
|
192
|
+
applies an optional post-fusion score enhancement. Combined with
|
|
193
|
+
Ebbinghaus exponential recency decay and FSRS stability strengthening by
|
|
194
|
+
access frequency.
|
|
192
195
|
"""
|
|
193
196
|
try:
|
|
194
197
|
from superlocalmemory.hooks.rules_engine import RulesEngine
|
|
@@ -215,8 +218,9 @@ def register_active_tools(server, get_engine: Callable) -> None:
|
|
|
215
218
|
search_query = "recent important decisions"
|
|
216
219
|
|
|
217
220
|
# 2-tier recall (industry pattern: Hindsight / Zep / Supermemory):
|
|
218
|
-
# PRIMARY: full
|
|
219
|
-
# + Hopfield + spreading-activation
|
|
221
|
+
# PRIMARY: full recall via daemon — five candidate producers (semantic
|
|
222
|
+
# + BM25 + temporal + Hopfield + spreading-activation) into RRF
|
|
223
|
+
# fusion, then entity-graph post-fusion enhancement, FSRS decay.
|
|
220
224
|
# Fast because Ollama embed model is kept warm (keep_alive=-1
|
|
221
225
|
# + eager pre-warm at daemon boot).
|
|
222
226
|
# EMERGENCY: direct FTS5 BM25 (Mem0 / Letta pattern). Used ONLY when
|
|
@@ -2,10 +2,13 @@
|
|
|
2
2
|
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
3
|
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
4
|
|
|
5
|
-
"""SuperLocalMemory V3 —
|
|
5
|
+
"""SuperLocalMemory V3 — retrieval orchestration.
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
Five parallel candidate producers (semantic, BM25, temporal, spreading
|
|
8
|
+
activation, and Hopfield) feed single-pass RRF fusion; optional profile hits
|
|
9
|
+
can join that fusion input. The entity graph may then score and boost fused
|
|
10
|
+
candidates when enabled and within the recall time budget. It is not a sixth
|
|
11
|
+
parallel candidate producer. Optional cross-encoder reranking follows fusion.
|
|
9
12
|
Replaces V1's broken 10-channel triple-re-fusion pipeline.
|
|
10
13
|
|
|
11
14
|
Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
@@ -54,7 +57,12 @@ class EmbeddingProvider(Protocol):
|
|
|
54
57
|
|
|
55
58
|
|
|
56
59
|
class RetrievalEngine:
|
|
57
|
-
"""
|
|
60
|
+
"""Retrieval orchestrator: five candidate producers -> RRF fusion.
|
|
61
|
+
|
|
62
|
+
Five parallel candidate producers (semantic, BM25, temporal,
|
|
63
|
+
spreading_activation, hopfield) feed single-pass RRF fusion, followed by
|
|
64
|
+
optional cross-encoder rerank and an optional entity-graph post-fusion
|
|
65
|
+
score enhancement. Entity graph is not a sixth parallel candidate producer.
|
|
58
66
|
|
|
59
67
|
Usage::
|
|
60
68
|
engine = RetrievalEngine(db, config, channels, embedder)
|
|
@@ -627,10 +635,10 @@ class RetrievalEngine:
|
|
|
627
635
|
v3.4.53: channels run in PARALLEL via ThreadPoolExecutor. Industry
|
|
628
636
|
standard (EverMemOS, szl-recall, ContentPilot 2026): all channels
|
|
629
637
|
are independent after embedding; running them serially wastes time
|
|
630
|
-
equal to the sum of all
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
638
|
+
equal to the sum of all producer latencies. When multiple producers are
|
|
639
|
+
enabled and healthy, parallel dispatch generally bounds the producer
|
|
640
|
+
phase by the slowest submitted producer, plus serial embedding and
|
|
641
|
+
result-collection overhead.
|
|
634
642
|
"""
|
|
635
643
|
import os as _os_e
|
|
636
644
|
import time as _time_e
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
"""Ask My Memory — SSE chat endpoint.
|
|
6
6
|
|
|
7
|
-
Flow: query →
|
|
7
|
+
Flow: query → full recall (five-producer fusion) → format context → LLM stream → SSE
|
|
8
8
|
Mode A: No LLM, returns formatted retrieval results.
|
|
9
9
|
Mode B: Ollama local streaming via /api/chat.
|
|
10
10
|
Mode C: Cloud LLM streaming (OpenAI-compatible).
|
|
@@ -315,7 +315,7 @@ async def _stream_openai_compat(
|
|
|
315
315
|
# ── Retrieval Helper ─────────────────────────────────────────────
|
|
316
316
|
|
|
317
317
|
def _recall_memories(query: str, limit: int) -> list:
|
|
318
|
-
"""Run
|
|
318
|
+
"""Run full recall via WorkerPool (synchronous, runs in executor)."""
|
|
319
319
|
from superlocalmemory.core.worker_pool import WorkerPool
|
|
320
320
|
pool = WorkerPool.shared()
|
|
321
321
|
result = pool.recall(query, limit=limit)
|
|
@@ -277,7 +277,7 @@ def _emit_event(
|
|
|
277
277
|
|
|
278
278
|
|
|
279
279
|
# v3.4.53: Limit concurrent full (non-fast) recalls. Without this, N parallel
|
|
280
|
-
# /recall calls spawn N ×
|
|
280
|
+
# /recall calls spawn N × full-recall threads → Ollama serialises, reranker
|
|
281
281
|
# lock queues, and total wall time is N × single-recall-time. 3 concurrent
|
|
282
282
|
# full recalls gives parallelism benefit without resource oversaturation.
|
|
283
283
|
import asyncio as _asyncio
|
|
@@ -775,7 +775,7 @@ async def lifespan(application: FastAPI):
|
|
|
775
775
|
# v3.4.52: Ensure covering indexes for SpreadingActivation queries.
|
|
776
776
|
# SQLite 3.45+ streaming merge (UNION ALL + ORDER BY + LIMIT) uses
|
|
777
777
|
# these to seek directly to top-K rows per subquery, avoiding a
|
|
778
|
-
# full sort. Without them full
|
|
778
|
+
# full sort. Without them full recall takes 7-10s on
|
|
779
779
|
# >1M edges (the SpreadingActivation 4-UNION query disk-sorts every
|
|
780
780
|
# node's neighbor list on each call). With them: sub-second.
|
|
781
781
|
try:
|
|
@@ -835,7 +835,7 @@ async def lifespan(application: FastAPI):
|
|
|
835
835
|
logger.warning("Embedding warmup failed: %s", exc)
|
|
836
836
|
|
|
837
837
|
def _warmup_recall():
|
|
838
|
-
"""v3.4.62: Fire a full
|
|
838
|
+
"""v3.4.62: Fire a full recall after embedding warms up.
|
|
839
839
|
|
|
840
840
|
Loads the graph_edges table (347K rows, ~100 MB) into the SQLite
|
|
841
841
|
page cache. Without this, the first user query takes 15-24s because
|
|
@@ -853,7 +853,7 @@ async def lifespan(application: FastAPI):
|
|
|
853
853
|
try:
|
|
854
854
|
t0 = _t.monotonic()
|
|
855
855
|
# Fire 2 warmup queries: one to load the graph page cache,
|
|
856
|
-
# second to warm the reranker subprocess + all
|
|
856
|
+
# second to warm the reranker subprocess + all producers.
|
|
857
857
|
# Without this, dashboard POST /api/search hits 11s cold.
|
|
858
858
|
for wq in ("memory recall performance", "context injection retrieval"):
|
|
859
859
|
engine.recall(wq, limit=5)
|
|
@@ -1710,7 +1710,20 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
1710
1710
|
content={"error": str(_identity_exc.detail)},
|
|
1711
1711
|
)
|
|
1712
1712
|
raise
|
|
1713
|
-
|
|
1713
|
+
# v3.7.6 (#71/#73/#74): require_http_mutation_actor above is the
|
|
1714
|
+
# authoritative write-auth boundary — it accepts the daemon
|
|
1715
|
+
# capability, the dashboard install token, a matching X-SLM-API-Key,
|
|
1716
|
+
# or an uncredentialed loopback caller, and fails closed for everyone
|
|
1717
|
+
# else. The legacy check_api_key gate only understands X-SLM-API-Key,
|
|
1718
|
+
# so running it as a second gate 401'd write paths that stage 1 had
|
|
1719
|
+
# already authorized: capability-authenticated daemon write-throughs
|
|
1720
|
+
# (MCP `remember`, #71) and install-token dashboard writes / config
|
|
1721
|
+
# tests (#73/#74) whenever an api_key file exists. Only fall back to
|
|
1722
|
+
# check_api_key when the mutation-actor gate did not run — i.e. for
|
|
1723
|
+
# non-write, non-recall requests, where it is a no-op for reads.
|
|
1724
|
+
if not requires_mutation_actor and not check_api_key(
|
|
1725
|
+
headers, is_write=is_write
|
|
1726
|
+
):
|
|
1714
1727
|
from fastapi.responses import JSONResponse
|
|
1715
1728
|
return JSONResponse(
|
|
1716
1729
|
status_code=401,
|
|
@@ -298,6 +298,36 @@ def _apply_single(
|
|
|
298
298
|
_, _, logged_hash, _, status = existing
|
|
299
299
|
if status == "complete":
|
|
300
300
|
if logged_hash != ddl_hash:
|
|
301
|
+
# v3.7.6 (#70): a complete migration whose logged DDL hash no
|
|
302
|
+
# longer matches the current text is only a real failure if the
|
|
303
|
+
# schema it guarantees is actually absent. Historically-benign
|
|
304
|
+
# DDL edits (e.g. M002's V3.4.21 <-> S9-W1 variants that build the
|
|
305
|
+
# identical end-state) would otherwise brick readiness forever on
|
|
306
|
+
# upgrade. Consult the migration's own verify(); if the schema is
|
|
307
|
+
# in place, reconcile the log to the current hash and treat as
|
|
308
|
+
# already-applied instead of failing the daemon into permanent
|
|
309
|
+
# not_ready. Absent/failing verify keeps the hard failure.
|
|
310
|
+
mod = _MODULES.get(migration.name)
|
|
311
|
+
verify_fn = (
|
|
312
|
+
getattr(mod, "verify", None) if mod is not None else None
|
|
313
|
+
)
|
|
314
|
+
if verify_fn is not None:
|
|
315
|
+
try:
|
|
316
|
+
if verify_fn(conn):
|
|
317
|
+
if not dry_run:
|
|
318
|
+
try:
|
|
319
|
+
_upsert_log(
|
|
320
|
+
conn, migration.name, ddl_hash, "complete"
|
|
321
|
+
)
|
|
322
|
+
except sqlite3.Error: # pragma: no cover
|
|
323
|
+
pass
|
|
324
|
+
return (
|
|
325
|
+
"skipped",
|
|
326
|
+
"drift reconciled via verify — schema present, "
|
|
327
|
+
"log re-hashed to current DDL",
|
|
328
|
+
)
|
|
329
|
+
except sqlite3.Error: # pragma: no cover
|
|
330
|
+
pass
|
|
301
331
|
detail = (
|
|
302
332
|
f"DDL drift detected for {migration.name}: "
|
|
303
333
|
f"logged={logged_hash[:8]}... current={ddl_hash[:8]}..."
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
|
|
4
|
+
"""Canonical sqlite-vec export contract for derived vector projections."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import sqlite3
|
|
9
|
+
from collections.abc import Iterator
|
|
10
|
+
from typing import TypeAlias
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class CanonicalVectorError(RuntimeError):
|
|
14
|
+
"""Canonical sqlite-vec data cannot be read without risking data loss."""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
CanonicalVector: TypeAlias = tuple[int, str, str, str, bytes]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def load_sqlite_vec_extension(conn: sqlite3.Connection) -> None:
|
|
21
|
+
"""Load sqlite-vec for this connection or fail the migration explicitly."""
|
|
22
|
+
try:
|
|
23
|
+
import sqlite_vec
|
|
24
|
+
|
|
25
|
+
conn.enable_load_extension(True)
|
|
26
|
+
sqlite_vec.load(conn)
|
|
27
|
+
except Exception as exc:
|
|
28
|
+
raise CanonicalVectorError(
|
|
29
|
+
"sqlite-vec could not be loaded; refusing to treat canonical vectors as empty"
|
|
30
|
+
) from exc
|
|
31
|
+
finally:
|
|
32
|
+
try:
|
|
33
|
+
conn.enable_load_extension(False)
|
|
34
|
+
except (AttributeError, sqlite3.Error):
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def count_canonical_vectors(conn: sqlite3.Connection, profile_id: str) -> int:
|
|
39
|
+
"""Count profile-scoped vectors that have a canonical fact identity."""
|
|
40
|
+
if not _canonical_vector_table_exists(conn):
|
|
41
|
+
return 0
|
|
42
|
+
load_sqlite_vec_extension(conn)
|
|
43
|
+
return _validate_canonical_vector_contract(conn, profile_id)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def iter_canonical_vectors(
|
|
47
|
+
conn: sqlite3.Connection, profile_id: str
|
|
48
|
+
) -> Iterator[CanonicalVector]:
|
|
49
|
+
"""Yield supported vec0 rows joined to canonical fact identity and lifecycle."""
|
|
50
|
+
if not _canonical_vector_table_exists(conn):
|
|
51
|
+
return
|
|
52
|
+
load_sqlite_vec_extension(conn)
|
|
53
|
+
_validate_canonical_vector_contract(conn, profile_id)
|
|
54
|
+
try:
|
|
55
|
+
rows = conn.execute(
|
|
56
|
+
"SELECT fe.rowid, em.fact_id, COALESCE(af.lifecycle, 'active'), "
|
|
57
|
+
"af.profile_id, fe.embedding "
|
|
58
|
+
"FROM fact_embeddings fe "
|
|
59
|
+
"JOIN embedding_metadata em ON em.vec_rowid = fe.rowid "
|
|
60
|
+
"JOIN atomic_facts af ON af.fact_id = em.fact_id "
|
|
61
|
+
"AND af.profile_id = em.profile_id "
|
|
62
|
+
"WHERE af.profile_id = ? AND fe.profile_id = af.profile_id "
|
|
63
|
+
"ORDER BY fe.rowid",
|
|
64
|
+
(profile_id,),
|
|
65
|
+
)
|
|
66
|
+
for rowid, fact_id, lifecycle, row_profile_id, blob in rows:
|
|
67
|
+
if not isinstance(blob, bytes):
|
|
68
|
+
raise CanonicalVectorError(
|
|
69
|
+
f"canonical vector {rowid} is not a float32 blob"
|
|
70
|
+
)
|
|
71
|
+
yield int(rowid), str(fact_id), str(lifecycle), str(row_profile_id), blob
|
|
72
|
+
except sqlite3.Error as exc:
|
|
73
|
+
raise CanonicalVectorError(
|
|
74
|
+
"canonical sqlite-vec rows are unreadable; refusing a partial projection"
|
|
75
|
+
) from exc
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _validate_canonical_vector_contract(
|
|
79
|
+
conn: sqlite3.Connection, profile_id: str
|
|
80
|
+
) -> int:
|
|
81
|
+
"""Prove every profile-owned metadata row maps to one vec0 partition row."""
|
|
82
|
+
try:
|
|
83
|
+
misowned = int(
|
|
84
|
+
conn.execute(
|
|
85
|
+
"SELECT COUNT(*) FROM embedding_metadata em "
|
|
86
|
+
"LEFT JOIN atomic_facts af ON af.fact_id = em.fact_id "
|
|
87
|
+
"WHERE (em.profile_id = ? OR af.profile_id = ?) "
|
|
88
|
+
"AND (af.fact_id IS NULL OR af.profile_id <> em.profile_id)",
|
|
89
|
+
(profile_id, profile_id),
|
|
90
|
+
).fetchone()[0]
|
|
91
|
+
)
|
|
92
|
+
expected = int(
|
|
93
|
+
conn.execute(
|
|
94
|
+
"SELECT COUNT(*) FROM embedding_metadata em "
|
|
95
|
+
"JOIN atomic_facts af ON af.fact_id = em.fact_id "
|
|
96
|
+
"AND af.profile_id = em.profile_id "
|
|
97
|
+
"WHERE af.profile_id = ?",
|
|
98
|
+
(profile_id,),
|
|
99
|
+
).fetchone()[0]
|
|
100
|
+
)
|
|
101
|
+
mapped = int(
|
|
102
|
+
conn.execute(
|
|
103
|
+
"SELECT COUNT(*) FROM embedding_metadata em "
|
|
104
|
+
"JOIN atomic_facts af ON af.fact_id = em.fact_id "
|
|
105
|
+
"AND af.profile_id = em.profile_id "
|
|
106
|
+
"JOIN fact_embeddings fe ON fe.rowid = em.vec_rowid "
|
|
107
|
+
"AND fe.profile_id = af.profile_id "
|
|
108
|
+
"WHERE af.profile_id = ?",
|
|
109
|
+
(profile_id,),
|
|
110
|
+
).fetchone()[0]
|
|
111
|
+
)
|
|
112
|
+
except sqlite3.Error as exc:
|
|
113
|
+
raise CanonicalVectorError(
|
|
114
|
+
"canonical vector contract is unreadable; refusing a partial projection"
|
|
115
|
+
) from exc
|
|
116
|
+
if misowned:
|
|
117
|
+
raise CanonicalVectorError(
|
|
118
|
+
f"canonical vector ownership mismatch for {misowned} metadata row(s)"
|
|
119
|
+
)
|
|
120
|
+
if mapped != expected:
|
|
121
|
+
raise CanonicalVectorError(
|
|
122
|
+
"canonical vector mapping is incomplete: "
|
|
123
|
+
f"metadata={expected}, mapped_vec0={mapped}"
|
|
124
|
+
)
|
|
125
|
+
return expected
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _canonical_vector_table_exists(conn: sqlite3.Connection) -> bool:
|
|
129
|
+
row = conn.execute(
|
|
130
|
+
"SELECT 1 FROM sqlite_master WHERE name='fact_embeddings' "
|
|
131
|
+
"AND type='table'"
|
|
132
|
+
).fetchone()
|
|
133
|
+
if row is not None:
|
|
134
|
+
return True
|
|
135
|
+
metadata = conn.execute(
|
|
136
|
+
"SELECT 1 FROM sqlite_master WHERE name='embedding_metadata' "
|
|
137
|
+
"AND type='table'"
|
|
138
|
+
).fetchone()
|
|
139
|
+
if metadata is not None:
|
|
140
|
+
try:
|
|
141
|
+
metadata_count = int(
|
|
142
|
+
conn.execute("SELECT COUNT(*) FROM embedding_metadata").fetchone()[0]
|
|
143
|
+
)
|
|
144
|
+
except sqlite3.Error as exc:
|
|
145
|
+
raise CanonicalVectorError("embedding metadata is unreadable") from exc
|
|
146
|
+
if metadata_count:
|
|
147
|
+
raise CanonicalVectorError(
|
|
148
|
+
"embedding metadata exists but fact_embeddings is missing"
|
|
149
|
+
)
|
|
150
|
+
shadow_payload = 0
|
|
151
|
+
for table in ("fact_embeddings_rowids", "fact_embeddings_vector_chunks00"):
|
|
152
|
+
exists = conn.execute(
|
|
153
|
+
"SELECT 1 FROM sqlite_master WHERE name=? AND type='table'",
|
|
154
|
+
(table,),
|
|
155
|
+
).fetchone()
|
|
156
|
+
if exists is not None:
|
|
157
|
+
try:
|
|
158
|
+
shadow_payload += int(
|
|
159
|
+
conn.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]
|
|
160
|
+
)
|
|
161
|
+
except sqlite3.Error as exc:
|
|
162
|
+
raise CanonicalVectorError(
|
|
163
|
+
"sqlite-vec shadow payload is unreadable"
|
|
164
|
+
) from exc
|
|
165
|
+
if shadow_payload:
|
|
166
|
+
raise CanonicalVectorError(
|
|
167
|
+
"sqlite-vec shadow payload exists without the fact_embeddings virtual table"
|
|
168
|
+
)
|
|
169
|
+
return False
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// SuperLocalMemory v3.4.1 — Ask My Memory Chat Interface
|
|
2
2
|
// Copyright (c) 2026 Varun Pratap Bhardwaj — AGPL-3.0-or-later
|
|
3
|
-
// SSE streaming chat grounded in
|
|
3
|
+
// SSE streaming chat grounded in multi-producer memory retrieval
|
|
4
4
|
|
|
5
5
|
// ============================================================================
|
|
6
6
|
// STATE
|
|
@@ -320,7 +320,7 @@ function _loadChatMode() {
|
|
|
320
320
|
+ '<i class="bi bi-info-circle"></i> <strong>Mode A</strong> — No LLM connected.<br>'
|
|
321
321
|
+ 'Chat returns raw memory retrieval results.<br>'
|
|
322
322
|
+ 'For AI-powered conversation, switch to <strong>Mode B</strong> (Ollama) or <strong>Mode C</strong> (Cloud) in the <strong>Settings</strong> tab.<br>'
|
|
323
|
-
+ '<br>You can also use the <strong>Recall Lab</strong> tab for full
|
|
323
|
+
+ '<br>You can also use the <strong>Recall Lab</strong> tab for full recall search.'
|
|
324
324
|
+ '</div>';
|
|
325
325
|
}
|
|
326
326
|
}
|
|
@@ -20,6 +20,8 @@ import struct
|
|
|
20
20
|
from pathlib import Path
|
|
21
21
|
from typing import Any
|
|
22
22
|
|
|
23
|
+
from superlocalmemory.storage.sqlite_vectors import iter_canonical_vectors
|
|
24
|
+
|
|
23
25
|
logger = logging.getLogger(__name__)
|
|
24
26
|
|
|
25
27
|
# Optional import
|
|
@@ -53,7 +55,11 @@ class LanceDBVectorBackend:
|
|
|
53
55
|
# Valid tier values (F-27: validated before interpolation)
|
|
54
56
|
VALID_TIERS: frozenset[str] = frozenset({"active", "warm", "cold", "archived"})
|
|
55
57
|
|
|
56
|
-
|
|
58
|
+
# Fallback embedding width when neither the caller nor an existing table
|
|
59
|
+
# specifies one. Matches the bundled nomic-embed-text-v1.5 model (768d).
|
|
60
|
+
DEFAULT_DIMENSION: int = 768
|
|
61
|
+
|
|
62
|
+
def __init__(self, db_path: str, dimension: int | None = None) -> None:
|
|
57
63
|
if not _LANCEDB_AVAILABLE:
|
|
58
64
|
raise LanceDBNotAvailable(
|
|
59
65
|
"LanceDB not installed. Run: pip install superlocalmemory[lancedb]"
|
|
@@ -61,22 +67,54 @@ class LanceDBVectorBackend:
|
|
|
61
67
|
path = Path(db_path)
|
|
62
68
|
path.mkdir(parents=True, exist_ok=True)
|
|
63
69
|
self._db_path = str(path)
|
|
70
|
+
# v3.7.6 (#72): the vector width is configurable so custom embedding
|
|
71
|
+
# endpoints (e.g. Qwen3-Embedding at 1024d) no longer collide with a
|
|
72
|
+
# hardcoded 768d schema/decode. An existing table's on-disk width always
|
|
73
|
+
# wins over the requested value to keep already-materialized stores
|
|
74
|
+
# readable after a config change.
|
|
75
|
+
self._dimension = int(dimension) if dimension else self.DEFAULT_DIMENSION
|
|
64
76
|
self._db = lancedb.connect(self._db_path) # type: ignore[union-attr]
|
|
65
77
|
self._table = self._open_or_create_table()
|
|
66
78
|
|
|
79
|
+
@property
|
|
80
|
+
def dimension(self) -> int:
|
|
81
|
+
"""Effective vector width used for this backend's schema and decode."""
|
|
82
|
+
return self._dimension
|
|
83
|
+
|
|
67
84
|
def _open_or_create_table(self):
|
|
68
|
-
"""Open existing table or create
|
|
85
|
+
"""Open existing table (adopting its width) or create one at self._dimension."""
|
|
69
86
|
try:
|
|
70
|
-
|
|
87
|
+
table = self._db.open_table("embeddings")
|
|
71
88
|
except Exception:
|
|
72
89
|
import pyarrow as pa
|
|
73
90
|
schema = pa.schema([
|
|
74
91
|
pa.field("fact_id", pa.string(), nullable=False),
|
|
75
|
-
pa.field(
|
|
92
|
+
pa.field(
|
|
93
|
+
"vector",
|
|
94
|
+
pa.list_(pa.float32(), list_size=self._dimension),
|
|
95
|
+
nullable=False,
|
|
96
|
+
),
|
|
76
97
|
pa.field("tier", pa.string(), nullable=False),
|
|
77
98
|
pa.field("profile_id", pa.string(), nullable=False),
|
|
78
99
|
])
|
|
79
100
|
return self._db.create_table("embeddings", schema=schema)
|
|
101
|
+
# Adopt the persisted width so decode/validation matches what is stored.
|
|
102
|
+
existing = self._table_vector_width(table)
|
|
103
|
+
if existing:
|
|
104
|
+
self._dimension = existing
|
|
105
|
+
return table
|
|
106
|
+
|
|
107
|
+
@staticmethod
|
|
108
|
+
def _table_vector_width(table) -> int | None:
|
|
109
|
+
"""Best-effort read of the persisted vector list width, or None."""
|
|
110
|
+
try:
|
|
111
|
+
field = table.schema.field("vector")
|
|
112
|
+
list_size = getattr(field.type, "list_size", None)
|
|
113
|
+
if isinstance(list_size, int) and list_size > 0:
|
|
114
|
+
return list_size
|
|
115
|
+
except Exception: # pragma: no cover — schema introspection is best-effort
|
|
116
|
+
logger.debug("Could not read persisted vector width", exc_info=True)
|
|
117
|
+
return None
|
|
80
118
|
|
|
81
119
|
def close(self) -> None:
|
|
82
120
|
"""Release this backend's native table and connection references."""
|
|
@@ -178,80 +216,49 @@ class LanceDBVectorBackend:
|
|
|
178
216
|
) -> int:
|
|
179
217
|
"""Export embeddings from sqlite-vec → LanceDB.
|
|
180
218
|
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
219
|
+
Reads the supported vec0 virtual table and joins row IDs through
|
|
220
|
+
``embedding_metadata``. Shadow-table layouts are sqlite-vec internals
|
|
221
|
+
and must not be treated as a stable migration API.
|
|
184
222
|
|
|
185
223
|
Returns number of vectors imported.
|
|
186
224
|
"""
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
except sqlite3.OperationalError:
|
|
193
|
-
logger.warning("fact_embeddings_rowids not found — no vectors to import")
|
|
194
|
-
return 0
|
|
195
|
-
|
|
196
|
-
# Get tiers
|
|
197
|
-
tier_map: dict[str, str] = {}
|
|
198
|
-
profile_map: dict[str, str] = {}
|
|
199
|
-
try:
|
|
200
|
-
for row in conn.execute(
|
|
201
|
-
"SELECT fact_id, COALESCE(lifecycle, 'active'), profile_id "
|
|
202
|
-
"FROM atomic_facts WHERE profile_id = ?",
|
|
203
|
-
(profile_id,),
|
|
204
|
-
):
|
|
205
|
-
tier_map[row[0]] = row[1]
|
|
206
|
-
profile_map[row[0]] = row[2] or "default"
|
|
207
|
-
except sqlite3.OperationalError:
|
|
208
|
-
pass
|
|
209
|
-
|
|
210
|
-
# Read vectors from sqlite-vec
|
|
211
|
-
try:
|
|
212
|
-
rows = conn.execute(
|
|
213
|
-
"SELECT rowid, vector FROM fact_embeddings_vector_chunks00"
|
|
214
|
-
).fetchall()
|
|
215
|
-
except sqlite3.OperationalError:
|
|
216
|
-
logger.warning("fact_embeddings_vector_chunks00 not found")
|
|
217
|
-
return 0
|
|
218
|
-
|
|
219
|
-
# Reconstruct and batch import
|
|
220
|
-
data = []
|
|
221
|
-
for rowid, blob in rows:
|
|
222
|
-
fact_id = row_map.get(rowid)
|
|
223
|
-
# The rowid mapping is global, but a staged Scale Engine is
|
|
224
|
-
# explicitly profile-scoped. Do not import a foreign profile by
|
|
225
|
-
# giving it a default tier/profile below.
|
|
226
|
-
if fact_id is None or fact_id not in profile_map:
|
|
227
|
-
continue
|
|
225
|
+
imported = 0
|
|
226
|
+
batch: list[tuple[str, list[float], str, str]] = []
|
|
227
|
+
for rowid, fact_id, tier, row_profile_id, blob in iter_canonical_vectors(
|
|
228
|
+
conn, profile_id
|
|
229
|
+
):
|
|
228
230
|
try:
|
|
229
231
|
vector = self._decode_vector_blob(blob)
|
|
230
232
|
except Exception as exc:
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
233
|
+
raise LanceDBError(
|
|
234
|
+
f"Invalid canonical vector for rowid {rowid}: {exc}"
|
|
235
|
+
) from exc
|
|
236
|
+
batch.append((fact_id, vector, tier, row_profile_id))
|
|
237
|
+
if len(batch) >= 256:
|
|
238
|
+
imported += self._flush_import_batch(batch)
|
|
239
|
+
batch.clear()
|
|
240
|
+
if batch:
|
|
241
|
+
imported += self._flush_import_batch(batch)
|
|
242
|
+
|
|
243
|
+
logger.info("LanceDB: imported %d vectors from sqlite-vec", imported)
|
|
244
|
+
return imported
|
|
245
|
+
|
|
246
|
+
def _flush_import_batch(
|
|
247
|
+
self, batch: list[tuple[str, list[float], str, str]]
|
|
248
|
+
) -> int:
|
|
249
|
+
"""Write one bounded-memory, single-profile canonical vector batch."""
|
|
250
|
+
by_profile: dict[str, list[tuple[str, list[float], str]]] = {}
|
|
251
|
+
for fact_id, vector, tier, profile_id in batch:
|
|
252
|
+
by_profile.setdefault(profile_id, []).append((fact_id, vector, tier))
|
|
253
|
+
imported = 0
|
|
254
|
+
for profile_id, records in by_profile.items():
|
|
255
|
+
imported += self.add_vectors(
|
|
256
|
+
[item[0] for item in records],
|
|
257
|
+
[item[1] for item in records],
|
|
258
|
+
[item[2] for item in records],
|
|
259
|
+
profile_id,
|
|
260
|
+
)
|
|
261
|
+
return imported
|
|
255
262
|
|
|
256
263
|
def _decode_vector_blob(self, blob: bytes) -> list[float]:
|
|
257
264
|
"""Decode sqlite-vec BLOB to list of floats.
|
|
@@ -259,13 +266,13 @@ class LanceDBVectorBackend:
|
|
|
259
266
|
F-33: Validates dimension and L2 norm.
|
|
260
267
|
sqlite-vec stores vectors as raw float32 little-endian bytes.
|
|
261
268
|
"""
|
|
262
|
-
expected_bytes =
|
|
269
|
+
expected_bytes = self._dimension * 4
|
|
263
270
|
if len(blob) != expected_bytes:
|
|
264
271
|
raise ValueError(
|
|
265
272
|
f"Unexpected vector blob size: {len(blob)} (expected {expected_bytes})"
|
|
266
273
|
)
|
|
267
274
|
|
|
268
|
-
vec = list(struct.unpack(f"{
|
|
275
|
+
vec = list(struct.unpack(f"{self._dimension}f", blob))
|
|
269
276
|
|
|
270
277
|
# F-33: Validate non-zero
|
|
271
278
|
norm = sum(v * v for v in vec) ** 0.5
|