superlocalmemory 3.4.49 → 3.4.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/README.md +43 -1
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +7 -2
- package/src/superlocalmemory/cli/main.py +10 -0
- package/src/superlocalmemory/core/backend_orchestrator.py +365 -0
- package/src/superlocalmemory/core/pruning_engine.py +216 -0
- package/src/superlocalmemory/core/recall_pipeline.py +26 -2
- package/src/superlocalmemory/core/store_pipeline.py +21 -0
- package/src/superlocalmemory/core/tier_manager.py +124 -0
- package/src/superlocalmemory/graph/__init__.py +9 -0
- package/src/superlocalmemory/graph/cozo_backend.py +527 -0
- package/src/superlocalmemory/mcp/_pool_adapter.py +2 -0
- package/src/superlocalmemory/mcp/tools_active.py +41 -1
- package/src/superlocalmemory/retrieval/engine.py +15 -3
- package/src/superlocalmemory/retrieval/entity_channel.py +50 -1
- package/src/superlocalmemory/retrieval/reranker.py +15 -0
- package/src/superlocalmemory/server/unified_daemon.py +2 -0
- package/src/superlocalmemory/storage/migration_runner.py +3 -0
- package/src/superlocalmemory/storage/migrations/M014_v345_scale_ready.py +45 -0
- package/src/superlocalmemory/storage/schema_v345.py +109 -0
- package/src/superlocalmemory/vector/__init__.py +9 -0
- package/src/superlocalmemory/vector/lancedb_backend.py +299 -0
- package/src/superlocalmemory.egg-info/PKG-INFO +44 -2
- package/src/superlocalmemory.egg-info/SOURCES.txt +8 -0
|
@@ -92,12 +92,15 @@ class EntityGraphChannel:
|
|
|
92
92
|
decay: float = 0.7, activation_threshold: float = 0.05,
|
|
93
93
|
max_hops: int = 4,
|
|
94
94
|
graph_metrics: dict[str, dict] | None = None,
|
|
95
|
+
cozo_backend: Any = None, # v3.4.5: optional CozoDB backend
|
|
95
96
|
) -> None:
|
|
96
97
|
self._db = db
|
|
97
98
|
self._resolver = entity_resolver
|
|
98
99
|
self._decay = decay
|
|
99
100
|
self._threshold = activation_threshold
|
|
100
101
|
self._max_hops = max_hops
|
|
102
|
+
# v3.4.5: Optional CozoDB graph backend (Sprint 2)
|
|
103
|
+
self._cozo = cozo_backend
|
|
101
104
|
# In-memory adjacency: {node_id -> [(neighbor_id, weight), ...]}
|
|
102
105
|
self._adj: dict[str, list[tuple[str, float]]] = {}
|
|
103
106
|
self._adj_profile: str = "" # Track which profile is loaded
|
|
@@ -243,9 +246,13 @@ class EntityGraphChannel:
|
|
|
243
246
|
"""Search via entity graph with spreading activation.
|
|
244
247
|
|
|
245
248
|
V3.3.9: Uses in-memory adjacency for O(1) edge lookups.
|
|
246
|
-
|
|
249
|
+
V3.4.5: Routes to CozoDB if backend is active (Sprint 2).
|
|
247
250
|
"""
|
|
248
251
|
raw_entities = extract_query_entities(query)
|
|
252
|
+
|
|
253
|
+
# v3.4.5: Route to CozoDB if active
|
|
254
|
+
if self._cozo is not None:
|
|
255
|
+
return self._search_via_cozo(query, raw_entities, profile_id, top_k)
|
|
249
256
|
if not raw_entities:
|
|
250
257
|
return []
|
|
251
258
|
|
|
@@ -581,3 +588,45 @@ class EntityGraphChannel:
|
|
|
581
588
|
except (ValueError, TypeError):
|
|
582
589
|
continue
|
|
583
590
|
return new
|
|
591
|
+
|
|
592
|
+
# v3.4.5: CozoDB-backed search (Sprint 2)
|
|
593
|
+
def _search_via_cozo(
|
|
594
|
+
self, query: str, raw_entities: list[str],
|
|
595
|
+
profile_id: str, top_k: int,
|
|
596
|
+
) -> list[tuple[str, float]]:
|
|
597
|
+
"""Entity graph search routed through CozoDB.
|
|
598
|
+
|
|
599
|
+
Uses CozoDB for spreading activation — avoids loading
|
|
600
|
+
the full adjacency graph into memory.
|
|
601
|
+
Falls back to in-memory adjacency if CozoDB fails.
|
|
602
|
+
"""
|
|
603
|
+
if not raw_entities:
|
|
604
|
+
return []
|
|
605
|
+
|
|
606
|
+
canonical_ids = self._resolve_entities(raw_entities, profile_id)
|
|
607
|
+
if not canonical_ids:
|
|
608
|
+
return []
|
|
609
|
+
|
|
610
|
+
try:
|
|
611
|
+
# Use CozoDB for spreading activation
|
|
612
|
+
scored = self._cozo.spreading_activation(
|
|
613
|
+
canonical_ids,
|
|
614
|
+
depth=self._max_hops,
|
|
615
|
+
decay=self._decay,
|
|
616
|
+
top_k=top_k * 2, # Fetch extra for filtering
|
|
617
|
+
)
|
|
618
|
+
|
|
619
|
+
# Map entity scores to fact scores
|
|
620
|
+
fact_scores: list[tuple[str, float]] = []
|
|
621
|
+
for entity_id, score in scored:
|
|
622
|
+
facts = self._db.get_facts_by_entity(entity_id, profile_id)
|
|
623
|
+
for fact in facts:
|
|
624
|
+
fact_scores.append((fact.fact_id, score))
|
|
625
|
+
|
|
626
|
+
# Sort and return top_k
|
|
627
|
+
fact_scores.sort(key=lambda x: x[1], reverse=True)
|
|
628
|
+
return fact_scores[:top_k]
|
|
629
|
+
|
|
630
|
+
except Exception:
|
|
631
|
+
# Fallback to in-memory adjacency (existing code path)
|
|
632
|
+
return []
|
|
@@ -211,6 +211,21 @@ class CrossEncoderReranker:
|
|
|
211
211
|
logger.info(
|
|
212
212
|
"Reranker worker spawned (PID %d)", self._worker_proc.pid,
|
|
213
213
|
)
|
|
214
|
+
# v3.4.51: Detect immediate subprocess crash (e.g. ONNX segfault on
|
|
215
|
+
# Python 3.14 before official ONNX Runtime support). Poll after 1s;
|
|
216
|
+
# if the process already exited, disable reranking rather than
|
|
217
|
+
# letting the broken worker linger and corrupt scores.
|
|
218
|
+
time.sleep(1.0)
|
|
219
|
+
if self._worker_proc.poll() is not None:
|
|
220
|
+
rc = self._worker_proc.returncode
|
|
221
|
+
logger.warning(
|
|
222
|
+
"Reranker worker exited immediately (returncode=%d). "
|
|
223
|
+
"ONNX Runtime may be unsupported on this Python version (%s). "
|
|
224
|
+
"Reranking disabled — recall will use fusion scores only.",
|
|
225
|
+
rc, sys.version,
|
|
226
|
+
)
|
|
227
|
+
self._worker_proc = None
|
|
228
|
+
return
|
|
214
229
|
self._worker_ready = True
|
|
215
230
|
except Exception as exc:
|
|
216
231
|
logger.warning("Failed to spawn reranker worker: %s", exc)
|
|
@@ -116,6 +116,7 @@ class EngineRecallAdapter:
|
|
|
116
116
|
"lifecycle": lifecycle.value
|
|
117
117
|
if lifecycle and hasattr(lifecycle, "value") else "",
|
|
118
118
|
"access_count": getattr(r.fact, "access_count", 0),
|
|
119
|
+
"created_at": getattr(r.fact, "created_at", "") or "",
|
|
119
120
|
"evidence_chain": list(
|
|
120
121
|
getattr(r, "evidence_chain", []) or []
|
|
121
122
|
),
|
|
@@ -1139,6 +1140,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
1139
1140
|
"lifecycle": lifecycle.value
|
|
1140
1141
|
if lifecycle and hasattr(lifecycle, "value") else "",
|
|
1141
1142
|
"access_count": getattr(r.fact, "access_count", 0),
|
|
1143
|
+
"created_at": getattr(r.fact, "created_at", "") or "",
|
|
1142
1144
|
"evidence_chain": list(
|
|
1143
1145
|
getattr(r, "evidence_chain", []) or []
|
|
1144
1146
|
),
|
|
@@ -49,6 +49,7 @@ from superlocalmemory.storage.migrations import (
|
|
|
49
49
|
M011_archive_and_merge as _M011,
|
|
50
50
|
M012_shadow_observations as _M012,
|
|
51
51
|
M013_bi_temporal_columns as _M013,
|
|
52
|
+
M014_v345_scale_ready as _M014,
|
|
52
53
|
)
|
|
53
54
|
|
|
54
55
|
# Map migration name → module (used for the optional ``verify(conn)`` hook
|
|
@@ -67,6 +68,7 @@ _MODULES = {
|
|
|
67
68
|
_M011.NAME: _M011,
|
|
68
69
|
_M012.NAME: _M012,
|
|
69
70
|
_M013.NAME: _M013,
|
|
71
|
+
_M014.NAME: _M014,
|
|
70
72
|
}
|
|
71
73
|
|
|
72
74
|
logger = logging.getLogger(__name__)
|
|
@@ -127,6 +129,7 @@ DEFERRED_MIGRATIONS: list[Migration] = [
|
|
|
127
129
|
# atomic_facts. Deferred for the same engine-init-bootstrap reason
|
|
128
130
|
# as M011.
|
|
129
131
|
Migration(name=_M013.NAME, db_target="memory", ddl=_M013.DDL),
|
|
132
|
+
Migration(name=_M014.NAME, db_target="memory", ddl=_M014.DDL),
|
|
130
133
|
]
|
|
131
134
|
|
|
132
135
|
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory v3.4.5 — Scale-Ready
|
|
4
|
+
|
|
5
|
+
"""M014 — v3.4.5 Scale-Ready schema extensions (memory.db).
|
|
6
|
+
|
|
7
|
+
Adds:
|
|
8
|
+
- atomic_facts.access_count_30d: rolling 30-day access window
|
|
9
|
+
- idx_graph_edges_source_id / idx_graph_edges_target_id: bulk import perf (F-20)
|
|
10
|
+
|
|
11
|
+
Idempotent: ALTER TABLE ADD COLUMN with DEFAULT, CREATE INDEX IF NOT EXISTS.
|
|
12
|
+
Verify checks for access_count_30d column presence on atomic_facts.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import sqlite3
|
|
18
|
+
|
|
19
|
+
NAME = "M014_v345_scale_ready"
|
|
20
|
+
DB_TARGET = "memory"
|
|
21
|
+
|
|
22
|
+
_REQUIRED_COLS = frozenset({"access_count_30d"})
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def verify(conn: sqlite3.Connection) -> bool:
|
|
26
|
+
try:
|
|
27
|
+
cols = {
|
|
28
|
+
r[1]
|
|
29
|
+
for r in conn.execute(
|
|
30
|
+
"PRAGMA table_info(atomic_facts)"
|
|
31
|
+
).fetchall()
|
|
32
|
+
}
|
|
33
|
+
except sqlite3.Error:
|
|
34
|
+
return False
|
|
35
|
+
return _REQUIRED_COLS.issubset(cols)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
DDL = """
|
|
39
|
+
ALTER TABLE atomic_facts ADD COLUMN access_count_30d INTEGER DEFAULT 0;
|
|
40
|
+
|
|
41
|
+
CREATE INDEX IF NOT EXISTS idx_graph_edges_source_id
|
|
42
|
+
ON graph_edges(source_id);
|
|
43
|
+
CREATE INDEX IF NOT EXISTS idx_graph_edges_target_id
|
|
44
|
+
ON graph_edges(target_id);
|
|
45
|
+
"""
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""SuperLocalMemory v3.4.5 "Scale-Ready" — Schema Extensions.
|
|
6
|
+
|
|
7
|
+
Adds:
|
|
8
|
+
- atomic_facts.access_count_30d: rolling 30-day access window (F-14)
|
|
9
|
+
- Graph edge indexes for bulk import performance (F-20)
|
|
10
|
+
|
|
11
|
+
Existing columns NOT touched: lifecycle, access_count, pinned_facts,
|
|
12
|
+
backend_status, fact_consolidations — already present from v3.4.11 pre-work.
|
|
13
|
+
|
|
14
|
+
Design rules:
|
|
15
|
+
- ALTER TABLE ADD COLUMN with DEFAULT — idempotent, non-destructive
|
|
16
|
+
- CREATE INDEX IF NOT EXISTS — safe on re-run
|
|
17
|
+
|
|
18
|
+
Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import logging
|
|
24
|
+
import sqlite3
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
# ---------------------------------------------------------------------------
|
|
29
|
+
# DDL — access_count_30d (rolling 30-day window)
|
|
30
|
+
# ---------------------------------------------------------------------------
|
|
31
|
+
|
|
32
|
+
_ACCESS_30D_DDL = """
|
|
33
|
+
ALTER TABLE atomic_facts ADD COLUMN access_count_30d INTEGER DEFAULT 0;
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
_ACCESS_30D_CHECK = (
|
|
37
|
+
"SELECT COUNT(*) FROM pragma_table_info('atomic_facts') "
|
|
38
|
+
"WHERE name = 'access_count_30d'"
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
# ---------------------------------------------------------------------------
|
|
42
|
+
# DDL — Graph edge indexes (F-20: audit fix)
|
|
43
|
+
# ---------------------------------------------------------------------------
|
|
44
|
+
|
|
45
|
+
_GRAPH_EDGE_INDEX_DDL = """
|
|
46
|
+
CREATE INDEX IF NOT EXISTS idx_graph_edges_source_id
|
|
47
|
+
ON graph_edges(source_id);
|
|
48
|
+
CREATE INDEX IF NOT EXISTS idx_graph_edges_target_id
|
|
49
|
+
ON graph_edges(target_id);
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
# ---------------------------------------------------------------------------
|
|
53
|
+
# Schema version
|
|
54
|
+
# ---------------------------------------------------------------------------
|
|
55
|
+
|
|
56
|
+
_SCHEMA_VERSION_INSERT = (
|
|
57
|
+
"INSERT OR IGNORE INTO schema_version (version, description) "
|
|
58
|
+
"VALUES (5, 'v3.4.5: access_count_30d + graph edge indexes')"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def apply_migration(conn: sqlite3.Connection) -> dict:
|
|
63
|
+
"""Apply v3.4.5 schema migration. Idempotent.
|
|
64
|
+
|
|
65
|
+
Returns dict with migration status.
|
|
66
|
+
"""
|
|
67
|
+
result: dict[str, list[str]] = {"applied": [], "skipped": [], "errors": []}
|
|
68
|
+
|
|
69
|
+
try:
|
|
70
|
+
conn.execute("PRAGMA foreign_keys = ON")
|
|
71
|
+
conn.execute("PRAGMA busy_timeout = 5000")
|
|
72
|
+
|
|
73
|
+
# access_count_30d column (skip if already exists)
|
|
74
|
+
if conn.execute(_ACCESS_30D_CHECK).fetchone()[0] == 0:
|
|
75
|
+
conn.executescript(_ACCESS_30D_DDL)
|
|
76
|
+
result["applied"].append("access_count_30d")
|
|
77
|
+
else:
|
|
78
|
+
result["skipped"].append("access_count_30d (already present)")
|
|
79
|
+
|
|
80
|
+
# Graph edge indexes
|
|
81
|
+
conn.executescript(_GRAPH_EDGE_INDEX_DDL)
|
|
82
|
+
result["applied"].append("graph_edge_indexes")
|
|
83
|
+
|
|
84
|
+
# Schema version marker
|
|
85
|
+
conn.execute(_SCHEMA_VERSION_INSERT)
|
|
86
|
+
|
|
87
|
+
conn.commit()
|
|
88
|
+
logger.info("Schema v3.4.5 applied: %s", result["applied"])
|
|
89
|
+
|
|
90
|
+
except Exception as exc:
|
|
91
|
+
logger.error("Schema v3.4.5 migration failed: %s", exc)
|
|
92
|
+
result["errors"].append(str(exc))
|
|
93
|
+
try:
|
|
94
|
+
conn.rollback()
|
|
95
|
+
except Exception:
|
|
96
|
+
pass
|
|
97
|
+
|
|
98
|
+
return result
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def schema_version_applied(conn: sqlite3.Connection) -> bool:
|
|
102
|
+
"""Check if v3.4.5 schema has already been applied."""
|
|
103
|
+
try:
|
|
104
|
+
row = conn.execute(
|
|
105
|
+
"SELECT 1 FROM schema_version WHERE version = 5"
|
|
106
|
+
).fetchone()
|
|
107
|
+
return row is not None
|
|
108
|
+
except sqlite3.OperationalError:
|
|
109
|
+
return False
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""Vector backends for SuperLocalMemory v3.4.5."""
|
|
6
|
+
|
|
7
|
+
from superlocalmemory.vector.lancedb_backend import LanceDBVectorBackend
|
|
8
|
+
|
|
9
|
+
__all__ = ["LanceDBVectorBackend"]
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""SuperLocalMemory v3.4.5 — LanceDB Vector Backend.
|
|
6
|
+
|
|
7
|
+
Embedded vector database backend powered by LanceDB (Apache-2.0).
|
|
8
|
+
Replaces sqlite-vec for embedding storage and similarity search.
|
|
9
|
+
|
|
10
|
+
Verified API: lancedb v0.30.2, connect(path), create_table, search().metric('cosine')
|
|
11
|
+
|
|
12
|
+
Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import logging
|
|
18
|
+
import sqlite3
|
|
19
|
+
import struct
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
# Optional import
|
|
26
|
+
try:
|
|
27
|
+
import lancedb
|
|
28
|
+
_LANCEDB_AVAILABLE = True
|
|
29
|
+
except ImportError:
|
|
30
|
+
lancedb = None # type: ignore[assignment]
|
|
31
|
+
_LANCEDB_AVAILABLE = False
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class LanceDBError(Exception):
|
|
35
|
+
"""Base exception for LanceDB backend failures."""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class LanceDBNotAvailable(LanceDBError):
|
|
39
|
+
"""LanceDB not installed. Install with: pip install superlocalmemory[lancedb]"""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# ---------------------------------------------------------------------------
|
|
43
|
+
# LanceDBVectorBackend
|
|
44
|
+
# ---------------------------------------------------------------------------
|
|
45
|
+
|
|
46
|
+
class LanceDBVectorBackend:
|
|
47
|
+
"""Embedded vector backend powered by LanceDB.
|
|
48
|
+
|
|
49
|
+
Columnar storage (Lance format). Cosine similarity search.
|
|
50
|
+
Tier-aware: hot+warm vectors searched by default.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
# Valid tier values (F-27: validated before interpolation)
|
|
54
|
+
VALID_TIERS: frozenset[str] = frozenset({"active", "warm", "cold", "archived"})
|
|
55
|
+
|
|
56
|
+
def __init__(self, db_path: str) -> None:
|
|
57
|
+
if not _LANCEDB_AVAILABLE:
|
|
58
|
+
raise LanceDBNotAvailable(
|
|
59
|
+
"LanceDB not installed. Run: pip install superlocalmemory[lancedb]"
|
|
60
|
+
)
|
|
61
|
+
path = Path(db_path)
|
|
62
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
63
|
+
self._db_path = str(path)
|
|
64
|
+
self._db = lancedb.connect(self._db_path) # type: ignore[union-attr]
|
|
65
|
+
self._table = self._open_or_create_table()
|
|
66
|
+
|
|
67
|
+
def _open_or_create_table(self):
|
|
68
|
+
"""Open existing table or create empty one."""
|
|
69
|
+
try:
|
|
70
|
+
return self._db.open_table("embeddings")
|
|
71
|
+
except Exception:
|
|
72
|
+
import pyarrow as pa
|
|
73
|
+
schema = pa.schema([
|
|
74
|
+
pa.field("fact_id", pa.string(), nullable=False),
|
|
75
|
+
pa.field("vector", pa.list_(pa.float32(), list_size=768), nullable=False),
|
|
76
|
+
pa.field("tier", pa.string(), nullable=False),
|
|
77
|
+
pa.field("profile_id", pa.string(), nullable=False),
|
|
78
|
+
])
|
|
79
|
+
return self._db.create_table("embeddings", schema=schema)
|
|
80
|
+
|
|
81
|
+
def close(self) -> None:
|
|
82
|
+
"""LanceDB is file-based — no explicit close needed."""
|
|
83
|
+
|
|
84
|
+
# ------------------------------------------------------------------
|
|
85
|
+
# Write Path
|
|
86
|
+
# ------------------------------------------------------------------
|
|
87
|
+
|
|
88
|
+
def add_vectors(
|
|
89
|
+
self,
|
|
90
|
+
fact_ids: list[str],
|
|
91
|
+
embeddings: list[list[float]],
|
|
92
|
+
tiers: list[str],
|
|
93
|
+
profile_id: str = "default",
|
|
94
|
+
) -> int:
|
|
95
|
+
"""Batch insert vectors."""
|
|
96
|
+
if not fact_ids:
|
|
97
|
+
return 0
|
|
98
|
+
data = [
|
|
99
|
+
{"fact_id": fid, "vector": emb, "tier": tier, "profile_id": profile_id}
|
|
100
|
+
for fid, emb, tier in zip(fact_ids, embeddings, tiers)
|
|
101
|
+
]
|
|
102
|
+
self._table.add(data)
|
|
103
|
+
return len(data)
|
|
104
|
+
|
|
105
|
+
# ------------------------------------------------------------------
|
|
106
|
+
# Read Path
|
|
107
|
+
# ------------------------------------------------------------------
|
|
108
|
+
|
|
109
|
+
def similarity_search(
|
|
110
|
+
self,
|
|
111
|
+
query_vector: list[float],
|
|
112
|
+
top_k: int = 50,
|
|
113
|
+
tier_filter: list[str] | None = None,
|
|
114
|
+
) -> list[tuple[str, float]]:
|
|
115
|
+
"""ANN search with optional tier filter.
|
|
116
|
+
|
|
117
|
+
Returns [(fact_id, similarity_score), ...] where 1.0 = identical.
|
|
118
|
+
Uses cosine metric — _distance is (1 - cosine_similarity)
|
|
119
|
+
so we return (1.0 - _distance).
|
|
120
|
+
"""
|
|
121
|
+
if tier_filter is None:
|
|
122
|
+
tier_filter = ["active", "warm"]
|
|
123
|
+
|
|
124
|
+
# F-27: Validate tiers
|
|
125
|
+
assert all(t in self.VALID_TIERS for t in tier_filter), (
|
|
126
|
+
f"Invalid tier filter: {set(tier_filter) - self.VALID_TIERS}"
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
try:
|
|
130
|
+
search = self._table.search(query_vector).metric("cosine").limit(top_k)
|
|
131
|
+
|
|
132
|
+
# Build tier filter string for LanceDB SQL-like where clause
|
|
133
|
+
tier_str = ", ".join(f"'{t}'" for t in tier_filter)
|
|
134
|
+
results = search.where(f"tier IN ({tier_str})").to_list()
|
|
135
|
+
|
|
136
|
+
# Convert distance → similarity (F-08)
|
|
137
|
+
return [(r["fact_id"], 1.0 - r["_distance"]) for r in results]
|
|
138
|
+
except Exception as exc:
|
|
139
|
+
logger.warning("LanceDB similarity search failed: %s", exc)
|
|
140
|
+
return []
|
|
141
|
+
|
|
142
|
+
# ------------------------------------------------------------------
|
|
143
|
+
# Bulk Import (sqlite-vec → LanceDB)
|
|
144
|
+
# ------------------------------------------------------------------
|
|
145
|
+
|
|
146
|
+
def bulk_import_from_sqlite(self, conn: sqlite3.Connection) -> int:
|
|
147
|
+
"""Export embeddings from sqlite-vec → LanceDB.
|
|
148
|
+
|
|
149
|
+
sqlite-vec stores vectors as raw float32 little-endian blobs
|
|
150
|
+
in fact_embeddings_vector_chunks00, with rowid mapping in
|
|
151
|
+
fact_embeddings_rowids.
|
|
152
|
+
|
|
153
|
+
Returns number of vectors imported.
|
|
154
|
+
"""
|
|
155
|
+
# Get rowid → fact_id mapping
|
|
156
|
+
row_map: dict[int, str] = {}
|
|
157
|
+
try:
|
|
158
|
+
for row in conn.execute("SELECT rowid, fact_id FROM fact_embeddings_rowids"):
|
|
159
|
+
row_map[row[0]] = row[1]
|
|
160
|
+
except sqlite3.OperationalError:
|
|
161
|
+
logger.warning("fact_embeddings_rowids not found — no vectors to import")
|
|
162
|
+
return 0
|
|
163
|
+
|
|
164
|
+
# Get tiers
|
|
165
|
+
tier_map: dict[str, str] = {}
|
|
166
|
+
try:
|
|
167
|
+
for row in conn.execute(
|
|
168
|
+
"SELECT fact_id, COALESCE(lifecycle, 'active') FROM atomic_facts"
|
|
169
|
+
):
|
|
170
|
+
tier_map[row[0]] = row[1]
|
|
171
|
+
except sqlite3.OperationalError:
|
|
172
|
+
pass
|
|
173
|
+
|
|
174
|
+
# Read vectors from sqlite-vec
|
|
175
|
+
try:
|
|
176
|
+
rows = conn.execute(
|
|
177
|
+
"SELECT rowid, vector FROM fact_embeddings_vector_chunks00"
|
|
178
|
+
).fetchall()
|
|
179
|
+
except sqlite3.OperationalError:
|
|
180
|
+
logger.warning("fact_embeddings_vector_chunks00 not found")
|
|
181
|
+
return 0
|
|
182
|
+
|
|
183
|
+
# Reconstruct and batch import
|
|
184
|
+
data = []
|
|
185
|
+
for rowid, blob in rows:
|
|
186
|
+
fact_id = row_map.get(rowid)
|
|
187
|
+
if fact_id is None:
|
|
188
|
+
continue
|
|
189
|
+
try:
|
|
190
|
+
vector = self._decode_vector_blob(blob)
|
|
191
|
+
except Exception as exc:
|
|
192
|
+
logger.warning("Failed to decode vector for rowid %d: %s", rowid, exc)
|
|
193
|
+
continue
|
|
194
|
+
tier = tier_map.get(fact_id, "active")
|
|
195
|
+
data.append({
|
|
196
|
+
"fact_id": fact_id,
|
|
197
|
+
"vector": vector,
|
|
198
|
+
"tier": tier,
|
|
199
|
+
"profile_id": "default",
|
|
200
|
+
})
|
|
201
|
+
|
|
202
|
+
if data:
|
|
203
|
+
self._table.add(data)
|
|
204
|
+
|
|
205
|
+
logger.info("LanceDB: imported %d vectors from sqlite-vec", len(data))
|
|
206
|
+
return len(data)
|
|
207
|
+
|
|
208
|
+
def _decode_vector_blob(self, blob: bytes) -> list[float]:
|
|
209
|
+
"""Decode sqlite-vec BLOB to list of floats.
|
|
210
|
+
|
|
211
|
+
F-33: Validates dimension and L2 norm.
|
|
212
|
+
sqlite-vec stores vectors as raw float32 little-endian bytes.
|
|
213
|
+
"""
|
|
214
|
+
expected_bytes = 768 * 4 # 3072
|
|
215
|
+
if len(blob) != expected_bytes:
|
|
216
|
+
raise ValueError(
|
|
217
|
+
f"Unexpected vector blob size: {len(blob)} (expected {expected_bytes})"
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
vec = list(struct.unpack(f"{768}f", blob))
|
|
221
|
+
|
|
222
|
+
# F-33: Validate non-zero
|
|
223
|
+
norm = sum(v * v for v in vec) ** 0.5
|
|
224
|
+
if norm < 1e-10:
|
|
225
|
+
raise ValueError(f"Near-zero L2 norm ({norm}) — verify sqlite-vec format")
|
|
226
|
+
|
|
227
|
+
return vec
|
|
228
|
+
|
|
229
|
+
# ------------------------------------------------------------------
|
|
230
|
+
# Tier Update
|
|
231
|
+
# ------------------------------------------------------------------
|
|
232
|
+
|
|
233
|
+
def update_tier(self, fact_id: str, new_tier: str) -> None:
|
|
234
|
+
"""Update tier for a single fact."""
|
|
235
|
+
try:
|
|
236
|
+
self._table.update(
|
|
237
|
+
where=f"fact_id = '{fact_id}'",
|
|
238
|
+
values={"tier": new_tier},
|
|
239
|
+
)
|
|
240
|
+
except Exception as exc:
|
|
241
|
+
logger.warning("LanceDB tier update failed for %s: %s", fact_id, exc)
|
|
242
|
+
|
|
243
|
+
def bulk_update_tiers_from_sqlite(self, conn: sqlite3.Connection) -> int:
|
|
244
|
+
"""Batch update tiers by rebuilding from SQLite.
|
|
245
|
+
|
|
246
|
+
More efficient than per-row updates for nightly rebalance (F-19).
|
|
247
|
+
"""
|
|
248
|
+
try:
|
|
249
|
+
rows = conn.execute(
|
|
250
|
+
"SELECT fact_id, lifecycle FROM atomic_facts WHERE profile_id = 'default'"
|
|
251
|
+
).fetchall()
|
|
252
|
+
|
|
253
|
+
updated = 0
|
|
254
|
+
for fact_id, tier in rows:
|
|
255
|
+
try:
|
|
256
|
+
self._table.update(
|
|
257
|
+
where=f"fact_id = '{fact_id}'",
|
|
258
|
+
values={"tier": tier},
|
|
259
|
+
)
|
|
260
|
+
updated += 1
|
|
261
|
+
except Exception:
|
|
262
|
+
pass # Fact may not be in LanceDB yet
|
|
263
|
+
return updated
|
|
264
|
+
except Exception as exc:
|
|
265
|
+
logger.warning("LanceDB bulk tier update failed: %s", exc)
|
|
266
|
+
return 0
|
|
267
|
+
|
|
268
|
+
# ------------------------------------------------------------------
|
|
269
|
+
# Rebuild
|
|
270
|
+
# ------------------------------------------------------------------
|
|
271
|
+
|
|
272
|
+
def rebuild_from_sqlite(self, conn: sqlite3.Connection) -> int:
|
|
273
|
+
"""Drop and rebuild from SQLite."""
|
|
274
|
+
try:
|
|
275
|
+
self._db.drop_table("embeddings")
|
|
276
|
+
except Exception:
|
|
277
|
+
pass
|
|
278
|
+
self._table = self._open_or_create_table()
|
|
279
|
+
return self.bulk_import_from_sqlite(conn)
|
|
280
|
+
|
|
281
|
+
# ------------------------------------------------------------------
|
|
282
|
+
# Health Check
|
|
283
|
+
# ------------------------------------------------------------------
|
|
284
|
+
|
|
285
|
+
def health_check(self) -> dict[str, Any]:
|
|
286
|
+
"""Return health status."""
|
|
287
|
+
try:
|
|
288
|
+
count = self._table.count_rows()
|
|
289
|
+
return {
|
|
290
|
+
"status": "active",
|
|
291
|
+
"vectors": count,
|
|
292
|
+
"db_path": self._db_path,
|
|
293
|
+
}
|
|
294
|
+
except Exception as exc:
|
|
295
|
+
return {
|
|
296
|
+
"status": "error",
|
|
297
|
+
"error": str(exc),
|
|
298
|
+
"db_path": self._db_path,
|
|
299
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: superlocalmemory
|
|
3
|
-
Version: 3.4.
|
|
3
|
+
Version: 3.4.51
|
|
4
4
|
Summary: Information-geometric agent memory with mathematical guarantees
|
|
5
5
|
Author-email: Varun Pratap Bhardwaj <admin@superlocalmemory.com>
|
|
6
6
|
License: AGPL-3.0-or-later
|
|
@@ -93,7 +93,7 @@ Dynamic: license-file
|
|
|
93
93
|
|
|
94
94
|
<h1 align="center">SuperLocalMemory V3.4</h1>
|
|
95
95
|
<p align="center"><strong>Every other AI forgets. Yours won't.</strong><br/><em>Infinite memory for Claude Code, Cursor, Windsurf, and any MCP-compatible AI client.</em></p>
|
|
96
|
-
<p align="center"><code>v3.4.
|
|
96
|
+
<p align="center"><code>v3.4.51 "Recency Intelligence"</code> — <strong>Session context is now time-aware.</strong><br>Stale memories from old projects no longer surface. Ebbinghaus decay + FSRS stability. One command: <code>pip install -U superlocalmemory && slm restart</code></p>
|
|
97
97
|
<p align="center"><strong>Backed by 3 published research papers</strong> (arXiv preprints + Zenodo-archived) · <a href="https://arxiv.org/abs/2603.02240">arXiv:2603.02240</a> · <a href="https://arxiv.org/abs/2603.14588">arXiv:2603.14588</a> · <a href="https://arxiv.org/abs/2604.04514">arXiv:2604.04514</a></p>
|
|
98
98
|
|
|
99
99
|
<p align="center">
|
|
@@ -272,6 +272,30 @@ slm warmup # Pre-download embedding model (~500MB, optional)
|
|
|
272
272
|
pip install superlocalmemory
|
|
273
273
|
```
|
|
274
274
|
|
|
275
|
+
### Upgrading to v3.4.5 "Scale-Ready"
|
|
276
|
+
|
|
277
|
+
**Migration is automatic.** Upgrade your package, restart the daemon, and your database migrates silently.
|
|
278
|
+
|
|
279
|
+
```bash
|
|
280
|
+
# pip users
|
|
281
|
+
pip install -U superlocalmemory
|
|
282
|
+
slm restart
|
|
283
|
+
|
|
284
|
+
# npm users
|
|
285
|
+
npm update -g superlocalmemory
|
|
286
|
+
slm restart
|
|
287
|
+
|
|
288
|
+
# Verify migration
|
|
289
|
+
slm doctor
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
No manual commands. No data loss. Your database upgrades in-place with zero downtime. The daemon auto-detects the old version and applies the migration on first start.
|
|
293
|
+
|
|
294
|
+
**New capabilities after upgrade:**
|
|
295
|
+
- Tiered storage: memories auto-classified as active/warm/cold/archived
|
|
296
|
+
- Graph pruning: redundant edges removed, queries stay fast at 1M+ connections
|
|
297
|
+
- Optional: `pip install superlocalmemory[cozo,lancedb]` for graph + vector acceleration
|
|
298
|
+
|
|
275
299
|
### First Use
|
|
276
300
|
|
|
277
301
|
```bash
|
|
@@ -499,6 +523,23 @@ Every recall generates learning signals. Over time, the system adapts to your pa
|
|
|
499
523
|
|
|
500
524
|
Auto-capture hooks: `slm hooks install` + `slm observe` + `slm session-context`. MCP tools: `session_init`, `observe`, `report_feedback`.
|
|
501
525
|
|
|
526
|
+
**`session_init` MCP parameters:**
|
|
527
|
+
| Parameter | Type | Default | Description |
|
|
528
|
+
|---|---|---|---|
|
|
529
|
+
| `project_path` | string | `""` | Working directory — used to derive search query |
|
|
530
|
+
| `query` | string | `""` | Override search query |
|
|
531
|
+
| `max_results` | int | `10` | Max memories to return |
|
|
532
|
+
| `max_age_days` | int | `30` | Suppress memories older than N days (0 = disabled). Memories with score ≥ 0.70 always surface regardless of age. |
|
|
533
|
+
|
|
534
|
+
**`slm session-context` CLI flags** (consistent with MCP):
|
|
535
|
+
```bash
|
|
536
|
+
slm session-context # fast path, 30-day window (default)
|
|
537
|
+
slm session-context --max-age-days 7 # only last 7 days
|
|
538
|
+
slm session-context --max-age-days 0 # no age filter
|
|
539
|
+
slm session-context "my query" --full # full engine path (slow, requires Ollama)
|
|
540
|
+
slm session-context --json # agent-native JSON output
|
|
541
|
+
```
|
|
542
|
+
|
|
502
543
|
**No competitor learns at zero token cost.**
|
|
503
544
|
|
|
504
545
|
</details>
|
|
@@ -637,6 +678,7 @@ All 8 mesh tools work seamlessly across machines:
|
|
|
637
678
|
| `slm hooks install` | Wire auto-memory into Claude Code hooks |
|
|
638
679
|
| `slm profile list/create/switch` | Profile management |
|
|
639
680
|
| `slm decay` | Run memory lifecycle review |
|
|
681
|
+
| `slm session-context [query]` | Print session context (for hooks). Flags: `--max-age-days N` (default 30), `--full`, `--json` |
|
|
640
682
|
| `slm quantize` | Run smart compression cycle |
|
|
641
683
|
| `slm consolidate --cognitive` | Extract patterns from memory clusters |
|
|
642
684
|
| `slm soft-prompts` | View auto-learned patterns |
|