superlocalmemory 3.4.49 → 3.4.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -0
- package/README.md +43 -1
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +7 -2
- package/src/superlocalmemory/cli/main.py +10 -0
- package/src/superlocalmemory/core/backend_orchestrator.py +365 -0
- package/src/superlocalmemory/core/ollama_embedder.py +14 -4
- package/src/superlocalmemory/core/pruning_engine.py +216 -0
- package/src/superlocalmemory/core/recall_pipeline.py +26 -2
- package/src/superlocalmemory/core/store_pipeline.py +21 -0
- package/src/superlocalmemory/core/tier_manager.py +124 -0
- package/src/superlocalmemory/graph/__init__.py +9 -0
- package/src/superlocalmemory/graph/cozo_backend.py +527 -0
- package/src/superlocalmemory/mcp/_daemon_proxy.py +1 -1
- package/src/superlocalmemory/mcp/_pool_adapter.py +2 -0
- package/src/superlocalmemory/mcp/tools_active.py +141 -1
- package/src/superlocalmemory/retrieval/engine.py +15 -3
- package/src/superlocalmemory/retrieval/entity_channel.py +50 -1
- package/src/superlocalmemory/retrieval/reranker.py +15 -0
- package/src/superlocalmemory/retrieval/spreading_activation.py +5 -2
- package/src/superlocalmemory/server/unified_daemon.py +73 -5
- package/src/superlocalmemory/storage/migration_runner.py +3 -0
- package/src/superlocalmemory/storage/migrations/M014_v345_scale_ready.py +45 -0
- package/src/superlocalmemory/storage/schema_v345.py +109 -0
- package/src/superlocalmemory/vector/__init__.py +9 -0
- package/src/superlocalmemory/vector/lancedb_backend.py +299 -0
- package/src/superlocalmemory.egg-info/PKG-INFO +44 -2
- package/src/superlocalmemory.egg-info/SOURCES.txt +8 -0
|
@@ -18,6 +18,7 @@ from __future__ import annotations
|
|
|
18
18
|
|
|
19
19
|
import logging
|
|
20
20
|
import os
|
|
21
|
+
import sqlite3
|
|
21
22
|
from pathlib import Path
|
|
22
23
|
from typing import Callable
|
|
23
24
|
|
|
@@ -27,6 +28,81 @@ MEMORY_DIR = Path.home() / ".superlocalmemory"
|
|
|
27
28
|
DB_PATH = MEMORY_DIR / "memory.db"
|
|
28
29
|
|
|
29
30
|
|
|
31
|
+
def _sqlite_emergency_recall(
|
|
32
|
+
query: str, limit: int, profile_id: str = "default",
|
|
33
|
+
max_age_days: int = 30,
|
|
34
|
+
) -> "PoolRecallResponse":
|
|
35
|
+
"""Emergency fallback: direct SQLite FTS5 BM25 when daemon is unreachable.
|
|
36
|
+
|
|
37
|
+
Uses the same ``atomic_facts_fts`` virtual table the daemon uses, with
|
|
38
|
+
native BM25 ranking via ``ORDER BY fts.rank``. This is the Mem0 / Letta
|
|
39
|
+
industry pattern — multi-process safe via SQLite WAL mode.
|
|
40
|
+
|
|
41
|
+
Quality degraded vs full 6-channel (no semantic, no entity graph, no
|
|
42
|
+
temporal/spreading-activation/Hopfield) but still provides real BM25
|
|
43
|
+
math + age gate. Returns ``degraded_mode=True`` via the caller's flag.
|
|
44
|
+
|
|
45
|
+
Used ONLY when Tier-1 (full daemon recall) fails completely. Normal
|
|
46
|
+
path is full 6-channel; this is the fire-alarm.
|
|
47
|
+
"""
|
|
48
|
+
from superlocalmemory.mcp._pool_adapter import PoolFact, PoolRecallItem, PoolRecallResponse
|
|
49
|
+
import re
|
|
50
|
+
try:
|
|
51
|
+
# FTS5 MATCH syntax: tokenize the query, drop special characters
|
|
52
|
+
# that confuse the parser (/, :, ., etc), and join with OR for
|
|
53
|
+
# broadest matching. Wrap each term in quotes to escape any
|
|
54
|
+
# remaining special-meaning chars.
|
|
55
|
+
tokens = re.findall(r"[A-Za-z0-9]+", query)
|
|
56
|
+
tokens = [t for t in tokens if len(t) >= 2]
|
|
57
|
+
if not tokens:
|
|
58
|
+
return PoolRecallResponse()
|
|
59
|
+
safe_query = " OR ".join(f'"{t}"' for t in tokens)
|
|
60
|
+
age_clause = (
|
|
61
|
+
f"AND f.created_at >= datetime('now', '-{int(max_age_days)} days') "
|
|
62
|
+
if max_age_days > 0 else ""
|
|
63
|
+
)
|
|
64
|
+
conn = sqlite3.connect(str(DB_PATH), timeout=5.0)
|
|
65
|
+
try:
|
|
66
|
+
rows = conn.execute(
|
|
67
|
+
f"""SELECT f.fact_id, f.content, f.memory_id, f.created_at,
|
|
68
|
+
fts.rank AS bm25_rank
|
|
69
|
+
FROM atomic_facts_fts AS fts
|
|
70
|
+
JOIN atomic_facts AS f ON f.fact_id = fts.fact_id
|
|
71
|
+
WHERE fts.atomic_facts_fts MATCH ?
|
|
72
|
+
AND f.profile_id = ?
|
|
73
|
+
{age_clause}
|
|
74
|
+
ORDER BY fts.rank
|
|
75
|
+
LIMIT ?""",
|
|
76
|
+
(safe_query, profile_id, limit * 2),
|
|
77
|
+
).fetchall()
|
|
78
|
+
finally:
|
|
79
|
+
conn.close()
|
|
80
|
+
# FTS5 rank is negative (lower = better). Normalize to [0.3, 0.9].
|
|
81
|
+
if not rows:
|
|
82
|
+
return PoolRecallResponse()
|
|
83
|
+
ranks = [r[4] for r in rows]
|
|
84
|
+
rmin, rmax = min(ranks), max(ranks)
|
|
85
|
+
rng = (rmax - rmin) or 1.0
|
|
86
|
+
items = [
|
|
87
|
+
PoolRecallItem(
|
|
88
|
+
fact=PoolFact(
|
|
89
|
+
fact_id=r[0] or "", content=r[1] or "",
|
|
90
|
+
memory_id=r[2] or "", created_at=r[3] or "",
|
|
91
|
+
),
|
|
92
|
+
score=round(0.3 + 0.6 * (1.0 - (r[4] - rmin) / rng), 3),
|
|
93
|
+
)
|
|
94
|
+
for r in rows
|
|
95
|
+
]
|
|
96
|
+
logger.warning(
|
|
97
|
+
"session_init: EMERGENCY FTS5 fallback (%d results). "
|
|
98
|
+
"Daemon unreachable — semantic/graph channels disabled.", len(items),
|
|
99
|
+
)
|
|
100
|
+
return PoolRecallResponse(results=items[:limit])
|
|
101
|
+
except Exception as exc:
|
|
102
|
+
logger.warning("Emergency FTS5 fallback failed: %s", exc)
|
|
103
|
+
return PoolRecallResponse()
|
|
104
|
+
|
|
105
|
+
|
|
30
106
|
def _get_agent_id(default: str = "mcp_client") -> str:
|
|
31
107
|
"""Resolve the calling agent's ID for attribution.
|
|
32
108
|
|
|
@@ -82,6 +158,7 @@ def register_active_tools(server, get_engine: Callable) -> None:
|
|
|
82
158
|
project_path: str = "",
|
|
83
159
|
query: str = "",
|
|
84
160
|
max_results: int = 10,
|
|
161
|
+
max_age_days: int = 30,
|
|
85
162
|
) -> dict:
|
|
86
163
|
"""Initialize session with relevant memory context.
|
|
87
164
|
|
|
@@ -91,6 +168,21 @@ def register_active_tools(server, get_engine: Callable) -> None:
|
|
|
91
168
|
- Learning status (signal count, ranking phase)
|
|
92
169
|
|
|
93
170
|
The AI should call this automatically before any other work.
|
|
171
|
+
|
|
172
|
+
Parameters:
|
|
173
|
+
project_path: Working directory path. Used to build the search query
|
|
174
|
+
when no explicit query is provided.
|
|
175
|
+
query: Override the search query. If omitted, derived from project_path
|
|
176
|
+
or falls back to "recent important decisions".
|
|
177
|
+
max_results: Maximum memories to return (default: 10).
|
|
178
|
+
max_age_days: Suppress memories older than this many days unless their
|
|
179
|
+
relevance score is ≥ 0.70 (architectural decisions that remain
|
|
180
|
+
permanently relevant still surface). Default: 30.
|
|
181
|
+
Set to 0 to disable the age gate entirely.
|
|
182
|
+
|
|
183
|
+
Scoring: Uses 6-channel fusion (semantic + BM25 + entity_graph + temporal +
|
|
184
|
+
spreading_activation + hopfield) with Ebbinghaus exponential recency decay
|
|
185
|
+
and FSRS stability strengthening by access frequency.
|
|
94
186
|
"""
|
|
95
187
|
try:
|
|
96
188
|
from superlocalmemory.hooks.rules_engine import RulesEngine
|
|
@@ -111,10 +203,56 @@ def register_active_tools(server, get_engine: Callable) -> None:
|
|
|
111
203
|
else:
|
|
112
204
|
search_query = "recent important decisions"
|
|
113
205
|
|
|
114
|
-
|
|
206
|
+
# 2-tier recall (industry pattern: Hindsight / Zep / Supermemory):
|
|
207
|
+
# PRIMARY: full 6-channel via daemon (semantic + BM25 + entity + temporal
|
|
208
|
+
# + Hopfield + spreading-activation, Fisher-Rao fusion, FSRS decay).
|
|
209
|
+
# Fast because Ollama embed model is kept warm (keep_alive=-1
|
|
210
|
+
# + eager pre-warm at daemon boot).
|
|
211
|
+
# EMERGENCY: direct FTS5 BM25 (Mem0 / Letta pattern). Used ONLY when
|
|
212
|
+
# daemon is completely unreachable. Returns degraded_mode=True.
|
|
213
|
+
from superlocalmemory.mcp._pool_adapter import PoolError
|
|
214
|
+
degraded_mode = False
|
|
215
|
+
try:
|
|
216
|
+
response = pool_recall(search_query, limit=max_results, fast=False)
|
|
217
|
+
except (PoolError, Exception) as exc:
|
|
218
|
+
logger.warning(
|
|
219
|
+
"session_init: daemon recall failed (%s) — using FTS5 emergency fallback. "
|
|
220
|
+
"Memory system is in DEGRADED MODE: semantic/graph channels unavailable.",
|
|
221
|
+
exc,
|
|
222
|
+
)
|
|
223
|
+
response = _sqlite_emergency_recall(
|
|
224
|
+
search_query, max_results,
|
|
225
|
+
profile_id=engine.profile_id,
|
|
226
|
+
max_age_days=max_age_days,
|
|
227
|
+
)
|
|
228
|
+
degraded_mode = True
|
|
229
|
+
|
|
230
|
+
# Age gate: suppress stale memories at session start.
|
|
231
|
+
# Memories older than max_age_days are excluded unless their score
|
|
232
|
+
# exceeds 0.7 (high-relevance architectural decisions always surface).
|
|
233
|
+
# max_age_days=0 disables the gate entirely.
|
|
234
|
+
from datetime import UTC, datetime as _dt
|
|
235
|
+
_now = _dt.now(UTC)
|
|
236
|
+
|
|
237
|
+
def _age_days(created_at_str: str) -> float:
|
|
238
|
+
if not created_at_str:
|
|
239
|
+
return 0.0
|
|
240
|
+
try:
|
|
241
|
+
created = _dt.fromisoformat(
|
|
242
|
+
created_at_str.replace("Z", "+00:00")
|
|
243
|
+
)
|
|
244
|
+
return max(0.0, (_now - created).total_seconds() / 86400.0)
|
|
245
|
+
except (ValueError, TypeError):
|
|
246
|
+
return 0.0
|
|
247
|
+
|
|
115
248
|
relevant = [
|
|
116
249
|
r for r in response.results
|
|
117
250
|
if r.score >= relevance_threshold
|
|
251
|
+
and (
|
|
252
|
+
max_age_days <= 0
|
|
253
|
+
or _age_days(r.fact.created_at) <= max_age_days
|
|
254
|
+
or r.score >= 0.7
|
|
255
|
+
)
|
|
118
256
|
]
|
|
119
257
|
|
|
120
258
|
# Build both return shapes from one recall. Calling recall twice
|
|
@@ -161,6 +299,8 @@ def register_active_tools(server, get_engine: Callable) -> None:
|
|
|
161
299
|
"context": context,
|
|
162
300
|
"memories": memories[:max_results],
|
|
163
301
|
"memory_count": len(memories),
|
|
302
|
+
"degraded_mode": degraded_mode,
|
|
303
|
+
"retrieval_mode": "emergency_fts5_bm25" if degraded_mode else "full_6_channel",
|
|
164
304
|
"learning": {
|
|
165
305
|
"feedback_signals": feedback_count,
|
|
166
306
|
"phase": 1 if feedback_count < 50 else (2 if feedback_count < 200 else 3),
|
|
@@ -683,7 +683,18 @@ class RetrievalEngine:
|
|
|
683
683
|
for ch, rk in sorted(fr.channel_ranks.items(), key=lambda x: x[1])
|
|
684
684
|
if rk < 1000
|
|
685
685
|
]
|
|
686
|
-
# Recency
|
|
686
|
+
# Recency decay: Ebbinghaus exponential + FSRS stability strengthening (v3.4.51).
|
|
687
|
+
#
|
|
688
|
+
# Base: R = e^(-λt), λ = ln(2)/S, S = effective half-life in days.
|
|
689
|
+
# FSRS v5 (Dae & Jarrett 2024): S grows with successful recall frequency.
|
|
690
|
+
# S_effective = S_base × min(2.0, 1 + 0.1 × access_count)
|
|
691
|
+
# → 0 recalls: S=30d 5 recalls: S=45d 10+ recalls: S=60d (max)
|
|
692
|
+
# Effect: frequently-recalled architectural decisions resist decay naturally;
|
|
693
|
+
# one-off session handoffs and debug notes decay at full rate.
|
|
694
|
+
#
|
|
695
|
+
# Boost range: [0.80×, 1.10×]
|
|
696
|
+
# 0d, 0acc → 1.10× 45d, 0acc → 0.91× 90d, 0acc → 0.84×
|
|
697
|
+
# 45d, 5acc → 0.95× 90d, 10acc → 0.90× (frequently used memories stay relevant)
|
|
687
698
|
age_days = 0.0
|
|
688
699
|
if fact.created_at:
|
|
689
700
|
try:
|
|
@@ -691,8 +702,9 @@ class RetrievalEngine:
|
|
|
691
702
|
age_days = max(0.0, (now - created).total_seconds() / 86400.0)
|
|
692
703
|
except (ValueError, TypeError):
|
|
693
704
|
pass
|
|
694
|
-
|
|
695
|
-
|
|
705
|
+
_access = max(0, getattr(fact, "access_count", 0) or 0)
|
|
706
|
+
_S = 30.0 * min(2.0, 1.0 + 0.1 * _access)
|
|
707
|
+
recency_boost = 0.8 + 0.3 * math.exp(-(math.log(2) / _S) * age_days)
|
|
696
708
|
|
|
697
709
|
# Content quality: penalize short/low-info facts that rank high
|
|
698
710
|
# due to BM25 name-matching (greetings like "Hey Caroline!" score high
|
|
@@ -92,12 +92,15 @@ class EntityGraphChannel:
|
|
|
92
92
|
decay: float = 0.7, activation_threshold: float = 0.05,
|
|
93
93
|
max_hops: int = 4,
|
|
94
94
|
graph_metrics: dict[str, dict] | None = None,
|
|
95
|
+
cozo_backend: Any = None, # v3.4.5: optional CozoDB backend
|
|
95
96
|
) -> None:
|
|
96
97
|
self._db = db
|
|
97
98
|
self._resolver = entity_resolver
|
|
98
99
|
self._decay = decay
|
|
99
100
|
self._threshold = activation_threshold
|
|
100
101
|
self._max_hops = max_hops
|
|
102
|
+
# v3.4.5: Optional CozoDB graph backend (Sprint 2)
|
|
103
|
+
self._cozo = cozo_backend
|
|
101
104
|
# In-memory adjacency: {node_id -> [(neighbor_id, weight), ...]}
|
|
102
105
|
self._adj: dict[str, list[tuple[str, float]]] = {}
|
|
103
106
|
self._adj_profile: str = "" # Track which profile is loaded
|
|
@@ -243,9 +246,13 @@ class EntityGraphChannel:
|
|
|
243
246
|
"""Search via entity graph with spreading activation.
|
|
244
247
|
|
|
245
248
|
V3.3.9: Uses in-memory adjacency for O(1) edge lookups.
|
|
246
|
-
|
|
249
|
+
V3.4.5: Routes to CozoDB if backend is active (Sprint 2).
|
|
247
250
|
"""
|
|
248
251
|
raw_entities = extract_query_entities(query)
|
|
252
|
+
|
|
253
|
+
# v3.4.5: Route to CozoDB if active
|
|
254
|
+
if self._cozo is not None:
|
|
255
|
+
return self._search_via_cozo(query, raw_entities, profile_id, top_k)
|
|
249
256
|
if not raw_entities:
|
|
250
257
|
return []
|
|
251
258
|
|
|
@@ -581,3 +588,45 @@ class EntityGraphChannel:
|
|
|
581
588
|
except (ValueError, TypeError):
|
|
582
589
|
continue
|
|
583
590
|
return new
|
|
591
|
+
|
|
592
|
+
# v3.4.5: CozoDB-backed search (Sprint 2)
|
|
593
|
+
def _search_via_cozo(
|
|
594
|
+
self, query: str, raw_entities: list[str],
|
|
595
|
+
profile_id: str, top_k: int,
|
|
596
|
+
) -> list[tuple[str, float]]:
|
|
597
|
+
"""Entity graph search routed through CozoDB.
|
|
598
|
+
|
|
599
|
+
Uses CozoDB for spreading activation — avoids loading
|
|
600
|
+
the full adjacency graph into memory.
|
|
601
|
+
Falls back to in-memory adjacency if CozoDB fails.
|
|
602
|
+
"""
|
|
603
|
+
if not raw_entities:
|
|
604
|
+
return []
|
|
605
|
+
|
|
606
|
+
canonical_ids = self._resolve_entities(raw_entities, profile_id)
|
|
607
|
+
if not canonical_ids:
|
|
608
|
+
return []
|
|
609
|
+
|
|
610
|
+
try:
|
|
611
|
+
# Use CozoDB for spreading activation
|
|
612
|
+
scored = self._cozo.spreading_activation(
|
|
613
|
+
canonical_ids,
|
|
614
|
+
depth=self._max_hops,
|
|
615
|
+
decay=self._decay,
|
|
616
|
+
top_k=top_k * 2, # Fetch extra for filtering
|
|
617
|
+
)
|
|
618
|
+
|
|
619
|
+
# Map entity scores to fact scores
|
|
620
|
+
fact_scores: list[tuple[str, float]] = []
|
|
621
|
+
for entity_id, score in scored:
|
|
622
|
+
facts = self._db.get_facts_by_entity(entity_id, profile_id)
|
|
623
|
+
for fact in facts:
|
|
624
|
+
fact_scores.append((fact.fact_id, score))
|
|
625
|
+
|
|
626
|
+
# Sort and return top_k
|
|
627
|
+
fact_scores.sort(key=lambda x: x[1], reverse=True)
|
|
628
|
+
return fact_scores[:top_k]
|
|
629
|
+
|
|
630
|
+
except Exception:
|
|
631
|
+
# Fallback to in-memory adjacency (existing code path)
|
|
632
|
+
return []
|
|
@@ -211,6 +211,21 @@ class CrossEncoderReranker:
|
|
|
211
211
|
logger.info(
|
|
212
212
|
"Reranker worker spawned (PID %d)", self._worker_proc.pid,
|
|
213
213
|
)
|
|
214
|
+
# v3.4.51: Detect immediate subprocess crash (e.g. ONNX segfault on
|
|
215
|
+
# Python 3.14 before official ONNX Runtime support). Poll after 1s;
|
|
216
|
+
# if the process already exited, disable reranking rather than
|
|
217
|
+
# letting the broken worker linger and corrupt scores.
|
|
218
|
+
time.sleep(1.0)
|
|
219
|
+
if self._worker_proc.poll() is not None:
|
|
220
|
+
rc = self._worker_proc.returncode
|
|
221
|
+
logger.warning(
|
|
222
|
+
"Reranker worker exited immediately (returncode=%d). "
|
|
223
|
+
"ONNX Runtime may be unsupported on this Python version (%s). "
|
|
224
|
+
"Reranking disabled — recall will use fusion scores only.",
|
|
225
|
+
rc, sys.version,
|
|
226
|
+
)
|
|
227
|
+
self._worker_proc = None
|
|
228
|
+
return
|
|
214
229
|
self._worker_ready = True
|
|
215
230
|
except Exception as exc:
|
|
216
231
|
logger.warning("Failed to spawn reranker worker: %s", exc)
|
|
@@ -57,8 +57,11 @@ class SpreadingActivationConfig:
|
|
|
57
57
|
enabled: bool = True # Ships enabled by default
|
|
58
58
|
# V3.4.40 (2026-05-09): per-node neighbor fan-out clamp.
|
|
59
59
|
# Hub nodes in dense graphs (5K+ edges) caused unbounded work per expansion.
|
|
60
|
-
# 100
|
|
61
|
-
|
|
60
|
+
# v3.4.52: reduced from 100 to 30 — GAM (ICLR 2026) shows that with
|
|
61
|
+
# covering indexes on weight DESC, 30 well-ranked neighbors provides
|
|
62
|
+
# sufficient spreading signal. Combined with streaming merge (SQLite 3.45+),
|
|
63
|
+
# this brings SpreadingActivation from 4.2s to ~60ms.
|
|
64
|
+
max_neighbors_per_node: int = 30
|
|
62
65
|
# v3.4.1: Graph intelligence integration
|
|
63
66
|
use_pagerank_bias: bool = False # Multiply propagation by target PageRank
|
|
64
67
|
community_boost: float = 0.0 # Boost same-community nodes (0.0 = disabled)
|
|
@@ -102,8 +102,8 @@ class EngineRecallAdapter:
|
|
|
102
102
|
results.append({
|
|
103
103
|
"fact_id": r.fact.fact_id,
|
|
104
104
|
"memory_id": r.fact.memory_id,
|
|
105
|
-
"content": r.fact.content[:300],
|
|
106
|
-
"source_content": memory_map.get(r.fact.memory_id, ""),
|
|
105
|
+
"content": _sanitize_json_text(r.fact.content[:300]),
|
|
106
|
+
"source_content": _sanitize_json_text(memory_map.get(r.fact.memory_id, "")),
|
|
107
107
|
"score": round(r.score, 4),
|
|
108
108
|
"confidence": round(r.confidence, 4),
|
|
109
109
|
"trust_score": round(r.trust_score, 4),
|
|
@@ -116,6 +116,7 @@ class EngineRecallAdapter:
|
|
|
116
116
|
"lifecycle": lifecycle.value
|
|
117
117
|
if lifecycle and hasattr(lifecycle, "value") else "",
|
|
118
118
|
"access_count": getattr(r.fact, "access_count", 0),
|
|
119
|
+
"created_at": getattr(r.fact, "created_at", "") or "",
|
|
119
120
|
"evidence_chain": list(
|
|
120
121
|
getattr(r, "evidence_chain", []) or []
|
|
121
122
|
),
|
|
@@ -155,6 +156,32 @@ from superlocalmemory.core.recall_gate import (
|
|
|
155
156
|
# daemon startup via engine._process_pending_memories().
|
|
156
157
|
_engine = None
|
|
157
158
|
|
|
159
|
+
# v3.4.52: Embedding model warm state. Set to True by the async pre-warm
|
|
160
|
+
# thread once Ollama has loaded the embedding model. /health reports this
|
|
161
|
+
# so MCP clients can wait for warm state before issuing recall calls.
|
|
162
|
+
_embedding_warm: bool = False
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _sanitize_json_text(text: str) -> str:
|
|
166
|
+
"""Strip control characters that break JSON serialization.
|
|
167
|
+
|
|
168
|
+
Facts ingested from agent conversations can contain raw \\n, \\r, \\t,
|
|
169
|
+
null bytes, and other ASCII control chars (0x00-0x1F) that survive
|
|
170
|
+
database round-trips but cause ``json.JSONDecodeError: Invalid control
|
|
171
|
+
character`` when FastAPI serialises the /recall response payload.
|
|
172
|
+
|
|
173
|
+
We replace them with spaces rather than dropping them so the byte
|
|
174
|
+
length is preserved and :300 truncation semantics stay predictable.
|
|
175
|
+
Python's ``str.isprintable()`` is too aggressive (it also drops
|
|
176
|
+
Unicode line separators), so we target only the ASCII control range.
|
|
177
|
+
"""
|
|
178
|
+
if not text:
|
|
179
|
+
return text
|
|
180
|
+
# Fast path: most facts are clean JSON text. Check in C before allocating.
|
|
181
|
+
if all(c >= " " or c in "\n\r\t" for c in text):
|
|
182
|
+
return text
|
|
183
|
+
return "".join(c if c >= " " or c in "\n\r\t" else " " for c in text)
|
|
184
|
+
|
|
158
185
|
|
|
159
186
|
# ---------------------------------------------------------------------------
|
|
160
187
|
# Observation debounce buffer (migrated from daemon.py)
|
|
@@ -476,6 +503,36 @@ async def lifespan(application: FastAPI):
|
|
|
476
503
|
# Set up observe buffer
|
|
477
504
|
_observe_buffer.set_engine(engine)
|
|
478
505
|
|
|
506
|
+
# v3.4.52: Ensure covering indexes for SpreadingActivation queries.
|
|
507
|
+
# SQLite 3.45+ streaming merge (UNION ALL + ORDER BY + LIMIT) uses
|
|
508
|
+
# these to seek directly to top-K rows per subquery, avoiding a
|
|
509
|
+
# full sort. Without them full 6-channel recall takes 7-10s on
|
|
510
|
+
# >1M edges (the SpreadingActivation 4-UNION query disk-sorts every
|
|
511
|
+
# node's neighbor list on each call). With them: sub-second.
|
|
512
|
+
try:
|
|
513
|
+
import sqlite3 as _sqlite3
|
|
514
|
+
_idx_conn = _sqlite3.connect(str(_memory_db))
|
|
515
|
+
_idx_conn.execute("PRAGMA journal_mode=WAL")
|
|
516
|
+
_idx_conn.execute(
|
|
517
|
+
"CREATE INDEX IF NOT EXISTS idx_edges_source_weight "
|
|
518
|
+
"ON graph_edges(profile_id, source_id, weight DESC)"
|
|
519
|
+
)
|
|
520
|
+
_idx_conn.execute(
|
|
521
|
+
"CREATE INDEX IF NOT EXISTS idx_edges_target_weight "
|
|
522
|
+
"ON graph_edges(profile_id, target_id, weight DESC)"
|
|
523
|
+
)
|
|
524
|
+
_idx_conn.execute(
|
|
525
|
+
"CREATE INDEX IF NOT EXISTS idx_assoc_source_weight "
|
|
526
|
+
"ON association_edges(profile_id, source_fact_id, weight DESC)"
|
|
527
|
+
)
|
|
528
|
+
_idx_conn.execute(
|
|
529
|
+
"CREATE INDEX IF NOT EXISTS idx_assoc_target_weight "
|
|
530
|
+
"ON association_edges(profile_id, target_fact_id, weight DESC)"
|
|
531
|
+
)
|
|
532
|
+
_idx_conn.close()
|
|
533
|
+
except Exception as _idx_exc:
|
|
534
|
+
logger.debug("SpreadingActivation covering indexes skipped: %s", _idx_exc)
|
|
535
|
+
|
|
479
536
|
# V3.4.37: Removed WorkerPool.warmup() — the recall_worker subprocess
|
|
480
537
|
# duplicated the daemon's MemoryEngine (800+ MB). QueueConsumer now
|
|
481
538
|
# uses the daemon's engine directly via EngineRecallAdapter.
|
|
@@ -491,13 +548,20 @@ async def lifespan(application: FastAPI):
|
|
|
491
548
|
# V3.4.11: Pre-warm embedding worker (load ONNX model on startup)
|
|
492
549
|
# Without this, first recall takes 60-90s for model load.
|
|
493
550
|
# Same pattern as reranker warmup above.
|
|
551
|
+
# v3.4.52: Sets module-level _embedding_warm flag so /health can
|
|
552
|
+
# report readiness. Combined with keep_alive=-1 in ollama_embedder.py
|
|
553
|
+
# this keeps the embedding model resident forever after first warm-up.
|
|
494
554
|
import threading
|
|
555
|
+
global _embedding_warm
|
|
556
|
+
_embedding_warm = False
|
|
495
557
|
def _warmup_embedder():
|
|
558
|
+
global _embedding_warm
|
|
496
559
|
try:
|
|
497
560
|
embedder = getattr(retrieval_eng, '_embedder', None) if retrieval_eng else None
|
|
498
561
|
if embedder and hasattr(embedder, 'embed'):
|
|
499
562
|
embedder.embed("warmup")
|
|
500
|
-
|
|
563
|
+
_embedding_warm = True
|
|
564
|
+
logger.info("Embedding worker pre-warmed (model resident, keep_alive=-1)")
|
|
501
565
|
except Exception as exc:
|
|
502
566
|
logger.warning("Embedding warmup failed: %s", exc)
|
|
503
567
|
threading.Thread(target=_warmup_embedder, daemon=True, name="embed-warmup").start()
|
|
@@ -1075,6 +1139,9 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
1075
1139
|
"pid": os.getpid(),
|
|
1076
1140
|
"engine": "initialized" if engine else "unavailable",
|
|
1077
1141
|
"version": getattr(application, 'version', 'unknown'),
|
|
1142
|
+
# v3.4.52: clients can poll this to wait for embedding model
|
|
1143
|
+
# readiness before issuing recall calls.
|
|
1144
|
+
"embedding_warm": _embedding_warm,
|
|
1078
1145
|
}
|
|
1079
1146
|
|
|
1080
1147
|
@application.get("/recall")
|
|
@@ -1124,8 +1191,8 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
1124
1191
|
results.append({
|
|
1125
1192
|
"fact_id": r.fact.fact_id,
|
|
1126
1193
|
"memory_id": r.fact.memory_id,
|
|
1127
|
-
"content": r.fact.content,
|
|
1128
|
-
"source_content": memory_map.get(r.fact.memory_id, ""),
|
|
1194
|
+
"content": _sanitize_json_text(r.fact.content),
|
|
1195
|
+
"source_content": _sanitize_json_text(memory_map.get(r.fact.memory_id, "")),
|
|
1129
1196
|
"score": round(r.score, 4),
|
|
1130
1197
|
"confidence": round(r.confidence, 4),
|
|
1131
1198
|
"trust_score": round(r.trust_score, 4),
|
|
@@ -1139,6 +1206,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
1139
1206
|
"lifecycle": lifecycle.value
|
|
1140
1207
|
if lifecycle and hasattr(lifecycle, "value") else "",
|
|
1141
1208
|
"access_count": getattr(r.fact, "access_count", 0),
|
|
1209
|
+
"created_at": getattr(r.fact, "created_at", "") or "",
|
|
1142
1210
|
"evidence_chain": list(
|
|
1143
1211
|
getattr(r, "evidence_chain", []) or []
|
|
1144
1212
|
),
|
|
@@ -49,6 +49,7 @@ from superlocalmemory.storage.migrations import (
|
|
|
49
49
|
M011_archive_and_merge as _M011,
|
|
50
50
|
M012_shadow_observations as _M012,
|
|
51
51
|
M013_bi_temporal_columns as _M013,
|
|
52
|
+
M014_v345_scale_ready as _M014,
|
|
52
53
|
)
|
|
53
54
|
|
|
54
55
|
# Map migration name → module (used for the optional ``verify(conn)`` hook
|
|
@@ -67,6 +68,7 @@ _MODULES = {
|
|
|
67
68
|
_M011.NAME: _M011,
|
|
68
69
|
_M012.NAME: _M012,
|
|
69
70
|
_M013.NAME: _M013,
|
|
71
|
+
_M014.NAME: _M014,
|
|
70
72
|
}
|
|
71
73
|
|
|
72
74
|
logger = logging.getLogger(__name__)
|
|
@@ -127,6 +129,7 @@ DEFERRED_MIGRATIONS: list[Migration] = [
|
|
|
127
129
|
# atomic_facts. Deferred for the same engine-init-bootstrap reason
|
|
128
130
|
# as M011.
|
|
129
131
|
Migration(name=_M013.NAME, db_target="memory", ddl=_M013.DDL),
|
|
132
|
+
Migration(name=_M014.NAME, db_target="memory", ddl=_M014.DDL),
|
|
130
133
|
]
|
|
131
134
|
|
|
132
135
|
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory v3.4.5 — Scale-Ready
|
|
4
|
+
|
|
5
|
+
"""M014 — v3.4.5 Scale-Ready schema extensions (memory.db).
|
|
6
|
+
|
|
7
|
+
Adds:
|
|
8
|
+
- atomic_facts.access_count_30d: rolling 30-day access window
|
|
9
|
+
- idx_graph_edges_source_id / idx_graph_edges_target_id: bulk import perf (F-20)
|
|
10
|
+
|
|
11
|
+
Idempotent: ALTER TABLE ADD COLUMN with DEFAULT, CREATE INDEX IF NOT EXISTS.
|
|
12
|
+
Verify checks for access_count_30d column presence on atomic_facts.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import sqlite3
|
|
18
|
+
|
|
19
|
+
NAME = "M014_v345_scale_ready"
|
|
20
|
+
DB_TARGET = "memory"
|
|
21
|
+
|
|
22
|
+
_REQUIRED_COLS = frozenset({"access_count_30d"})
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def verify(conn: sqlite3.Connection) -> bool:
|
|
26
|
+
try:
|
|
27
|
+
cols = {
|
|
28
|
+
r[1]
|
|
29
|
+
for r in conn.execute(
|
|
30
|
+
"PRAGMA table_info(atomic_facts)"
|
|
31
|
+
).fetchall()
|
|
32
|
+
}
|
|
33
|
+
except sqlite3.Error:
|
|
34
|
+
return False
|
|
35
|
+
return _REQUIRED_COLS.issubset(cols)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
DDL = """
|
|
39
|
+
ALTER TABLE atomic_facts ADD COLUMN access_count_30d INTEGER DEFAULT 0;
|
|
40
|
+
|
|
41
|
+
CREATE INDEX IF NOT EXISTS idx_graph_edges_source_id
|
|
42
|
+
ON graph_edges(source_id);
|
|
43
|
+
CREATE INDEX IF NOT EXISTS idx_graph_edges_target_id
|
|
44
|
+
ON graph_edges(target_id);
|
|
45
|
+
"""
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""SuperLocalMemory v3.4.5 "Scale-Ready" — Schema Extensions.
|
|
6
|
+
|
|
7
|
+
Adds:
|
|
8
|
+
- atomic_facts.access_count_30d: rolling 30-day access window (F-14)
|
|
9
|
+
- Graph edge indexes for bulk import performance (F-20)
|
|
10
|
+
|
|
11
|
+
Existing columns NOT touched: lifecycle, access_count, pinned_facts,
|
|
12
|
+
backend_status, fact_consolidations — already present from v3.4.11 pre-work.
|
|
13
|
+
|
|
14
|
+
Design rules:
|
|
15
|
+
- ALTER TABLE ADD COLUMN with DEFAULT — idempotent, non-destructive
|
|
16
|
+
- CREATE INDEX IF NOT EXISTS — safe on re-run
|
|
17
|
+
|
|
18
|
+
Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import logging
|
|
24
|
+
import sqlite3
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
# ---------------------------------------------------------------------------
|
|
29
|
+
# DDL — access_count_30d (rolling 30-day window)
|
|
30
|
+
# ---------------------------------------------------------------------------
|
|
31
|
+
|
|
32
|
+
_ACCESS_30D_DDL = """
|
|
33
|
+
ALTER TABLE atomic_facts ADD COLUMN access_count_30d INTEGER DEFAULT 0;
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
_ACCESS_30D_CHECK = (
|
|
37
|
+
"SELECT COUNT(*) FROM pragma_table_info('atomic_facts') "
|
|
38
|
+
"WHERE name = 'access_count_30d'"
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
# ---------------------------------------------------------------------------
|
|
42
|
+
# DDL — Graph edge indexes (F-20: audit fix)
|
|
43
|
+
# ---------------------------------------------------------------------------
|
|
44
|
+
|
|
45
|
+
_GRAPH_EDGE_INDEX_DDL = """
|
|
46
|
+
CREATE INDEX IF NOT EXISTS idx_graph_edges_source_id
|
|
47
|
+
ON graph_edges(source_id);
|
|
48
|
+
CREATE INDEX IF NOT EXISTS idx_graph_edges_target_id
|
|
49
|
+
ON graph_edges(target_id);
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
# ---------------------------------------------------------------------------
|
|
53
|
+
# Schema version
|
|
54
|
+
# ---------------------------------------------------------------------------
|
|
55
|
+
|
|
56
|
+
_SCHEMA_VERSION_INSERT = (
|
|
57
|
+
"INSERT OR IGNORE INTO schema_version (version, description) "
|
|
58
|
+
"VALUES (5, 'v3.4.5: access_count_30d + graph edge indexes')"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def apply_migration(conn: sqlite3.Connection) -> dict:
|
|
63
|
+
"""Apply v3.4.5 schema migration. Idempotent.
|
|
64
|
+
|
|
65
|
+
Returns dict with migration status.
|
|
66
|
+
"""
|
|
67
|
+
result: dict[str, list[str]] = {"applied": [], "skipped": [], "errors": []}
|
|
68
|
+
|
|
69
|
+
try:
|
|
70
|
+
conn.execute("PRAGMA foreign_keys = ON")
|
|
71
|
+
conn.execute("PRAGMA busy_timeout = 5000")
|
|
72
|
+
|
|
73
|
+
# access_count_30d column (skip if already exists)
|
|
74
|
+
if conn.execute(_ACCESS_30D_CHECK).fetchone()[0] == 0:
|
|
75
|
+
conn.executescript(_ACCESS_30D_DDL)
|
|
76
|
+
result["applied"].append("access_count_30d")
|
|
77
|
+
else:
|
|
78
|
+
result["skipped"].append("access_count_30d (already present)")
|
|
79
|
+
|
|
80
|
+
# Graph edge indexes
|
|
81
|
+
conn.executescript(_GRAPH_EDGE_INDEX_DDL)
|
|
82
|
+
result["applied"].append("graph_edge_indexes")
|
|
83
|
+
|
|
84
|
+
# Schema version marker
|
|
85
|
+
conn.execute(_SCHEMA_VERSION_INSERT)
|
|
86
|
+
|
|
87
|
+
conn.commit()
|
|
88
|
+
logger.info("Schema v3.4.5 applied: %s", result["applied"])
|
|
89
|
+
|
|
90
|
+
except Exception as exc:
|
|
91
|
+
logger.error("Schema v3.4.5 migration failed: %s", exc)
|
|
92
|
+
result["errors"].append(str(exc))
|
|
93
|
+
try:
|
|
94
|
+
conn.rollback()
|
|
95
|
+
except Exception:
|
|
96
|
+
pass
|
|
97
|
+
|
|
98
|
+
return result
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def schema_version_applied(conn: sqlite3.Connection) -> bool:
|
|
102
|
+
"""Check if v3.4.5 schema has already been applied."""
|
|
103
|
+
try:
|
|
104
|
+
row = conn.execute(
|
|
105
|
+
"SELECT 1 FROM schema_version WHERE version = 5"
|
|
106
|
+
).fetchone()
|
|
107
|
+
return row is not None
|
|
108
|
+
except sqlite3.OperationalError:
|
|
109
|
+
return False
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""Vector backends for SuperLocalMemory v3.4.5."""
|
|
6
|
+
|
|
7
|
+
from superlocalmemory.vector.lancedb_backend import LanceDBVectorBackend
|
|
8
|
+
|
|
9
|
+
__all__ = ["LanceDBVectorBackend"]
|