superlocalmemory 4.0.8 → 4.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +146 -1
- package/README.md +6 -6
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/scripts/ensure-venv.sh +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +3 -3
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +2 -2
- package/pyproject.toml +1 -1
- package/scripts/postinstall.js +4 -0
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/_lazy_init.py +1 -1
- package/src/superlocalmemory/cli/commands.py +119 -9
- package/src/superlocalmemory/cli/db_migrate.py +0 -2
- package/src/superlocalmemory/cli/gdpr_io.py +1 -1
- package/src/superlocalmemory/cli/main.py +5 -5
- package/src/superlocalmemory/cli/service_installer.py +2 -1
- package/src/superlocalmemory/cli/setup_wizard.py +1 -1
- package/src/superlocalmemory/core/config.py +41 -7
- package/src/superlocalmemory/core/context_cache.py +0 -2
- package/src/superlocalmemory/core/engine.py +371 -63
- package/src/superlocalmemory/core/evidence_bundle.py +3 -1
- package/src/superlocalmemory/core/install_detector.py +131 -0
- package/src/superlocalmemory/core/progressive_abstraction.py +1 -1
- package/src/superlocalmemory/core/security_primitives.py +3 -6
- package/src/superlocalmemory/core/store_pipeline.py +94 -26
- package/src/superlocalmemory/core/topic_signature.py +0 -2
- package/src/superlocalmemory/core/transactions/concrete_owners.py +15 -8
- package/src/superlocalmemory/dynamics/eap_scheduler.py +17 -6
- package/src/superlocalmemory/encoding/graph_builder.py +2 -2
- package/src/superlocalmemory/encoding/scene_builder.py +8 -2
- package/src/superlocalmemory/hooks/adapter_base.py +0 -2
- package/src/superlocalmemory/hooks/context_payload.py +0 -2
- package/src/superlocalmemory/hooks/portable_kit.py +8 -8
- package/src/superlocalmemory/hooks/post_tool_async_hook.py +0 -2
- package/src/superlocalmemory/hooks/prewarm_auth.py +0 -2
- package/src/superlocalmemory/hooks/user_prompt_hook.py +0 -2
- package/src/superlocalmemory/infra/backup.py +44 -8
- package/src/superlocalmemory/integrations/bounded_loops_mcp.py +24 -7
- package/src/superlocalmemory/learning/arm_catalog.py +0 -2
- package/src/superlocalmemory/learning/bandit.py +0 -2
- package/src/superlocalmemory/learning/bandit_cache.py +0 -2
- package/src/superlocalmemory/learning/dedup_hnsw.py +11 -11
- package/src/superlocalmemory/learning/ensemble.py +0 -2
- package/src/superlocalmemory/learning/labeler.py +0 -2
- package/src/superlocalmemory/learning/legacy_migration.py +0 -2
- package/src/superlocalmemory/learning/model_cache.py +0 -2
- package/src/superlocalmemory/learning/ranker.py +0 -2
- package/src/superlocalmemory/learning/reward_archive.py +6 -1
- package/src/superlocalmemory/learning/reward_proxy.py +0 -2
- package/src/superlocalmemory/learning/signal_worker.py +0 -2
- package/src/superlocalmemory/math/fisher.py +1 -1
- package/src/superlocalmemory/math/hopfield.py +4 -1
- package/src/superlocalmemory/math/langevin.py +1 -1
- package/src/superlocalmemory/math/sheaf.py +7 -3
- package/src/superlocalmemory/mcp/cli_fallback.py +1 -1
- package/src/superlocalmemory/mcp/profiles.py +1 -1
- package/src/superlocalmemory/mcp/server.py +1 -1
- package/src/superlocalmemory/mcp/tools_active.py +56 -0
- package/src/superlocalmemory/mcp/tools_core.py +1 -1
- package/src/superlocalmemory/optimize/cache/manager.py +2 -2
- package/src/superlocalmemory/optimize/compress/ccr.py +1 -1
- package/src/superlocalmemory/optimize/compress/router.py +1 -1
- package/src/superlocalmemory/optimize/proxy/_helpers.py +2 -2
- package/src/superlocalmemory/optimize/proxy/server.py +1 -1
- package/src/superlocalmemory/optimize/proxy/vertex_surface.py +2 -2
- package/src/superlocalmemory/optimize/storage/db.py +2 -2
- package/src/superlocalmemory/retrieval/agentic.py +1 -1
- package/src/superlocalmemory/retrieval/ann_index.py +9 -2
- package/src/superlocalmemory/retrieval/bm25_channel.py +2 -2
- package/src/superlocalmemory/retrieval/bridge_discovery.py +2 -2
- package/src/superlocalmemory/retrieval/engine.py +272 -43
- package/src/superlocalmemory/retrieval/entity_channel.py +1 -1
- package/src/superlocalmemory/retrieval/hopfield_channel.py +8 -2
- package/src/superlocalmemory/retrieval/profile_channel.py +1 -1
- package/src/superlocalmemory/retrieval/quantization_aware_search.py +1 -1
- package/src/superlocalmemory/retrieval/remote_reranker.py +2 -2
- package/src/superlocalmemory/retrieval/reranker.py +3 -3
- package/src/superlocalmemory/retrieval/semantic_channel.py +3 -3
- package/src/superlocalmemory/retrieval/spreading_activation.py +8 -8
- package/src/superlocalmemory/retrieval/strategy.py +94 -0
- package/src/superlocalmemory/retrieval/temporal_channel.py +167 -10
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +1 -1
- package/src/superlocalmemory/retrieval/vector_store.py +88 -10
- package/src/superlocalmemory/server/recall_serializer.py +10 -0
- package/src/superlocalmemory/server/routes/brain.py +2 -2
- package/src/superlocalmemory/server/routes/prewarm.py +4 -4
- package/src/superlocalmemory/server/unified_daemon.py +459 -7
- package/src/superlocalmemory/storage/_schema_version.py +46 -3
- package/src/superlocalmemory/storage/backup.py +531 -0
- package/src/superlocalmemory/storage/database.py +11 -4
- package/src/superlocalmemory/storage/embedding_codec.py +129 -0
- package/src/superlocalmemory/storage/embedding_migrator.py +5 -3
- package/src/superlocalmemory/storage/migration_runner.py +142 -2
- package/src/superlocalmemory/storage/migrations/__init__.py +1 -1
- package/src/superlocalmemory/storage/migrations.py +15 -1
- package/src/superlocalmemory/storage/models.py +7 -0
- package/src/superlocalmemory/storage/quantized_store.py +4 -2
- package/src/superlocalmemory/summaries/session_summary.py +1 -1
- package/src/superlocalmemory/ui/css/legacy-dashboard.css +1 -1
- package/src/superlocalmemory/ui/css/neural-glass.css +1 -1
- package/src/superlocalmemory/ui/js/core.js +1 -1
- package/src/superlocalmemory/ui/js/od-brain.js +1 -1
|
@@ -234,7 +234,7 @@ class SpreadingActivation:
|
|
|
234
234
|
|
|
235
235
|
# Return top-K sorted by activation
|
|
236
236
|
results = sorted(
|
|
237
|
-
activations.items(), key=lambda x: x[1],
|
|
237
|
+
activations.items(), key=lambda x: (-x[1], x[0]),
|
|
238
238
|
)
|
|
239
239
|
return filter_authorized_results(
|
|
240
240
|
self._db,
|
|
@@ -296,7 +296,7 @@ class SpreadingActivation:
|
|
|
296
296
|
continue
|
|
297
297
|
score = (float(np.dot(q_vec, fact_vec) / denominator) + 1.0) / 2.0
|
|
298
298
|
scored.append((fact.fact_id, score))
|
|
299
|
-
return sorted(scored, key=lambda item: item[1],
|
|
299
|
+
return sorted(scored, key=lambda item: (-item[1], item[0]))[:self._config.top_m]
|
|
300
300
|
|
|
301
301
|
def _propagate(
|
|
302
302
|
self,
|
|
@@ -391,7 +391,7 @@ class SpreadingActivation:
|
|
|
391
391
|
|
|
392
392
|
# Step 3: Lateral inhibition -- keep only top-M
|
|
393
393
|
sorted_nodes = sorted(
|
|
394
|
-
new_activations.items(), key=lambda x: x[1],
|
|
394
|
+
new_activations.items(), key=lambda x: (-x[1], x[0]),
|
|
395
395
|
)
|
|
396
396
|
top_m_nodes = sorted_nodes[: cfg.top_m]
|
|
397
397
|
|
|
@@ -445,30 +445,30 @@ class SpreadingActivation:
|
|
|
445
445
|
SELECT * FROM (
|
|
446
446
|
SELECT target_id AS neighbor_id, weight FROM graph_edges AS ge
|
|
447
447
|
WHERE source_id = ? AND {graph_where}
|
|
448
|
-
ORDER BY weight DESC LIMIT ?
|
|
448
|
+
ORDER BY weight DESC, target_id ASC LIMIT ?
|
|
449
449
|
)
|
|
450
450
|
UNION ALL
|
|
451
451
|
SELECT * FROM (
|
|
452
452
|
SELECT target_fact_id AS neighbor_id, weight
|
|
453
453
|
FROM association_edges AS ae
|
|
454
454
|
WHERE source_fact_id = ? AND {assoc_where}
|
|
455
|
-
ORDER BY weight DESC LIMIT ?
|
|
455
|
+
ORDER BY weight DESC, target_fact_id ASC LIMIT ?
|
|
456
456
|
)
|
|
457
457
|
UNION ALL
|
|
458
458
|
SELECT * FROM (
|
|
459
459
|
SELECT source_id AS neighbor_id, weight FROM graph_edges AS ge
|
|
460
460
|
WHERE target_id = ? AND {graph_where}
|
|
461
|
-
ORDER BY weight DESC LIMIT ?
|
|
461
|
+
ORDER BY weight DESC, source_id ASC LIMIT ?
|
|
462
462
|
)
|
|
463
463
|
UNION ALL
|
|
464
464
|
SELECT * FROM (
|
|
465
465
|
SELECT source_fact_id AS neighbor_id, weight
|
|
466
466
|
FROM association_edges AS ae
|
|
467
467
|
WHERE target_fact_id = ? AND {assoc_where}
|
|
468
|
-
ORDER BY weight DESC LIMIT ?
|
|
468
|
+
ORDER BY weight DESC, source_fact_id ASC LIMIT ?
|
|
469
469
|
)
|
|
470
470
|
)
|
|
471
|
-
ORDER BY weight DESC
|
|
471
|
+
ORDER BY weight DESC, neighbor_id ASC
|
|
472
472
|
LIMIT ?
|
|
473
473
|
""",
|
|
474
474
|
(
|
|
@@ -22,6 +22,17 @@ STRATEGY_PRESETS: dict[str, dict[str, float]] = {
|
|
|
22
22
|
"opinion": {"semantic": 1.8, "bm25": 0.6, "entity_graph": 0.8, "temporal": 0.3, "spreading_activation": 0.5, "hopfield": 0.5},
|
|
23
23
|
"factual": {"semantic": 1.2, "bm25": 1.4, "entity_graph": 1.0, "temporal": 0.6, "spreading_activation": 0.8, "hopfield": 0.8},
|
|
24
24
|
"entity": {"semantic": 1.0, "bm25": 1.2, "entity_graph": 3.0, "temporal": 0.5, "spreading_activation": 1.5, "hopfield": 0.9},
|
|
25
|
+
# A question about the present carries almost no topical information in its
|
|
26
|
+
# words: "what am I working on" matches anything containing "working",
|
|
27
|
+
# including a note from a month ago about a working tree. Word-matching was
|
|
28
|
+
# set to 1.5 here, ABOVE the 1.4 of the preset this one exists to replace,
|
|
29
|
+
# so the new path boosted the exact signal that caused the original failure.
|
|
30
|
+
# It is now the weakest signal, because for this question it is the least
|
|
31
|
+
# informative one; time is what the question is actually about.
|
|
32
|
+
"recency": {
|
|
33
|
+
"temporal": 2.5, "bm25": 0.7, "semantic": 1.2,
|
|
34
|
+
"entity_graph": 1.0, "spreading_activation": 0.8, "hopfield": 0.5,
|
|
35
|
+
},
|
|
25
36
|
"general": {},
|
|
26
37
|
"vague": {"semantic": 0.8, "bm25": 0.5, "entity_graph": 0.6, "temporal": 0.3, "spreading_activation": 1.5, "hopfield": 1.1},
|
|
27
38
|
}
|
|
@@ -32,6 +43,17 @@ _TEMPORAL_WORDS: frozenset[str] = frozenset({
|
|
|
32
43
|
"until", "while", "between", "january", "february", "march",
|
|
33
44
|
"april", "may", "june", "july", "august", "september", "october",
|
|
34
45
|
"november", "december",
|
|
46
|
+
"now", "today", "yesterday",
|
|
47
|
+
"tonight", "tomorrow",
|
|
48
|
+
# "current" was removed earlier because "what is the current database schema"
|
|
49
|
+
# was being answered by newest-fact recency instead of topical search.
|
|
50
|
+
# "latest" and "currently" carry the same over-trigger: in "what is the
|
|
51
|
+
# latest authentication design" or "what is the currently supported format"
|
|
52
|
+
# they mean "most recent version of X", not "at what time did X happen".
|
|
53
|
+
# Routing those queries to temporal causes the recency fallback to dump the
|
|
54
|
+
# 50 newest facts with no topic filter at temporal weight 2.0, burying the
|
|
55
|
+
# subject the user named. Present-activity phrases ("currently working",
|
|
56
|
+
# "currently doing", "currently focused") are covered by _RECENCY_PHRASES.
|
|
35
57
|
})
|
|
36
58
|
|
|
37
59
|
_MULTI_HOP_PHRASES: tuple[str, ...] = (
|
|
@@ -63,6 +85,20 @@ _CAUSAL_TEMPORAL_WORDS: frozenset[str] = frozenset({
|
|
|
63
85
|
"join", "visit", "return",
|
|
64
86
|
})
|
|
65
87
|
|
|
88
|
+
# Phrases that signal a present-activity or ongoing-state question.
|
|
89
|
+
# Checked after multi_hop, entity and temporal word filters so retrospective
|
|
90
|
+
# queries ("what did I work on recently") do not short-circuit here.
|
|
91
|
+
# Order inside the tuple is irrelevant; all are substring-matched against the
|
|
92
|
+
# lowercased query.
|
|
93
|
+
_RECENCY_PHRASES: tuple[str, ...] = (
|
|
94
|
+
"working on",
|
|
95
|
+
"am i working", "are we working", "been working",
|
|
96
|
+
"what's happening", "what is happening",
|
|
97
|
+
"right now", "these days", "at the moment",
|
|
98
|
+
"currently doing", "currently working", "currently focused",
|
|
99
|
+
"what have i been", "what am i doing", "what have we been",
|
|
100
|
+
)
|
|
101
|
+
|
|
66
102
|
_AGGREGATION_WORDS: frozenset[str] = frozenset({
|
|
67
103
|
"all", "list", "every", "everything", "various", "different",
|
|
68
104
|
"many", "several", "multiple", "summarize", "overview",
|
|
@@ -101,9 +137,42 @@ class QueryStrategy:
|
|
|
101
137
|
confidence: float = 0.5
|
|
102
138
|
|
|
103
139
|
|
|
140
|
+
|
|
141
|
+
def _recency_enabled(config: object | None = None) -> bool:
|
|
142
|
+
"""Whether the present-tense path is switched on.
|
|
143
|
+
|
|
144
|
+
The env var SLM_DISABLE_RECENCY_STRATEGY=1 is checked first (incident
|
|
145
|
+
override). If unset, the RetrievalConfig.enable_recency_strategy field is
|
|
146
|
+
consulted when a config object is supplied. Defaults to on when neither
|
|
147
|
+
is present.
|
|
148
|
+
|
|
149
|
+
Kept as a module-level function so the classifier stays usable without an
|
|
150
|
+
engine, which is how the gate and several tests call it.
|
|
151
|
+
"""
|
|
152
|
+
import os
|
|
153
|
+
|
|
154
|
+
if os.environ.get("SLM_DISABLE_RECENCY_STRATEGY", "0") == "1":
|
|
155
|
+
return False
|
|
156
|
+
if config is not None and not getattr(config, "enable_recency_strategy", True):
|
|
157
|
+
return False
|
|
158
|
+
return True
|
|
159
|
+
|
|
160
|
+
|
|
104
161
|
class QueryStrategyClassifier:
|
|
105
162
|
"""Classifies queries and produces adaptive channel weights."""
|
|
106
163
|
|
|
164
|
+
def __init__(self, config: object | None = None) -> None:
|
|
165
|
+
"""Initialise the classifier.
|
|
166
|
+
|
|
167
|
+
Args:
|
|
168
|
+
config: Optional RetrievalConfig-like object. When supplied,
|
|
169
|
+
``enable_recency_strategy=False`` on that object acts as a
|
|
170
|
+
persistent rollback without requiring an env var change.
|
|
171
|
+
The env var SLM_DISABLE_RECENCY_STRATEGY=1 always takes
|
|
172
|
+
precedence.
|
|
173
|
+
"""
|
|
174
|
+
self._config = config
|
|
175
|
+
|
|
107
176
|
def classify(self, query: str, base_weights: dict[str, float]) -> QueryStrategy:
|
|
108
177
|
"""Classify query and return adapted weights."""
|
|
109
178
|
qtype = self._detect_type(query)
|
|
@@ -135,6 +204,18 @@ class QueryStrategyClassifier:
|
|
|
135
204
|
if len(proper_nouns) >= 2 and words & _CAUSAL_TEMPORAL_WORDS:
|
|
136
205
|
return "multi_hop"
|
|
137
206
|
|
|
207
|
+
# Checked BEFORE the single-word test below. "what am I working on right
|
|
208
|
+
# now" contains "now", so the word test claimed it first and routed a
|
|
209
|
+
# question about the present down the retrospective path — which weights
|
|
210
|
+
# word-matching at 1.5 against this path's 0.7, and word-matching on
|
|
211
|
+
# "working" is exactly what surfaced a month-old note about a working
|
|
212
|
+
# tree. A whole phrase states intent; a single word only hints at it.
|
|
213
|
+
#
|
|
214
|
+
# The enable_recency_strategy config field and SLM_DISABLE_RECENCY_STRATEGY
|
|
215
|
+
# env var both feed into _recency_enabled(). When either signals disabled,
|
|
216
|
+
# this block is skipped and the query falls through to subsequent checks.
|
|
217
|
+
if _recency_enabled(self._config) and any(p in q for p in _RECENCY_PHRASES):
|
|
218
|
+
return "recency"
|
|
138
219
|
if words & _TEMPORAL_WORDS:
|
|
139
220
|
return "temporal"
|
|
140
221
|
if words & _AGGREGATION_WORDS:
|
|
@@ -152,3 +233,16 @@ class QueryStrategyClassifier:
|
|
|
152
233
|
if any(p in q for p in _VAGUE_PHRASES):
|
|
153
234
|
return "vague"
|
|
154
235
|
return "general"
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
_DEFAULT_CLASSIFIER = QueryStrategyClassifier()
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def classify_query(query: str, base_weights: dict[str, float] | None = None) -> QueryStrategy:
|
|
242
|
+
"""Classify a query and return adapted channel weights.
|
|
243
|
+
|
|
244
|
+
Convenience function for callers that do not maintain a classifier instance.
|
|
245
|
+
Uses an empty weight dict when base_weights is omitted, which leaves
|
|
246
|
+
absolute weights undefined — useful for type-detection only.
|
|
247
|
+
"""
|
|
248
|
+
return _DEFAULT_CLASSIFIER.classify(query, base_weights or {})
|
|
@@ -81,6 +81,7 @@ class TemporalChannel:
|
|
|
81
81
|
top_k: int = 30,
|
|
82
82
|
include_global: bool | None = None,
|
|
83
83
|
include_shared: bool | None = None,
|
|
84
|
+
query_type: str = "general",
|
|
84
85
|
) -> list[tuple[str, float]]:
|
|
85
86
|
"""Search for temporally relevant facts.
|
|
86
87
|
|
|
@@ -117,11 +118,41 @@ class TemporalChannel:
|
|
|
117
118
|
)
|
|
118
119
|
|
|
119
120
|
# Strategy 2: Date proximity search
|
|
120
|
-
if query_dt is None
|
|
121
|
-
|
|
121
|
+
if query_dt is None:
|
|
122
|
+
recent: list[tuple[str, float]] = []
|
|
123
|
+
# For a question that IS about the present ("what am I working on"),
|
|
124
|
+
# recency is the answer, and it runs regardless of what else matched.
|
|
125
|
+
# It used to sit inside a guard that also required the entity search
|
|
126
|
+
# to be empty — inherited from the case where there is simply nothing
|
|
127
|
+
# to do — so on a real store, where something almost always matches,
|
|
128
|
+
# it effectively never ran.
|
|
129
|
+
#
|
|
130
|
+
# For a merely time-FLAVOURED question ("what is the latest
|
|
131
|
+
# authentication design"), recency is a last resort, not the answer:
|
|
132
|
+
# this channel returns up to 50 newest facts with no regard for topic,
|
|
133
|
+
# and at temporal's weight of 2.0 that buries the very subject the
|
|
134
|
+
# user named. So there it runs only when nothing else matched at all.
|
|
135
|
+
if query_type == "recency" or (
|
|
136
|
+
query_type == "temporal" and not entity_results
|
|
137
|
+
):
|
|
138
|
+
recent = self._recency_fallback(
|
|
139
|
+
profile_id,
|
|
140
|
+
include_global=include_global,
|
|
141
|
+
include_shared=include_shared,
|
|
142
|
+
)
|
|
143
|
+
if not entity_results:
|
|
144
|
+
return recent
|
|
145
|
+
if recent:
|
|
146
|
+
# Both signals are real: an entity the question named, and the
|
|
147
|
+
# fact that the question is about now. Recency leads because
|
|
148
|
+
# that is what this channel was asked about; entity matches
|
|
149
|
+
# follow, and anything already present keeps its better place.
|
|
150
|
+
seen = {fid for fid, _ in recent}
|
|
151
|
+
return recent + [(f, s) for f, s in entity_results if f not in seen]
|
|
122
152
|
|
|
123
153
|
events = self._load_events(
|
|
124
154
|
profile_id, include_global=include_global, include_shared=include_shared,
|
|
155
|
+
near_date=query_dt.date().isoformat() if query_dt is not None else None,
|
|
125
156
|
)
|
|
126
157
|
scored: dict[str, float] = {}
|
|
127
158
|
|
|
@@ -155,7 +186,7 @@ class TemporalChannel:
|
|
|
155
186
|
fid = ev["fact_id"]
|
|
156
187
|
scored[fid] = max(scored.get(fid, 0.0), best)
|
|
157
188
|
|
|
158
|
-
results = sorted(scored.items(), key=lambda x: x[1],
|
|
189
|
+
results = sorted(scored.items(), key=lambda x: (-x[1], x[0]))
|
|
159
190
|
return results[:top_k]
|
|
160
191
|
|
|
161
192
|
def _entity_temporal_search(
|
|
@@ -208,7 +239,13 @@ class TemporalChannel:
|
|
|
208
239
|
"SELECT te.fact_id FROM temporal_events AS te "
|
|
209
240
|
"JOIN canonical_entities AS ce ON ce.entity_id = te.entity_id "
|
|
210
241
|
"JOIN atomic_facts AS af ON af.fact_id = te.fact_id "
|
|
211
|
-
f"WHERE {where} AND LOWER(ce.canonical_name) = LOWER(?)"
|
|
242
|
+
f"WHERE {where} AND LOWER(ce.canonical_name) = LOWER(?) "
|
|
243
|
+
# The score below is derived from each row's POSITION in this
|
|
244
|
+
# result. Position must reflect temporal order so that the comment
|
|
245
|
+
# "first events more likely relevant" holds: oldest fact first,
|
|
246
|
+
# tie-broken by fact_id so two facts created in the same instant
|
|
247
|
+
# produce the same score on two runs.
|
|
248
|
+
"ORDER BY af.created_at ASC, te.fact_id ASC",
|
|
212
249
|
(*params, name),
|
|
213
250
|
)
|
|
214
251
|
for row in rows:
|
|
@@ -227,7 +264,102 @@ class TemporalChannel:
|
|
|
227
264
|
profile_id: str,
|
|
228
265
|
include_global: bool | None = None,
|
|
229
266
|
include_shared: bool | None = None,
|
|
267
|
+
near_date: str | None = None,
|
|
230
268
|
) -> list[dict]:
|
|
269
|
+
"""Load a bounded slice of temporal events.
|
|
270
|
+
|
|
271
|
+
``near_date`` decides WHICH slice. Without it the newest events are
|
|
272
|
+
taken, which suits "what is recent". With it the events closest to that
|
|
273
|
+
date are taken, which is the only slice that can answer a question about
|
|
274
|
+
a particular time — the newest-first bound silently excluded anything
|
|
275
|
+
old, so a question about last year returned nothing rather than slowly.
|
|
276
|
+
"""
|
|
277
|
+
if include_global is None:
|
|
278
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
279
|
+
if include_shared is None:
|
|
280
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
281
|
+
where, params = _scope_where(
|
|
282
|
+
profile_id,
|
|
283
|
+
include_global=include_global,
|
|
284
|
+
include_shared=include_shared,
|
|
285
|
+
prefix="af",
|
|
286
|
+
)
|
|
287
|
+
# The bound has to match what the caller is looking for. Taking the
|
|
288
|
+
# newest 5,000 rows is right when the question is "what is recent", and
|
|
289
|
+
# wrong when it is "what happened in March 2024" — those events carry old
|
|
290
|
+
# rowids and were simply never loaded, so the answer was missing rather
|
|
291
|
+
# than slow. When a target date is known, bound by proximity to THAT date
|
|
292
|
+
# instead; the scan stays bounded either way.
|
|
293
|
+
if near_date is not None:
|
|
294
|
+
# Include events that carry only interval_start/interval_end with no
|
|
295
|
+
# referenced_date or observation_date. The original filter required
|
|
296
|
+
# at least one of the point-date columns to be non-NULL, which
|
|
297
|
+
# excluded duration events ("during March 2024") entirely. The
|
|
298
|
+
# ORDER BY now uses the best available date column so that duration
|
|
299
|
+
# events are ranked by their interval_start when no point date exists.
|
|
300
|
+
rows = self._db.execute(
|
|
301
|
+
"SELECT te.fact_id, te.observation_date, te.referenced_date, "
|
|
302
|
+
"te.interval_start, te.interval_end, af.created_at "
|
|
303
|
+
"FROM temporal_events AS te "
|
|
304
|
+
"JOIN atomic_facts AS af ON af.fact_id = te.fact_id "
|
|
305
|
+
f"WHERE {where} "
|
|
306
|
+
" AND (te.referenced_date IS NOT NULL "
|
|
307
|
+
" OR te.observation_date IS NOT NULL "
|
|
308
|
+
" OR te.interval_start IS NOT NULL) "
|
|
309
|
+
# Thousands of events can tie on the proximity expression when
|
|
310
|
+
# they share a date, and a tie with no secondary key is broken by
|
|
311
|
+
# storage order. That decides which of them survive the LIMIT.
|
|
312
|
+
"ORDER BY ABS(julianday(COALESCE(te.referenced_date, "
|
|
313
|
+
" te.observation_date, "
|
|
314
|
+
" te.interval_start)) "
|
|
315
|
+
" - julianday(?)) ASC, te.fact_id ASC "
|
|
316
|
+
"LIMIT 5000",
|
|
317
|
+
(*params, near_date),
|
|
318
|
+
)
|
|
319
|
+
else:
|
|
320
|
+
rows = self._db.execute(
|
|
321
|
+
"SELECT te.fact_id, te.observation_date, te.referenced_date, "
|
|
322
|
+
"te.interval_start, te.interval_end, af.created_at "
|
|
323
|
+
"FROM temporal_events AS te "
|
|
324
|
+
"JOIN atomic_facts AS af ON af.fact_id = te.fact_id "
|
|
325
|
+
f"WHERE {where} "
|
|
326
|
+
"ORDER BY te.rowid DESC LIMIT 5000",
|
|
327
|
+
(*params,),
|
|
328
|
+
)
|
|
329
|
+
return [dict(r) for r in rows]
|
|
330
|
+
|
|
331
|
+
def _recency_fallback(
|
|
332
|
+
self,
|
|
333
|
+
profile_id: str,
|
|
334
|
+
include_global: bool | None,
|
|
335
|
+
include_shared: bool | None,
|
|
336
|
+
) -> list[tuple[str, float]]:
|
|
337
|
+
"""Return recently created facts with Gaussian age-decay scoring.
|
|
338
|
+
|
|
339
|
+
Called when the query carries no date and the caller has said the
|
|
340
|
+
question is about the present. It deliberately does not depend on the
|
|
341
|
+
entity search being empty — requiring that made this unreachable on any
|
|
342
|
+
store where something matches, which is most of them.
|
|
343
|
+
|
|
344
|
+
One entry per fact. Facts scored here compete in fusion against semantic
|
|
345
|
+
and BM25 results, and fusion ranks facts, so repeating a fact spends
|
|
346
|
+
ranks without adding candidates.
|
|
347
|
+
|
|
348
|
+
Scoring: Gaussian with sigma=7 days. Facts older than 90 days score
|
|
349
|
+
below 0.01 and are excluded. Returns at most 50 (fact_id, score) pairs,
|
|
350
|
+
ordered highest-score first.
|
|
351
|
+
|
|
352
|
+
Source table: atomic_facts, not temporal_events. The materializer
|
|
353
|
+
populates temporal_events asynchronously and only for facts with both
|
|
354
|
+
canonical entities and resolved dates. A plain note written moments ago
|
|
355
|
+
never receives a temporal_events row until that background pass runs, so
|
|
356
|
+
a join against temporal_events makes newly written facts structurally
|
|
357
|
+
invisible here — exactly when the caller needs them most.
|
|
358
|
+
"""
|
|
359
|
+
_SIGMA = 7.0 # days — tighter than _proximity_score's 30d
|
|
360
|
+
_MAX_AGE_DAYS = 90.0 # cut-off: exp(-(90^2)/(2*7^2)) ≈ 0.0
|
|
361
|
+
now_dt = datetime.now(tz=timezone.utc)
|
|
362
|
+
|
|
231
363
|
if include_global is None:
|
|
232
364
|
include_global = bool(getattr(self, "include_global", False))
|
|
233
365
|
if include_shared is None:
|
|
@@ -239,14 +371,39 @@ class TemporalChannel:
|
|
|
239
371
|
prefix="af",
|
|
240
372
|
)
|
|
241
373
|
rows = self._db.execute(
|
|
242
|
-
"SELECT
|
|
243
|
-
"
|
|
244
|
-
"
|
|
245
|
-
"
|
|
246
|
-
|
|
374
|
+
"SELECT af.fact_id, af.created_at "
|
|
375
|
+
"FROM atomic_facts AS af "
|
|
376
|
+
f"WHERE {where} "
|
|
377
|
+
" AND af.created_at >= datetime('now', '-90 days') "
|
|
378
|
+
"ORDER BY af.created_at DESC, af.fact_id ASC "
|
|
379
|
+
"LIMIT 50",
|
|
247
380
|
(*params,),
|
|
248
381
|
)
|
|
249
|
-
|
|
382
|
+
|
|
383
|
+
best: dict[str, float] = {}
|
|
384
|
+
for row in rows:
|
|
385
|
+
d = dict(row)
|
|
386
|
+
fid = d.get("fact_id")
|
|
387
|
+
if not fid:
|
|
388
|
+
continue
|
|
389
|
+
created = _parse_iso(d.get("created_at"))
|
|
390
|
+
if created is None:
|
|
391
|
+
continue
|
|
392
|
+
utc_created = _as_utc(created)
|
|
393
|
+
if utc_created is None:
|
|
394
|
+
continue
|
|
395
|
+
age_days = max(
|
|
396
|
+
0.0,
|
|
397
|
+
(now_dt - utc_created).total_seconds() / 86400.0,
|
|
398
|
+
)
|
|
399
|
+
if age_days > _MAX_AGE_DAYS:
|
|
400
|
+
continue
|
|
401
|
+
score = math.exp(-(age_days ** 2) / (2.0 * _SIGMA * _SIGMA))
|
|
402
|
+
if score > 0.01 and score > best.get(fid, 0.0):
|
|
403
|
+
best[fid] = score
|
|
404
|
+
|
|
405
|
+
out = sorted(best.items(), key=lambda kv: (-kv[1], kv[0]))
|
|
406
|
+
return out[:50]
|
|
250
407
|
|
|
251
408
|
@staticmethod
|
|
252
409
|
def _try_parse(text: str) -> datetime | None:
|
|
@@ -460,7 +460,7 @@ class TemporalValidityFilter:
|
|
|
460
460
|
]
|
|
461
461
|
# Re-sort descending so demoted facts fall below currently-valid
|
|
462
462
|
# facts in this channel's rank order.
|
|
463
|
-
new_list.sort(key=lambda pair: pair[1],
|
|
463
|
+
new_list.sort(key=lambda pair: (-pair[1], pair[0]))
|
|
464
464
|
demoted[channel_name] = new_list
|
|
465
465
|
return demoted
|
|
466
466
|
|
|
@@ -25,6 +25,7 @@ from typing import Generator
|
|
|
25
25
|
|
|
26
26
|
import numpy as np
|
|
27
27
|
|
|
28
|
+
from superlocalmemory.storage.embedding_codec import encode_embedding
|
|
28
29
|
from superlocalmemory.storage.write_lock import get_write_lock
|
|
29
30
|
|
|
30
31
|
logger = logging.getLogger(__name__)
|
|
@@ -249,8 +250,19 @@ class VectorStore:
|
|
|
249
250
|
|
|
250
251
|
@staticmethod
|
|
251
252
|
def _serialize_f32(vector: list[float]) -> bytes:
|
|
252
|
-
"""Serialize float list to raw bytes for sqlite-vec.
|
|
253
|
-
|
|
253
|
+
"""Serialize float list to raw bytes for sqlite-vec.
|
|
254
|
+
|
|
255
|
+
Delegates to the shared embedding codec so that a format change in
|
|
256
|
+
that codec propagates here automatically.
|
|
257
|
+
"""
|
|
258
|
+
result = encode_embedding(vector)
|
|
259
|
+
if result is None:
|
|
260
|
+
# encode_embedding returns None only for None input; _serialize_f32
|
|
261
|
+
# is never called with None, so this branch is unreachable in
|
|
262
|
+
# production. Return empty bytes rather than raising so the caller
|
|
263
|
+
# sees an empty query result rather than an unhandled exception.
|
|
264
|
+
return b""
|
|
265
|
+
return result
|
|
254
266
|
|
|
255
267
|
# -- CRUD Operations ----------------------------------------------------
|
|
256
268
|
|
|
@@ -478,14 +490,28 @@ class VectorStore:
|
|
|
478
490
|
(*base_params, search_k),
|
|
479
491
|
).fetchall()
|
|
480
492
|
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
493
|
+
# Rank the whole candidate list, THEN cut it. The expansion above
|
|
494
|
+
# can leave `rows` longer than top_k, and the SQL has no ORDER BY —
|
|
495
|
+
# vec0 applies k, then a relational join emits the survivors in an
|
|
496
|
+
# order SQLite is free to choose. Cutting first therefore cut by
|
|
497
|
+
# position and only ranked what survived, which can drop a nearer
|
|
498
|
+
# fact in favour of a farther one.
|
|
499
|
+
#
|
|
500
|
+
# Measured on the 0.95 GB archive: 29 of 60 queries do take the
|
|
501
|
+
# expansion path (orphan vectors are common), and the produced set
|
|
502
|
+
# matched the true nearest-k in all 29 — the join does emit in
|
|
503
|
+
# distance order in practice. So this was latent, not active. It is
|
|
504
|
+
# fixed because "the planner happens to" is not a guarantee: add an
|
|
505
|
+
# index, change SQLite, or change the join and the answer moves.
|
|
506
|
+
#
|
|
507
|
+
# The tie-break on fact_id makes the cut total, so two facts at
|
|
508
|
+
# equal distance cannot swap across the top_k boundary between runs.
|
|
509
|
+
results: list[tuple[str, float]] = [
|
|
510
|
+
(str(row["fact_id"]), max(0.0, 1.0 - row["distance"]))
|
|
511
|
+
for row in rows
|
|
512
|
+
]
|
|
513
|
+
results.sort(key=lambda x: (-x[1], x[0]))
|
|
514
|
+
return results[:top_k]
|
|
489
515
|
|
|
490
516
|
except Exception as exc:
|
|
491
517
|
logger.debug("search failed: %s", exc)
|
|
@@ -596,6 +622,58 @@ class VectorStore:
|
|
|
596
622
|
except Exception:
|
|
597
623
|
return False
|
|
598
624
|
|
|
625
|
+
def is_searchable_by_meaning(
|
|
626
|
+
self,
|
|
627
|
+
fact_id: str,
|
|
628
|
+
profile_id: str | None = None,
|
|
629
|
+
) -> bool:
|
|
630
|
+
"""Return True if search() would be able to return this fact.
|
|
631
|
+
|
|
632
|
+
The check mirrors search()'s own join:
|
|
633
|
+
|
|
634
|
+
FROM fact_embeddings AS fe
|
|
635
|
+
JOIN embedding_metadata AS em
|
|
636
|
+
ON em.vec_rowid = fe.rowid
|
|
637
|
+
AND em.profile_id = fe.profile_id
|
|
638
|
+
|
|
639
|
+
A fact that has a vector in fact_embeddings but no row in
|
|
640
|
+
embedding_metadata will NOT be returned by search(), so this method
|
|
641
|
+
returns False for it — even though raw_vector_present() would return
|
|
642
|
+
True. Callers that need to decide whether a fact requires re-embedding
|
|
643
|
+
must use this method, not raw_vector_present().
|
|
644
|
+
|
|
645
|
+
Returns False on any error or when the store is unavailable (fail-
|
|
646
|
+
closed: never optimistic).
|
|
647
|
+
"""
|
|
648
|
+
if not self._available:
|
|
649
|
+
return False
|
|
650
|
+
try:
|
|
651
|
+
with self._managed_connection() as conn:
|
|
652
|
+
if profile_id is not None:
|
|
653
|
+
sql = (
|
|
654
|
+
"SELECT 1 FROM fact_embeddings AS fe "
|
|
655
|
+
"JOIN embedding_metadata AS em "
|
|
656
|
+
"ON em.vec_rowid = fe.rowid "
|
|
657
|
+
"AND em.profile_id = fe.profile_id "
|
|
658
|
+
"WHERE em.fact_id = ? "
|
|
659
|
+
"AND fe.profile_id = ? "
|
|
660
|
+
"LIMIT 1"
|
|
661
|
+
)
|
|
662
|
+
row = conn.execute(sql, (fact_id, profile_id)).fetchone()
|
|
663
|
+
else:
|
|
664
|
+
sql = (
|
|
665
|
+
"SELECT 1 FROM fact_embeddings AS fe "
|
|
666
|
+
"JOIN embedding_metadata AS em "
|
|
667
|
+
"ON em.vec_rowid = fe.rowid "
|
|
668
|
+
"AND em.profile_id = fe.profile_id "
|
|
669
|
+
"WHERE em.fact_id = ? "
|
|
670
|
+
"LIMIT 1"
|
|
671
|
+
)
|
|
672
|
+
row = conn.execute(sql, (fact_id,)).fetchone()
|
|
673
|
+
return row is not None
|
|
674
|
+
except Exception:
|
|
675
|
+
return False
|
|
676
|
+
|
|
599
677
|
def count(self, profile_id: str | None = None) -> int:
|
|
600
678
|
"""Count complete metadata/vector pairs in the store.
|
|
601
679
|
|
|
@@ -298,4 +298,14 @@ def recall_response_metadata(response: Any) -> dict:
|
|
|
298
298
|
# Q2b: thematic community summary (pure pass-through; computed upstream
|
|
299
299
|
# in the engine where DB access is available). None on most recalls.
|
|
300
300
|
"thematic_context": getattr(response, "community_context", None),
|
|
301
|
+
# Channels abandoned at the hang guard, so their candidates are absent
|
|
302
|
+
# from this answer. Empty on a healthy recall, which is the normal case.
|
|
303
|
+
# Non-empty is the one situation in which asking the same question twice
|
|
304
|
+
# may legitimately give different answers, so it has to travel with the
|
|
305
|
+
# response rather than living only in a server log — otherwise a caller
|
|
306
|
+
# comparing two runs has no way to tell an incomplete answer from a
|
|
307
|
+
# changed one. A list, because JSON has no tuple.
|
|
308
|
+
"incomplete_channels": list(
|
|
309
|
+
getattr(response, "incomplete_channels", ()) or ()
|
|
310
|
+
),
|
|
301
311
|
}
|
|
@@ -806,7 +806,7 @@ def _compute_cross_platform() -> dict:
|
|
|
806
806
|
``memory.db`` (LLD-07 M004). On any adapter error, that adapter
|
|
807
807
|
reports ``active: false`` with ``reason: error:<ExcName>`` rather
|
|
808
808
|
than crashing the whole Brain endpoint (LLD-04 §2 — "honest, never
|
|
809
|
-
fake"). An unimportable adapter means the install is missing
|
|
809
|
+
fake"). An unimportable adapter means the install is missing the
|
|
810
810
|
components, which is legitimate for an older 3.4.20 → 3.4.22 upgrade
|
|
811
811
|
mid-migration.
|
|
812
812
|
"""
|
|
@@ -953,7 +953,7 @@ def _compute_active_clients(profile_id: str) -> dict:
|
|
|
953
953
|
try:
|
|
954
954
|
from superlocalmemory.hooks.session_registry import active_client_summary
|
|
955
955
|
clients = active_client_summary(profile_id, within_seconds=300)
|
|
956
|
-
except Exception as exc: # distinguish failure from emptiness (
|
|
956
|
+
except Exception as exc: # distinguish failure from emptiness (important: these are different states)
|
|
957
957
|
registry_ok = False
|
|
958
958
|
reg_status = "error"
|
|
959
959
|
logger.debug("active_clients: registry error: %s", exc)
|
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
|
|
5
5
|
"""POST /internal/prewarm — populates the context cache for a session.
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
no FastAPI route mounted it. The hot-path ``post_tool_async_hook``
|
|
10
|
-
to ``/internal/prewarm`` after every tool call to refresh the
|
|
7
|
+
Bug fix: ``hooks/prewarm_auth.authorize`` (which gates loopback →
|
|
8
|
+
origin → install-token → body-size) was implemented and unit-tested,
|
|
9
|
+
but no FastAPI route mounted it. The hot-path ``post_tool_async_hook``
|
|
10
|
+
POSTs to ``/internal/prewarm`` after every tool call to refresh the
|
|
11
11
|
``active_brain_cache`` row for the current session/topic. Without a
|
|
12
12
|
route registered here, those POSTs 404'd silently, the cache never
|
|
13
13
|
populated, and every ``UserPromptSubmit`` ended up a structural miss.
|