superlocalmemory 3.8.0 → 3.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +112 -0
- package/README.md +32 -120
- package/package.json +9 -2
- package/plugin/.claude-plugin/plugin.json +1 -2
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +2 -2
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +3 -5
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +2 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +3 -5
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +2 -1
- package/scripts/postinstall.js +7 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +494 -9
- package/src/superlocalmemory/cli/daemon.py +7 -0
- package/src/superlocalmemory/cli/loop_cmd.py +2 -7
- package/src/superlocalmemory/cli/main.py +72 -7
- package/src/superlocalmemory/cli/setup_wizard.py +142 -16
- package/src/superlocalmemory/cli/version_banner.py +17 -3
- package/src/superlocalmemory/core/backend_orchestrator.py +18 -16
- package/src/superlocalmemory/core/component_healer.py +144 -0
- package/src/superlocalmemory/core/component_registry.py +487 -0
- package/src/superlocalmemory/core/config.py +21 -0
- package/src/superlocalmemory/core/embedding_worker.py +4 -5
- package/src/superlocalmemory/core/embeddings.py +132 -45
- package/src/superlocalmemory/core/engine.py +29 -22
- package/src/superlocalmemory/core/engine_ingestion.py +332 -45
- package/src/superlocalmemory/core/ingestion_command.py +154 -25
- package/src/superlocalmemory/core/injection.py +12 -7
- package/src/superlocalmemory/core/maintenance.py +43 -0
- package/src/superlocalmemory/core/maintenance_scheduler.py +44 -6
- package/src/superlocalmemory/core/recall_pipeline.py +42 -4
- package/src/superlocalmemory/core/store_pipeline.py +195 -20
- package/src/superlocalmemory/hooks/hook_handlers.py +6 -1
- package/src/superlocalmemory/hooks/portable_kit.py +34 -2
- package/src/superlocalmemory/learning/model_rollback.py +3 -0
- package/src/superlocalmemory/learning/ranker_retrain_online.py +2 -0
- package/src/superlocalmemory/learning/reward.py +50 -0
- package/src/superlocalmemory/learning/source_quality.py +523 -1
- package/src/superlocalmemory/loops/ledger.py +25 -5
- package/src/superlocalmemory/mcp/_daemon_proxy.py +6 -2
- package/src/superlocalmemory/mcp/_pool_adapter.py +4 -1
- package/src/superlocalmemory/mcp/server.py +11 -30
- package/src/superlocalmemory/mcp/tools_active.py +1 -1
- package/src/superlocalmemory/mcp/tools_core.py +21 -5
- package/src/superlocalmemory/mcp/tools_learning.py +2 -2
- package/src/superlocalmemory/retrieval/bridge_discovery.py +14 -0
- package/src/superlocalmemory/retrieval/engine.py +53 -21
- package/src/superlocalmemory/retrieval/reranker.py +3 -4
- package/src/superlocalmemory/retrieval/spreading_activation.py +68 -38
- package/src/superlocalmemory/server/config_file.py +90 -0
- package/src/superlocalmemory/server/origin.py +50 -0
- package/src/superlocalmemory/server/routes/backup.py +293 -70
- package/src/superlocalmemory/server/routes/behavioral.py +342 -61
- package/src/superlocalmemory/server/routes/brain.py +57 -16
- package/src/superlocalmemory/server/routes/config_api.py +84 -82
- package/src/superlocalmemory/server/routes/entity.py +100 -23
- package/src/superlocalmemory/server/routes/evolution.py +103 -100
- package/src/superlocalmemory/server/routes/learning.py +286 -105
- package/src/superlocalmemory/server/routes/learning_telemetry.py +153 -0
- package/src/superlocalmemory/server/routes/memories.py +8 -3
- package/src/superlocalmemory/server/routes/mesh.py +121 -32
- package/src/superlocalmemory/server/routes/ratelimit.py +33 -25
- package/src/superlocalmemory/server/routes/stats.py +93 -155
- package/src/superlocalmemory/server/routes/token.py +3 -13
- package/src/superlocalmemory/server/routes/v3_api.py +184 -20
- package/src/superlocalmemory/server/unified_daemon.py +732 -41
- package/src/superlocalmemory/storage/embedding_migrator.py +235 -0
- package/src/superlocalmemory/storage/migration_runner.py +79 -1
- package/src/superlocalmemory/storage/migrations/M010_evolution_config.py +5 -0
- package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +270 -0
- package/src/superlocalmemory/storage/migrations/M029_behavioral_history_indexes.py +137 -0
- package/src/superlocalmemory/storage/migrations/M030_entity_explorer_indexes.py +93 -0
- package/src/superlocalmemory/storage/migrations/__init__.py +4 -0
- package/src/superlocalmemory/storage/schema.py +49 -1
- package/src/superlocalmemory/storage/schema_v32.py +2 -0
- package/src/superlocalmemory/storage/schema_v347.py +4 -0
- package/src/superlocalmemory/ui/index.html +6 -8
- package/src/superlocalmemory/ui/js/core.js +52 -9
- package/src/superlocalmemory/ui/js/dashboard.js +169 -82
- package/src/superlocalmemory/ui/js/od-backup.js +156 -65
- package/src/superlocalmemory/ui/js/od-brain.js +88 -51
- package/src/superlocalmemory/ui/js/od-components.js +147 -0
- package/src/superlocalmemory/ui/js/od-entities.js +65 -22
- package/src/superlocalmemory/ui/js/od-graph.js +46 -4
- package/src/superlocalmemory/ui/js/od-health.js +18 -0
- package/src/superlocalmemory/ui/js/od-memories.js +84 -5
- package/src/superlocalmemory/ui/js/od-mesh.js +23 -9
- package/src/superlocalmemory/ui/js/od-operations.js +36 -0
- package/src/superlocalmemory/ui/js/od-settings.js +186 -63
- package/src/superlocalmemory/ui/js/od-shell.js +249 -33
- package/src/superlocalmemory/ui/js/od-skills.js +44 -17
- package/src/superlocalmemory/ui/js/settings.js +15 -1
- package/plugin-src/.mcp.json +0 -12
- package/plugin-src/agents/slm-governance-advisor.md +0 -80
- package/plugin-src/agents/slm-loop-runner.md +0 -71
- package/plugin-src/agents/slm-memory-advisor.md +0 -49
- package/plugin-src/agents/slm-optimize-advisor.md +0 -44
- package/plugin-src/commands/slm-loop.md +0 -31
- package/plugin-src/hooks/.gitkeep +0 -0
- package/plugin-src/hooks/hooks.json +0 -102
- package/plugin-src/manifest.json +0 -30
- package/plugin-src/requirements.txt +0 -1
- package/plugin-src/rules/CLAUDE.md.fragment +0 -44
- package/plugin-src/scripts/ensure-venv.bat +0 -122
- package/plugin-src/scripts/ensure-venv.sh +0 -105
- package/plugin-src/scripts/slm-launch +0 -62
- package/plugin-src/scripts/slm-launch.bat +0 -23
- package/plugin-src/settings.json +0 -25
- package/plugin-src/skills/slm-governance/SKILL.md +0 -248
- package/plugin-src/skills/slm-loop/SKILL.md +0 -99
- package/plugin-src/skills/slm-mesh/SKILL.md +0 -282
- package/plugin-src/skills/slm-profile/SKILL.md +0 -148
- package/plugin-src/skills/slm-scope/SKILL.md +0 -176
|
@@ -203,7 +203,7 @@ class _FilteredServer:
|
|
|
203
203
|
"""
|
|
204
204
|
__slots__ = ("_server", "_allowed")
|
|
205
205
|
|
|
206
|
-
def __init__(self, real_server:
|
|
206
|
+
def __init__(self, real_server: SLMFastMCP, allowed: frozenset[str]) -> None:
|
|
207
207
|
self._server = real_server
|
|
208
208
|
self._allowed = allowed
|
|
209
209
|
|
|
@@ -265,29 +265,20 @@ from superlocalmemory.mcp.tools_loops import register_loop_tools
|
|
|
265
265
|
register_loop_tools(_target, get_engine) # v3.8.0: bounded-loop tools (CLI+command+MCP)
|
|
266
266
|
|
|
267
267
|
|
|
268
|
-
#
|
|
269
|
-
#
|
|
270
|
-
#
|
|
271
|
-
#
|
|
272
|
-
# the first tool call arrives (1-2s later), the engine is already warm.
|
|
273
|
-
# This applies to ALL IDEs: Claude Code, Cursor, Antigravity, Gemini CLI, etc.
|
|
268
|
+
# Keep stdio MCP processes thin until a tool truly needs a local LIGHT engine.
|
|
269
|
+
# Every open IDE/task owns a stdio process; eagerly opening memory.db in all of
|
|
270
|
+
# them multiplied RAM and SQLite writers on machines with many long-lived
|
|
271
|
+
# sessions. The shared daemon owns model warmup and common remember/recall work.
|
|
274
272
|
def _eager_warmup() -> None:
|
|
275
|
-
"""
|
|
273
|
+
"""Ensure the shared daemon is running without opening a per-client engine.
|
|
276
274
|
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
275
|
+
Mesh registration is intentionally lazy: a local stdio session is not a
|
|
276
|
+
remote peer, and heartbeat writes should begin only after a mesh tool is
|
|
277
|
+
actually used.
|
|
280
278
|
"""
|
|
281
|
-
import logging
|
|
282
279
|
_logger = logging.getLogger(__name__)
|
|
283
|
-
try:
|
|
284
|
-
get_engine()
|
|
285
|
-
_logger.info("MCP engine pre-warmed successfully")
|
|
286
|
-
except Exception as exc:
|
|
287
|
-
_logger.warning("MCP engine pre-warmup failed: %s", exc)
|
|
288
280
|
|
|
289
|
-
# Measurement / test harnesses set this to skip daemon-start
|
|
290
|
-
# mesh-register. The LIGHT engine init above still runs.
|
|
281
|
+
# Measurement / test harnesses set this to skip daemon-start.
|
|
291
282
|
if _os.environ.get("SLM_DISABLE_WARMUP_SIDE_EFFECTS") == "1":
|
|
292
283
|
return
|
|
293
284
|
|
|
@@ -300,22 +291,12 @@ def _eager_warmup() -> None:
|
|
|
300
291
|
except Exception as exc:
|
|
301
292
|
_logger.warning("Daemon auto-start failed: %s", exc)
|
|
302
293
|
|
|
303
|
-
# V3.4.6: Auto-register this MCP session as a mesh peer immediately.
|
|
304
|
-
# Previously, registration was lazy (only on first mesh tool call).
|
|
305
|
-
# Now every Claude session appears on the mesh from startup.
|
|
306
|
-
try:
|
|
307
|
-
from superlocalmemory.mcp.tools_mesh import auto_register_mesh
|
|
308
|
-
auto_register_mesh()
|
|
309
|
-
_logger.info("Mesh peer auto-registered at startup")
|
|
310
|
-
except Exception as exc:
|
|
311
|
-
_logger.warning("Mesh auto-register failed: %s", exc)
|
|
312
|
-
|
|
313
294
|
import threading
|
|
314
295
|
|
|
315
296
|
# v3.6.7: Suppress standalone-process behaviours when the MCP server is
|
|
316
297
|
# imported inside the daemon (SLM_MCP_EMBEDDED=1). Three threads are safe
|
|
317
298
|
# to run in a dedicated `slm mcp` subprocess but harmful inside the daemon:
|
|
318
|
-
# mcp-warmup —
|
|
299
|
+
# mcp-warmup — ensures the shared daemon only; never creates an engine.
|
|
319
300
|
# parent-watchdog — calls os._exit(0) if its parent IDE quits, which would
|
|
320
301
|
# kill the daemon along with it.
|
|
321
302
|
# stdin-eof-monitor — monitors stdin pipe; meaningless inside the daemon.
|
|
@@ -227,7 +227,7 @@ def register_active_tools(server, get_engine: Callable) -> None:
|
|
|
227
227
|
# thread so the async MCP event loop is not stalled — same
|
|
228
228
|
# fix class as #34 mesh tools deadlock.
|
|
229
229
|
response = await asyncio.to_thread(
|
|
230
|
-
pool_recall, search_query, limit=max_results, fast=
|
|
230
|
+
pool_recall, search_query, limit=max_results, fast=None,
|
|
231
231
|
)
|
|
232
232
|
except (PoolError, Exception) as exc:
|
|
233
233
|
logger.warning(
|
|
@@ -84,8 +84,9 @@ def _record_recall_hits(
|
|
|
84
84
|
enqueue_shown_flip,
|
|
85
85
|
)
|
|
86
86
|
|
|
87
|
-
|
|
88
|
-
|
|
87
|
+
pid = profile_id
|
|
88
|
+
if not pid:
|
|
89
|
+
pid = get_engine().profile_id
|
|
89
90
|
slm_dir = canonical_data_root()
|
|
90
91
|
|
|
91
92
|
shown_ids = [r.get("fact_id", "") for r in results[:10]
|
|
@@ -121,7 +122,7 @@ def _record_recall_hits(
|
|
|
121
122
|
def register_core_tools(server, get_engine: Callable) -> None:
|
|
122
123
|
"""Register the 13 core MCP tools on *server*."""
|
|
123
124
|
|
|
124
|
-
@server.tool(
|
|
125
|
+
@server.tool()
|
|
125
126
|
async def remember(
|
|
126
127
|
content: str, tags: str = "", project: str = "",
|
|
127
128
|
importance: int = 5, session_id: str = "",
|
|
@@ -281,13 +282,28 @@ def register_core_tools(server, get_engine: Callable) -> None:
|
|
|
281
282
|
@server.tool(annotations=ToolAnnotations(readOnlyHint=True))
|
|
282
283
|
async def recall(
|
|
283
284
|
query: str, limit: int = CANONICAL_RECALL_LIMIT, agent_id: str = "mcp_client",
|
|
284
|
-
session_id: str = "", fast: bool =
|
|
285
|
+
session_id: str = "", fast: bool | None = None,
|
|
285
286
|
include_global: bool | None = None,
|
|
286
287
|
include_shared: bool | None = None,
|
|
287
288
|
window: str = "",
|
|
288
289
|
) -> dict:
|
|
289
290
|
"""Search memories through hybrid retrieval, RRF fusion, and reranking.
|
|
290
291
|
|
|
292
|
+
Fast local retrieval (six channels + reranker) returns in ~1-2s. This
|
|
293
|
+
tool does NOT run an internal LLM reformulation round — YOU (the calling
|
|
294
|
+
model) are the reasoner. Drive refinement using the confidence signals
|
|
295
|
+
in the response:
|
|
296
|
+
• ``no_confident_match: true`` → nothing cleared the evidence floor.
|
|
297
|
+
Do NOT invent a memory. Rewrite the query into 1-3 more specific
|
|
298
|
+
sub-queries (split multi-hop questions; try entity names, synonyms,
|
|
299
|
+
or a broader phrasing) and call ``recall`` again before concluding
|
|
300
|
+
the information is unknown.
|
|
301
|
+
• ``answer_confidence`` low / ``abstained: true`` → the top hit is
|
|
302
|
+
weak. Re-query with a sharper phrasing, or widen with
|
|
303
|
+
``include_shared=true`` / ``include_global=true`` if appropriate.
|
|
304
|
+
• Confident match → use it directly; no second call needed.
|
|
305
|
+
One extra targeted recall is cheap and beats a wrong "not found".
|
|
306
|
+
|
|
291
307
|
Optional ``session_id`` threads through to the
|
|
292
308
|
engine's outcome-queue so PostToolUse / Stop hooks can attach
|
|
293
309
|
engagement signals to this recall. Claude Code should pass its
|
|
@@ -356,7 +372,7 @@ def register_core_tools(server, get_engine: Callable) -> None:
|
|
|
356
372
|
# block behind a single threading.Lock. See worker_pool.py.
|
|
357
373
|
result = await asyncio.to_thread(
|
|
358
374
|
pool.recall, query, limit=limit, session_id=effective_sid,
|
|
359
|
-
fast=
|
|
375
|
+
fast=fast,
|
|
360
376
|
include_global=include_global,
|
|
361
377
|
include_shared=include_shared,
|
|
362
378
|
window=window or None,
|
|
@@ -143,7 +143,7 @@ def register_learning_tools(server, get_engine: Callable) -> None:
|
|
|
143
143
|
logger.debug("get_assertions failed: %s", exc)
|
|
144
144
|
return {"assertions": [], "count": 0, "error": str(exc)}
|
|
145
145
|
|
|
146
|
-
@server.tool(
|
|
146
|
+
@server.tool()
|
|
147
147
|
async def reinforce_assertion(assertion_id: str) -> dict:
|
|
148
148
|
"""Reinforce a behavioral assertion (increase confidence).
|
|
149
149
|
|
|
@@ -171,7 +171,7 @@ def register_learning_tools(server, get_engine: Callable) -> None:
|
|
|
171
171
|
except Exception as exc:
|
|
172
172
|
return {"success": False, "error": str(exc)}
|
|
173
173
|
|
|
174
|
-
@server.tool(
|
|
174
|
+
@server.tool()
|
|
175
175
|
async def contradict_assertion(assertion_id: str) -> dict:
|
|
176
176
|
"""Contradict a behavioral assertion (decrease confidence).
|
|
177
177
|
|
|
@@ -27,6 +27,7 @@ License: AGPL-3.0-or-later
|
|
|
27
27
|
from __future__ import annotations
|
|
28
28
|
|
|
29
29
|
import logging
|
|
30
|
+
import time
|
|
30
31
|
from typing import TYPE_CHECKING
|
|
31
32
|
|
|
32
33
|
from superlocalmemory.retrieval.scope_policy import (
|
|
@@ -71,6 +72,7 @@ class BridgeDiscovery:
|
|
|
71
72
|
*,
|
|
72
73
|
include_global: bool = False,
|
|
73
74
|
include_shared: bool = False,
|
|
75
|
+
time_budget_s: float = 0.4,
|
|
74
76
|
) -> list[tuple[str, float]]:
|
|
75
77
|
"""Find bridge facts connecting seed results.
|
|
76
78
|
|
|
@@ -110,9 +112,19 @@ class BridgeDiscovery:
|
|
|
110
112
|
|
|
111
113
|
bridges: list[tuple[str, float]] = []
|
|
112
114
|
seen = set(seed_ids)
|
|
115
|
+
# v3.8.2: bound the per-entity get_facts_by_entity fan-out. On a dense
|
|
116
|
+
# entity graph (M5: 3.3k entities / 208k edges) a single recall could
|
|
117
|
+
# issue 200+ DB round-trips here — the primary 3.8 full-mode latency
|
|
118
|
+
# spike (observed 7.6s). Bridges are a SUPPLEMENTARY post-fusion boost
|
|
119
|
+
# (score x0.8, only added if not already found), so truncating them
|
|
120
|
+
# under a wall-clock budget is quality-safe: the ranked channels have
|
|
121
|
+
# already returned the core results.
|
|
122
|
+
deadline = time.monotonic() + time_budget_s
|
|
113
123
|
|
|
114
124
|
# Check consecutive pairs for entity overlap
|
|
115
125
|
for i in range(len(seed_ids) - 1):
|
|
126
|
+
if time.monotonic() > deadline:
|
|
127
|
+
break
|
|
116
128
|
fact_a = seed_facts.get(seed_ids[i])
|
|
117
129
|
fact_b = seed_facts.get(seed_ids[i + 1])
|
|
118
130
|
if not fact_a or not fact_b:
|
|
@@ -128,6 +140,8 @@ class BridgeDiscovery:
|
|
|
128
140
|
# Strategy 1: Entity bridge (union minus intersection)
|
|
129
141
|
bridge_entities = (entities_a | entities_b) - (entities_a & entities_b)
|
|
130
142
|
for eid in bridge_entities:
|
|
143
|
+
if time.monotonic() > deadline:
|
|
144
|
+
break
|
|
131
145
|
entity_facts = self._db.get_facts_by_entity(
|
|
132
146
|
eid,
|
|
133
147
|
profile_id,
|
|
@@ -679,32 +679,47 @@ class RetrievalEngine:
|
|
|
679
679
|
fused: list,
|
|
680
680
|
ch_results: dict[str, list[tuple[str, float]]],
|
|
681
681
|
effective_limit: int,
|
|
682
|
-
min_per_channel: int = 2,
|
|
683
682
|
) -> list:
|
|
684
|
-
"""
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
683
|
+
"""Keep strong lexical and structure evidence visible in the result cap.
|
|
684
|
+
|
|
685
|
+
A semantic channel with a larger weight can fill a small result limit
|
|
686
|
+
even when BM25 has an exact, high-signal hit. That broke the
|
|
687
|
+
``queryable now`` ingestion contract: a freshly inserted FTS row could
|
|
688
|
+
exist durably but remain invisible to immediate recall. Reserve one
|
|
689
|
+
capped slot for a strong BM25 hit and two for a structure channel when
|
|
690
|
+
such candidates exist, without returning more than ``effective_limit``.
|
|
690
691
|
"""
|
|
691
|
-
|
|
692
|
+
channel_minimums = (
|
|
693
|
+
("bm25", 1, 0.0),
|
|
694
|
+
("entity_graph", 2, 0.0),
|
|
695
|
+
)
|
|
692
696
|
top_ids = {fr.fact_id for fr in top}
|
|
693
697
|
|
|
694
698
|
promoted = []
|
|
695
|
-
for ch_name in
|
|
699
|
+
for ch_name, minimum, score_floor in channel_minimums:
|
|
696
700
|
ch_items = ch_results.get(ch_name, [])
|
|
697
701
|
if not ch_items:
|
|
698
702
|
continue
|
|
699
703
|
|
|
700
|
-
|
|
701
|
-
|
|
704
|
+
eligible_ids = {
|
|
705
|
+
fid
|
|
706
|
+
for fid, score in ch_items
|
|
707
|
+
if (
|
|
708
|
+
float(score) > score_floor
|
|
709
|
+
if ch_name == "bm25"
|
|
710
|
+
else float(score) >= score_floor
|
|
711
|
+
)
|
|
712
|
+
}
|
|
713
|
+
if not eligible_ids:
|
|
714
|
+
continue
|
|
715
|
+
|
|
716
|
+
present = sum(1 for fid in eligible_ids if fid in top_ids)
|
|
717
|
+
if present >= minimum:
|
|
702
718
|
continue
|
|
703
719
|
|
|
704
|
-
needed =
|
|
705
|
-
ch_fids = {fid for fid, _ in ch_items}
|
|
720
|
+
needed = minimum - present
|
|
706
721
|
for fr in fused:
|
|
707
|
-
if fr.fact_id in
|
|
722
|
+
if fr.fact_id in eligible_ids and fr.fact_id not in top_ids:
|
|
708
723
|
promoted.append(fr)
|
|
709
724
|
top_ids.add(fr.fact_id)
|
|
710
725
|
needed -= 1
|
|
@@ -714,10 +729,15 @@ class RetrievalEngine:
|
|
|
714
729
|
if not promoted:
|
|
715
730
|
return top
|
|
716
731
|
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
732
|
+
selected = promoted[:effective_limit]
|
|
733
|
+
result = list(top[:effective_limit])
|
|
734
|
+
free_slots = max(0, effective_limit - len(result))
|
|
735
|
+
result.extend(selected[:free_slots])
|
|
736
|
+
remaining = selected[free_slots:]
|
|
737
|
+
if remaining:
|
|
738
|
+
keep = max(0, effective_limit - len(remaining))
|
|
739
|
+
result = result[:keep] + remaining
|
|
740
|
+
return result[:effective_limit]
|
|
721
741
|
|
|
722
742
|
# -- Channel execution --------------------------------------------------
|
|
723
743
|
|
|
@@ -853,14 +873,26 @@ class RetrievalEngine:
|
|
|
853
873
|
q_emb, profile_id, self._config.bm25_top_k,
|
|
854
874
|
)
|
|
855
875
|
|
|
856
|
-
#
|
|
876
|
+
# Each local channel gets a strict latency budget. A slow graph walk
|
|
877
|
+
# must not make an interactive recall wait 30 seconds; completed
|
|
878
|
+
# channels still participate in fusion and the timeout is observable.
|
|
879
|
+
channel_timeout_seconds = 1.0
|
|
880
|
+
# One shared deadline keeps parallel dispatch genuinely bounded. A
|
|
881
|
+
# per-future timeout here would serialise the wait and turn five slow
|
|
882
|
+
# channels into five seconds of UI latency.
|
|
883
|
+
done, pending = concurrent.futures.wait(
|
|
884
|
+
futures.values(), timeout=channel_timeout_seconds,
|
|
885
|
+
)
|
|
857
886
|
for name, fut in futures.items():
|
|
887
|
+
if fut in pending:
|
|
888
|
+
logger.warning("Channel %s exceeded %.1fs latency budget", name, channel_timeout_seconds)
|
|
889
|
+
continue
|
|
858
890
|
try:
|
|
859
|
-
ch_name, result = fut.result(
|
|
891
|
+
ch_name, result = fut.result()
|
|
860
892
|
if result:
|
|
861
893
|
out[ch_name] = result
|
|
862
894
|
except Exception as exc:
|
|
863
|
-
logger.warning("Channel %s
|
|
895
|
+
logger.warning("Channel %s failed: %s", name, exc)
|
|
864
896
|
|
|
865
897
|
# Apply registered post-retrieval filters (forgetting filter, etc.)
|
|
866
898
|
if hasattr(self, '_registry') and self._registry._filters:
|
|
@@ -25,9 +25,8 @@ import sys
|
|
|
25
25
|
import threading
|
|
26
26
|
import time
|
|
27
27
|
import weakref
|
|
28
|
-
from typing import Any
|
|
29
|
-
|
|
30
28
|
from pathlib import Path
|
|
29
|
+
from typing import Any
|
|
31
30
|
|
|
32
31
|
from superlocalmemory.infra.data_root import state_path
|
|
33
32
|
from superlocalmemory.storage.models import AtomicFact
|
|
@@ -57,9 +56,9 @@ _live_rerankers: set[weakref.ref] = set()
|
|
|
57
56
|
|
|
58
57
|
logger = logging.getLogger(__name__)
|
|
59
58
|
|
|
60
|
-
_IDLE_TIMEOUT_SECONDS =
|
|
59
|
+
_IDLE_TIMEOUT_SECONDS = 1800 # V3.8.1: keep interactive sessions warm.
|
|
61
60
|
# V3.3.12: Configurable via SLM_RERANKER_IDLE_TIMEOUT env var.
|
|
62
|
-
#
|
|
61
|
+
# Low-RAM installations can retain aggressive recycling through the override.
|
|
63
62
|
# Set ``SLM_RERANKER_IDLE_TIMEOUT=120`` + ``slm restart`` to revert.
|
|
64
63
|
_IDLE_TIMEOUT_SECONDS = int(os.environ.get("SLM_RERANKER_IDLE_TIMEOUT", _IDLE_TIMEOUT_SECONDS))
|
|
65
64
|
_SUBPROCESS_RESPONSE_TIMEOUT = 15 # v3.4.52: 15s (was 180s). Long timeout blocked the
|
|
@@ -139,14 +139,19 @@ class SpreadingActivation:
|
|
|
139
139
|
)
|
|
140
140
|
# Owner-partitioned vector indexes cannot discover opted-in peers.
|
|
141
141
|
# Add visible external embeddings with the same cosine seed signal.
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
142
|
+
# v3.8.2 perf: external (global/shared) facts only matter for a
|
|
143
|
+
# cross-scope read. For the default personal scope this query always
|
|
144
|
+
# returns [] — skip it to remove a per-recall DB round-trip.
|
|
145
|
+
external_facts: list = []
|
|
146
|
+
if include_global or include_shared:
|
|
147
|
+
try:
|
|
148
|
+
external_facts = self._db.get_external_visible_facts(
|
|
149
|
+
profile_id,
|
|
150
|
+
include_global=include_global,
|
|
151
|
+
include_shared=include_shared,
|
|
152
|
+
)
|
|
153
|
+
except Exception:
|
|
154
|
+
external_facts = []
|
|
150
155
|
q_vec = np.array(query, dtype=np.float32)
|
|
151
156
|
q_norm = float(np.linalg.norm(q_vec))
|
|
152
157
|
combined = {fact_id: score for fact_id, score in seed_results}
|
|
@@ -162,18 +167,26 @@ class SpreadingActivation:
|
|
|
162
167
|
continue
|
|
163
168
|
score = (float(np.dot(q_vec, fact_vec) / denominator) + 1.0) / 2.0
|
|
164
169
|
combined[fact.fact_id] = max(combined.get(fact.fact_id, 0.0), score)
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
170
|
+
# v3.8.2 perf: seeds come from this profile's own vector index /
|
|
171
|
+
# get_all_facts(profile_id), so for personal scope they are already
|
|
172
|
+
# authorized. Only re-authorize when a cross-scope read merged in
|
|
173
|
+
# global/shared candidates. filter_authorized_results below remains the
|
|
174
|
+
# security net on the returned set.
|
|
175
|
+
if include_global or include_shared:
|
|
176
|
+
allowed_seeds = authorized_fact_ids(
|
|
177
|
+
self._db,
|
|
178
|
+
combined,
|
|
179
|
+
profile_id,
|
|
180
|
+
include_global=include_global,
|
|
181
|
+
include_shared=include_shared,
|
|
182
|
+
)
|
|
183
|
+
seed_results = [
|
|
184
|
+
(fact_id, score)
|
|
185
|
+
for fact_id, score in combined.items()
|
|
186
|
+
if fact_id in allowed_seeds
|
|
187
|
+
]
|
|
188
|
+
else:
|
|
189
|
+
seed_results = list(combined.items())
|
|
177
190
|
if not seed_results:
|
|
178
191
|
return []
|
|
179
192
|
|
|
@@ -186,13 +199,18 @@ class SpreadingActivation:
|
|
|
186
199
|
)
|
|
187
200
|
cached = self._get_cached_results(query_hash, profile_id)
|
|
188
201
|
if cached:
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
202
|
+
# v3.8.2 perf: cached activations were produced from this profile's
|
|
203
|
+
# own propagation; personal-scope hits need no re-authorization.
|
|
204
|
+
# Cross-scope hits still pass the fail-closed filter.
|
|
205
|
+
if include_global or include_shared:
|
|
206
|
+
return filter_authorized_results(
|
|
207
|
+
self._db,
|
|
208
|
+
cached,
|
|
209
|
+
profile_id,
|
|
210
|
+
include_global=include_global,
|
|
211
|
+
include_shared=include_shared,
|
|
212
|
+
)[:top_k]
|
|
213
|
+
return cached[:top_k]
|
|
196
214
|
|
|
197
215
|
# Run 5-step spreading activation
|
|
198
216
|
activations = self._propagate(
|
|
@@ -312,7 +330,15 @@ class SpreadingActivation:
|
|
|
312
330
|
if activation < 0.001:
|
|
313
331
|
continue
|
|
314
332
|
|
|
315
|
-
# Get neighbors from BOTH tables (Rule 13) — cached per node
|
|
333
|
+
# Get neighbors from BOTH tables (Rule 13) — cached per node.
|
|
334
|
+
# v3.8.2 perf-fix: _get_unified_neighbors already filters edges by
|
|
335
|
+
# the scope predicate (graph_edges via _scope_where; association_edges
|
|
336
|
+
# by profile_id), so for the default personal scope the returned
|
|
337
|
+
# neighbors are inherently authorized. The per-node re-authorization
|
|
338
|
+
# (2 DB round-trips/node, ~30–60 per recall — the primary 3.8 latency
|
|
339
|
+
# regression) is only required when a cross-scope read can surface
|
|
340
|
+
# global/shared neighbors. filter_authorized_results() on the returned
|
|
341
|
+
# set remains the security net for every scope.
|
|
316
342
|
if node_id not in neighbor_cache:
|
|
317
343
|
raw_neighbors = self._get_unified_neighbors(
|
|
318
344
|
node_id,
|
|
@@ -320,16 +346,20 @@ class SpreadingActivation:
|
|
|
320
346
|
include_global=include_global,
|
|
321
347
|
include_shared=include_shared,
|
|
322
348
|
)
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
349
|
+
if include_global or include_shared:
|
|
350
|
+
allowed_neighbors = authorized_fact_ids(
|
|
351
|
+
self._db,
|
|
352
|
+
(neighbor_id for neighbor_id, _weight in raw_neighbors),
|
|
353
|
+
profile_id,
|
|
354
|
+
include_global=include_global,
|
|
355
|
+
include_shared=include_shared,
|
|
356
|
+
)
|
|
357
|
+
neighbor_cache[node_id] = [
|
|
358
|
+
item for item in raw_neighbors
|
|
359
|
+
if item[0] in allowed_neighbors
|
|
360
|
+
]
|
|
361
|
+
else:
|
|
362
|
+
neighbor_cache[node_id] = raw_neighbors
|
|
333
363
|
neighbors = neighbor_cache[node_id]
|
|
334
364
|
|
|
335
365
|
# Out-degree for fan effect normalization
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Process-safe, durable read/modify/write access to ``config.json``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import tempfile
|
|
8
|
+
import threading
|
|
9
|
+
from contextlib import contextmanager
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Callable, Iterator
|
|
12
|
+
|
|
13
|
+
_PROCESS_LOCK = threading.RLock()
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@contextmanager
|
|
17
|
+
def _file_lock(path: Path) -> Iterator[None]:
|
|
18
|
+
"""Serialize config access across daemon, CLI, and worker processes."""
|
|
19
|
+
lock_path = path.with_name(f".{path.name}.lock")
|
|
20
|
+
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
21
|
+
with _PROCESS_LOCK, lock_path.open("a+b") as handle:
|
|
22
|
+
if os.name == "nt":
|
|
23
|
+
import msvcrt
|
|
24
|
+
|
|
25
|
+
handle.seek(0)
|
|
26
|
+
handle.write(b"\0")
|
|
27
|
+
handle.flush()
|
|
28
|
+
handle.seek(0)
|
|
29
|
+
msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
|
|
30
|
+
else:
|
|
31
|
+
import fcntl
|
|
32
|
+
|
|
33
|
+
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
|
34
|
+
try:
|
|
35
|
+
yield
|
|
36
|
+
finally:
|
|
37
|
+
if os.name == "nt":
|
|
38
|
+
handle.seek(0)
|
|
39
|
+
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
|
40
|
+
else:
|
|
41
|
+
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _read_unlocked(path: Path) -> dict:
|
|
45
|
+
if not path.exists():
|
|
46
|
+
return {}
|
|
47
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
48
|
+
if not isinstance(value, dict):
|
|
49
|
+
raise ValueError("config root must be a JSON object")
|
|
50
|
+
return value
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def read_config(path: Path) -> dict:
|
|
54
|
+
"""Return one coherent config snapshot under the interprocess lock."""
|
|
55
|
+
with _file_lock(path):
|
|
56
|
+
return _read_unlocked(path)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def update_config(path: Path, update: Callable[[dict], None]) -> dict:
|
|
60
|
+
"""Atomically update and durably replace a config JSON object."""
|
|
61
|
+
with _file_lock(path):
|
|
62
|
+
data = _read_unlocked(path)
|
|
63
|
+
update(data)
|
|
64
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
65
|
+
descriptor, temp_name = tempfile.mkstemp(
|
|
66
|
+
prefix=f".{path.name}.",
|
|
67
|
+
suffix=".tmp",
|
|
68
|
+
dir=path.parent,
|
|
69
|
+
)
|
|
70
|
+
temp_path = Path(temp_name)
|
|
71
|
+
try:
|
|
72
|
+
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
|
|
73
|
+
json.dump(data, stream, indent=2)
|
|
74
|
+
stream.write("\n")
|
|
75
|
+
stream.flush()
|
|
76
|
+
os.fsync(stream.fileno())
|
|
77
|
+
os.chmod(temp_path, 0o600)
|
|
78
|
+
os.replace(temp_path, path)
|
|
79
|
+
if hasattr(os, "O_DIRECTORY"):
|
|
80
|
+
directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
|
|
81
|
+
try:
|
|
82
|
+
os.fsync(directory_fd)
|
|
83
|
+
finally:
|
|
84
|
+
os.close(directory_fd)
|
|
85
|
+
finally:
|
|
86
|
+
temp_path.unlink(missing_ok=True)
|
|
87
|
+
return data
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
__all__ = ("read_config", "update_config")
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Exact browser-origin validation for the local dashboard."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from urllib.parse import urlsplit
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
_LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"})
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def origin_is_loopback(origin: str) -> bool:
|
|
12
|
+
"""Return whether an Origin is absent or an exact HTTP(S) loopback URL."""
|
|
13
|
+
if not origin:
|
|
14
|
+
return True
|
|
15
|
+
try:
|
|
16
|
+
parsed = urlsplit(origin)
|
|
17
|
+
# Accessing port validates malformed/non-numeric port values.
|
|
18
|
+
_ = parsed.port
|
|
19
|
+
except (TypeError, ValueError):
|
|
20
|
+
return False
|
|
21
|
+
return (
|
|
22
|
+
parsed.scheme in {"http", "https"}
|
|
23
|
+
and parsed.hostname is not None
|
|
24
|
+
and parsed.hostname.lower() in _LOOPBACK_HOSTS
|
|
25
|
+
and parsed.username is None
|
|
26
|
+
and parsed.password is None
|
|
27
|
+
and parsed.path in {"", "/"}
|
|
28
|
+
and not parsed.query
|
|
29
|
+
and not parsed.fragment
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def origin_is_daemon(origin: str, *, port: int) -> bool:
|
|
34
|
+
"""Return whether ``origin`` is one of this daemon's loopback aliases.
|
|
35
|
+
|
|
36
|
+
A loopback host alone is not a browser trust boundary: another local web
|
|
37
|
+
server can run on a different port. Credentialless dashboard writes are
|
|
38
|
+
therefore limited to the port owned by this daemon. Authenticated local
|
|
39
|
+
integrations are handled separately by the write-identity boundary.
|
|
40
|
+
"""
|
|
41
|
+
if not origin_is_loopback(origin):
|
|
42
|
+
return False
|
|
43
|
+
try:
|
|
44
|
+
parsed = urlsplit(origin)
|
|
45
|
+
return parsed.port == port
|
|
46
|
+
except (TypeError, ValueError):
|
|
47
|
+
return False
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
__all__ = ("origin_is_daemon", "origin_is_loopback")
|