superlocalmemory 3.6.10 → 3.6.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -0
- package/README.md +62 -6
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/skills/slm-optimize/README.md +55 -0
- package/skills/slm-optimize/SKILL.md +139 -0
- package/src/superlocalmemory/cli/commands.py +1 -0
- package/src/superlocalmemory/cli/daemon.py +0 -407
- package/src/superlocalmemory/cli/main.py +3 -1
- package/src/superlocalmemory/core/context_cache.py +4 -1
- package/src/superlocalmemory/core/fact_consolidator.py +4 -1
- package/src/superlocalmemory/core/remote_mode.py +197 -0
- package/src/superlocalmemory/core/summarizer.py +4 -1
- package/src/superlocalmemory/llm/backbone.py +7 -1
- package/src/superlocalmemory/mcp/agent_context.py +7 -3
- package/src/superlocalmemory/mcp/server.py +4 -0
- package/src/superlocalmemory/mcp/tools_core.py +13 -1
- package/src/superlocalmemory/mcp/tools_mesh.py +14 -6
- package/src/superlocalmemory/mcp/tools_optimize.py +304 -0
- package/src/superlocalmemory/mesh/broker.py +15 -4
- package/src/superlocalmemory/optimize/compress/router.py +9 -4
- package/src/superlocalmemory/optimize/storage/db.py +40 -2
- package/src/superlocalmemory/server/api.py +11 -3
- package/src/superlocalmemory/server/routes/mesh.py +13 -0
- package/src/superlocalmemory/server/routes/token.py +14 -2
- package/src/superlocalmemory/server/routes/v3_api.py +83 -17
- package/src/superlocalmemory/server/ui.py +15 -4
- package/src/superlocalmemory/server/unified_daemon.py +96 -160
- package/src/superlocalmemory/storage/database.py +10 -1
- package/src/superlocalmemory/ui/js/auto-settings.js +24 -0
- package/src/superlocalmemory.egg-info/PKG-INFO +63 -7
- package/src/superlocalmemory.egg-info/SOURCES.txt +2 -0
|
@@ -15,8 +15,12 @@ import contextvars
|
|
|
15
15
|
import os
|
|
16
16
|
import re
|
|
17
17
|
|
|
18
|
+
# v3.6.12 (parity-1): default is "" (the "no agent routed" sentinel), NOT the
|
|
19
|
+
# user-visible "mcp_client". Sanitized agent ids are [A-Za-z0-9._-], so "" can
|
|
20
|
+
# never collide — a client that explicitly routes to /mcp/mcp_client is now
|
|
21
|
+
# distinguishable from a bare /mcp/ request with no agent segment.
|
|
18
22
|
_current_agent_id: contextvars.ContextVar[str] = contextvars.ContextVar(
|
|
19
|
-
"slm_agent_id", default="
|
|
23
|
+
"slm_agent_id", default=""
|
|
20
24
|
)
|
|
21
25
|
|
|
22
26
|
# Agent ids arrive from an untrusted URL path segment. They are ATTRIBUTION
|
|
@@ -42,8 +46,8 @@ def get_current_agent_id(env_fallback: bool = True) -> str:
|
|
|
42
46
|
fall through to the SLM_AGENT_ID env var instead.
|
|
43
47
|
"""
|
|
44
48
|
ctx_id = _current_agent_id.get()
|
|
45
|
-
if ctx_id
|
|
46
|
-
return ctx_id
|
|
49
|
+
if ctx_id:
|
|
50
|
+
return ctx_id # an explicitly-routed agent id (incl. "mcp_client")
|
|
47
51
|
if env_fallback:
|
|
48
52
|
return os.environ.get("SLM_AGENT_ID", "mcp_client")
|
|
49
53
|
return "mcp_client"
|
|
@@ -104,6 +104,8 @@ _ESSENTIAL_TOOLS: set[str] = {
|
|
|
104
104
|
"reinforce_assertion", "contradict_assertion",
|
|
105
105
|
# v3.4.11: Skill evolution (3)
|
|
106
106
|
"evolve_skill", "skill_health", "skill_lineage",
|
|
107
|
+
# v3.6.11: Surface B Optimize tools (5)
|
|
108
|
+
"slm_compress", "slm_retrieve", "slm_cache_set", "slm_cache_get", "slm_optimize_stats",
|
|
107
109
|
}
|
|
108
110
|
|
|
109
111
|
# v3.4.4: Mesh tools — enabled if mesh_enabled in config or SLM_MCP_MESH_TOOLS=1
|
|
@@ -189,6 +191,8 @@ register_code_graph_tools(_target, get_engine) # CodeGraph: filtered like other
|
|
|
189
191
|
register_mesh_tools(_target, get_engine) # v3.4.4: Mesh P2P tools — ships with SLM, no separate slm-mesh needed
|
|
190
192
|
register_learning_tools(_target, get_engine) # v3.4.7: Two-way learning tools
|
|
191
193
|
register_evolution_tools(_target, get_engine) # v3.4.11: Skill evolution tools
|
|
194
|
+
from superlocalmemory.mcp.tools_optimize import register_optimize_tools
|
|
195
|
+
register_optimize_tools(_target) # v3.6.11: Surface B Optimize tools (proxy-free)
|
|
192
196
|
|
|
193
197
|
|
|
194
198
|
# V3.3.21: Eager engine warmup — start initializing BEFORE first tool call.
|
|
@@ -309,7 +309,10 @@ def register_core_tools(server, get_engine: Callable) -> None:
|
|
|
309
309
|
try:
|
|
310
310
|
engine = get_engine()
|
|
311
311
|
pid = profile_id or engine.profile_id
|
|
312
|
-
|
|
312
|
+
# v3.6.12 (search-2): push the limit into the query — was loading the
|
|
313
|
+
# ENTIRE facts table (deserializing every 768-float embedding) just
|
|
314
|
+
# to return the top N. get_all_facts preserves created_at DESC order.
|
|
315
|
+
facts = engine._db.get_all_facts(pid, limit=limit)
|
|
313
316
|
items = []
|
|
314
317
|
for f in facts:
|
|
315
318
|
items.append({
|
|
@@ -401,6 +404,15 @@ def register_core_tools(server, get_engine: Callable) -> None:
|
|
|
401
404
|
# Dashboard not installed — profile switch still works for MCP/CLI
|
|
402
405
|
logger.debug("Dashboard routes not available, profile set in engine only")
|
|
403
406
|
|
|
407
|
+
# v3.6.12 (search-3): recall/delete run in a separate worker
|
|
408
|
+
# subprocess that caches its engine (and profile_id) at init. Recycle
|
|
409
|
+
# it so the NEXT recall uses the new profile instead of the stale one.
|
|
410
|
+
try:
|
|
411
|
+
from superlocalmemory.core.worker_pool import WorkerPool
|
|
412
|
+
WorkerPool.shared().shutdown()
|
|
413
|
+
except Exception:
|
|
414
|
+
logger.debug("worker-pool recycle on profile switch skipped")
|
|
415
|
+
|
|
404
416
|
return {
|
|
405
417
|
"success": True,
|
|
406
418
|
"previous_profile": old,
|
|
@@ -76,7 +76,7 @@ def _mesh_request(method: str, path: str, body: dict | None = None) -> dict | No
|
|
|
76
76
|
|
|
77
77
|
def _ensure_registered() -> None:
|
|
78
78
|
"""Register this session with the mesh broker if not already."""
|
|
79
|
-
global _REGISTERED, _PROJECT_PATH
|
|
79
|
+
global _REGISTERED, _PROJECT_PATH, _PEER_ID
|
|
80
80
|
if _REGISTERED:
|
|
81
81
|
return
|
|
82
82
|
|
|
@@ -89,6 +89,11 @@ def _ensure_registered() -> None:
|
|
|
89
89
|
"agent_type": os.environ.get("CLAUDE_AGENT_TYPE", "claude_code"),
|
|
90
90
|
})
|
|
91
91
|
if result:
|
|
92
|
+
# v3.6.12 (mesh-1): the broker mints its OWN peer_id (RegisterRequest has
|
|
93
|
+
# no peer_id field, so our body value is dropped by pydantic). Adopt the
|
|
94
|
+
# broker's id BEFORE starting the heartbeat, otherwise heartbeat/send/
|
|
95
|
+
# inbox all target a non-existent peer → 404s and the session is reaped.
|
|
96
|
+
_PEER_ID = result.get("peer_id", _PEER_ID)
|
|
92
97
|
_REGISTERED = True
|
|
93
98
|
_start_heartbeat()
|
|
94
99
|
pending = result.get("pending_messages", 0)
|
|
@@ -191,7 +196,7 @@ def register_mesh_tools(server, get_engine: Callable) -> None:
|
|
|
191
196
|
_mesh_request, "POST", "/send",
|
|
192
197
|
{"from_peer": _PEER_ID, "to_peer": to, "content": message},
|
|
193
198
|
)
|
|
194
|
-
return result or {"error": "Failed to send message"}
|
|
199
|
+
return result or {"ok": False, "error": "Failed to send message"}
|
|
195
200
|
|
|
196
201
|
@server.tool()
|
|
197
202
|
async def mesh_inbox() -> dict:
|
|
@@ -207,8 +212,11 @@ def register_mesh_tools(server, get_engine: Callable) -> None:
|
|
|
207
212
|
_mesh_request, "GET", f"/inbox/{_PEER_ID}?project_path={project}",
|
|
208
213
|
)
|
|
209
214
|
msg_list = (messages or {}).get("messages", [])
|
|
210
|
-
# Auto-mark unread messages as read
|
|
211
|
-
|
|
215
|
+
# Auto-mark unread messages as read. v3.6.12 (failopen-2): use .get("id")
|
|
216
|
+
# — a malformed broker message without an "id" key used to raise KeyError
|
|
217
|
+
# out to the agent, violating the never-raise contract.
|
|
218
|
+
unread_ids = [m["id"] for m in msg_list
|
|
219
|
+
if not m.get("read") and m.get("id") is not None]
|
|
212
220
|
if unread_ids:
|
|
213
221
|
await asyncio.to_thread(
|
|
214
222
|
_mesh_request, "POST", f"/inbox/{_PEER_ID}/read",
|
|
@@ -239,7 +247,7 @@ def register_mesh_tools(server, get_engine: Callable) -> None:
|
|
|
239
247
|
_mesh_request, "POST", "/state",
|
|
240
248
|
{"key": key, "value": value, "set_by": _PEER_ID},
|
|
241
249
|
)
|
|
242
|
-
return result or {"error": "Failed to set state"}
|
|
250
|
+
return result or {"ok": False, "error": "Failed to set state"}
|
|
243
251
|
|
|
244
252
|
if key:
|
|
245
253
|
result = await asyncio.to_thread(_mesh_request, "GET", f"/state/{key}")
|
|
@@ -266,7 +274,7 @@ def register_mesh_tools(server, get_engine: Callable) -> None:
|
|
|
266
274
|
_mesh_request, "POST", "/lock",
|
|
267
275
|
{"file_path": file_path, "action": action, "locked_by": _PEER_ID},
|
|
268
276
|
)
|
|
269
|
-
return result or {"error": "Lock operation failed"}
|
|
277
|
+
return result or {"ok": False, "error": "Lock operation failed"}
|
|
270
278
|
|
|
271
279
|
@server.tool(annotations=ToolAnnotations(readOnlyHint=True))
|
|
272
280
|
async def mesh_events() -> dict:
|
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""SLM v3.6.11 — Surface B: MCP Optimize Tools.
|
|
6
|
+
|
|
7
|
+
Five proxy-free tools exposing compression (reversible via CCR) and
|
|
8
|
+
routed-result caching WITHOUT touching ANTHROPIC_BASE_URL, so the full
|
|
9
|
+
1M context window is preserved on any Claude subscription.
|
|
10
|
+
|
|
11
|
+
Primary Claude conversation turns CANNOT be cached without a proxy.
|
|
12
|
+
These tools cache results the agent explicitly routes through SLM.
|
|
13
|
+
|
|
14
|
+
Fail-open: every tool body is wrapped in try/except Exception.
|
|
15
|
+
Any internal error returns the input unchanged with ok:False — never raises.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import hashlib
|
|
21
|
+
import logging
|
|
22
|
+
import threading
|
|
23
|
+
import time
|
|
24
|
+
|
|
25
|
+
from mcp.types import ToolAnnotations
|
|
26
|
+
|
|
27
|
+
from superlocalmemory.mcp.agent_context import get_current_agent_id
|
|
28
|
+
from superlocalmemory.optimize.compress.ccr import CCRStore, _UUID4_RE
|
|
29
|
+
from superlocalmemory.optimize.compress.router import CompressRouter
|
|
30
|
+
from superlocalmemory.optimize.storage.db import CacheDB, _normalize_tenant_id
|
|
31
|
+
|
|
32
|
+
logger = logging.getLogger("slm.mcp.tools_optimize")
|
|
33
|
+
|
|
34
|
+
# ─── Size caps (CWE-400 guards) ───────────────────────────────────────────────
|
|
35
|
+
|
|
36
|
+
_MAX_COMPRESS_BYTES: int = 1_000_000
|
|
37
|
+
_MAX_KV_VALUE_BYTES: int = 1_000_000
|
|
38
|
+
_MAX_KV_KEY_CHARS: int = 512
|
|
39
|
+
|
|
40
|
+
# ─── Exported tool name list (used by server.py + tests) ─────────────────────
|
|
41
|
+
|
|
42
|
+
_OPTIMIZE_TOOL_NAMES = (
|
|
43
|
+
"slm_compress",
|
|
44
|
+
"slm_retrieve",
|
|
45
|
+
"slm_cache_set",
|
|
46
|
+
"slm_cache_get",
|
|
47
|
+
"slm_optimize_stats",
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
# ─── In-module KV counters (thread-safe; MetricsCollector is process-scoped) ─
|
|
51
|
+
|
|
52
|
+
_kv_lock = threading.Lock()
|
|
53
|
+
_kv_hits: int = 0
|
|
54
|
+
_kv_misses: int = 0
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _tenant() -> str:
|
|
58
|
+
# get_current_agent_id() never returns ""; "mcp_client" is its stdio sentinel.
|
|
59
|
+
return get_current_agent_id()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# ─── Tool registration ────────────────────────────────────────────────────────
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def register_optimize_tools(server) -> None:
|
|
66
|
+
"""Register the 5 Surface B optimize tools on *server*.
|
|
67
|
+
|
|
68
|
+
*server* is duck-typed: must support @server.tool() decorator pattern.
|
|
69
|
+
Compatible with FastMCP, _FilteredServer, and test _MockServer.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
@server.tool(annotations=ToolAnnotations(readOnlyHint=False, destructiveHint=False))
|
|
73
|
+
async def slm_compress(
|
|
74
|
+
content: str,
|
|
75
|
+
mode: str = "auto",
|
|
76
|
+
reversible: bool = True,
|
|
77
|
+
ttl_seconds: int = 86400,
|
|
78
|
+
) -> dict:
|
|
79
|
+
"""Compress text or tool output to reduce context window usage.
|
|
80
|
+
|
|
81
|
+
Returns compressed text. If lossy and reversible=True, also returns a
|
|
82
|
+
ccr_id — pass it to slm_retrieve to recover the exact original.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
content: Text to compress (max 1MB).
|
|
86
|
+
mode: "normalize" (lossless whitespace) | "auto" | "aggressive".
|
|
87
|
+
reversible: Store original in CCR for later retrieval.
|
|
88
|
+
ttl_seconds: CCR lifetime in seconds (default 24h).
|
|
89
|
+
"""
|
|
90
|
+
try:
|
|
91
|
+
if not isinstance(content, str) or not content:
|
|
92
|
+
return {
|
|
93
|
+
"ok": False, "compressed": content or "",
|
|
94
|
+
"strategy": "none", "tokens_before": 0, "tokens_after": 0,
|
|
95
|
+
"ratio": 1.0, "lossy": False, "ccr_id": None,
|
|
96
|
+
"note": "empty input",
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
note_parts: list[str] = []
|
|
100
|
+
if len(content.encode("utf-8")) > _MAX_COMPRESS_BYTES:
|
|
101
|
+
reversible = False
|
|
102
|
+
note_parts.append("content over 1MB: ccr skipped")
|
|
103
|
+
|
|
104
|
+
if mode == "normalize":
|
|
105
|
+
# @staticmethod — lossless whitespace collapse, no config/daemon dep.
|
|
106
|
+
normalized = CompressRouter._normalize_whitespace(content)
|
|
107
|
+
tb = len(content.split())
|
|
108
|
+
ta = len(normalized.split())
|
|
109
|
+
ratio = round(ta / tb, 4) if tb else 1.0
|
|
110
|
+
return {
|
|
111
|
+
"ok": True, "compressed": normalized, "strategy": "normalize",
|
|
112
|
+
"tokens_before": tb, "tokens_after": ta, "ratio": ratio,
|
|
113
|
+
"lossy": False, "ccr_id": None,
|
|
114
|
+
"note": " | ".join(note_parts) or None,
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
if mode == "aggressive":
|
|
118
|
+
note_parts.append(
|
|
119
|
+
"aggressive mode requires daemon compress_mode=aggressive in config"
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
res = CompressRouter.get_instance().compress_text(content)
|
|
123
|
+
|
|
124
|
+
ccr_id = None
|
|
125
|
+
if res.lossy and reversible:
|
|
126
|
+
stored = CCRStore.get_instance().store(
|
|
127
|
+
content.encode("utf-8"),
|
|
128
|
+
tenant_id=_tenant(),
|
|
129
|
+
ttl_seconds=ttl_seconds,
|
|
130
|
+
)
|
|
131
|
+
ccr_id = stored or None
|
|
132
|
+
if ccr_id:
|
|
133
|
+
note_parts.append("reversible: call slm_retrieve with this ccr_id")
|
|
134
|
+
|
|
135
|
+
ratio = (
|
|
136
|
+
round(res.tokens_after / res.tokens_before, 4)
|
|
137
|
+
if res.tokens_before else 1.0
|
|
138
|
+
)
|
|
139
|
+
return {
|
|
140
|
+
"ok": True, "compressed": res.compressed_text, "strategy": res.strategy,
|
|
141
|
+
"tokens_before": res.tokens_before, "tokens_after": res.tokens_after,
|
|
142
|
+
"ratio": ratio, "lossy": res.lossy, "ccr_id": ccr_id,
|
|
143
|
+
"note": " | ".join(note_parts) or None,
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
except Exception as exc:
|
|
147
|
+
logger.error("slm_compress failed (fail-open): %s", exc)
|
|
148
|
+
t = len(content.split()) if isinstance(content, str) else 0
|
|
149
|
+
return {
|
|
150
|
+
"ok": False,
|
|
151
|
+
"compressed": content if isinstance(content, str) else "",
|
|
152
|
+
"strategy": "none", "tokens_before": t, "tokens_after": t,
|
|
153
|
+
"ratio": 1.0, "lossy": False, "ccr_id": None,
|
|
154
|
+
"note": f"internal error: {exc}",
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
@server.tool(annotations=ToolAnnotations(readOnlyHint=True))
|
|
158
|
+
async def slm_retrieve(ccr_id: str) -> dict:
|
|
159
|
+
"""Retrieve original text stored during a lossy slm_compress call.
|
|
160
|
+
|
|
161
|
+
Do not log or share ccr_ids — they are unguessable session tokens, but
|
|
162
|
+
if exposed they allow retrieval by anyone with the daemon's decryption key.
|
|
163
|
+
|
|
164
|
+
Args:
|
|
165
|
+
ccr_id: UUID4 returned by slm_compress when reversible=True.
|
|
166
|
+
"""
|
|
167
|
+
try:
|
|
168
|
+
if not ccr_id or not _UUID4_RE.match(ccr_id):
|
|
169
|
+
return {
|
|
170
|
+
"ok": False, "content": None, "size_bytes": 0,
|
|
171
|
+
"error": "ccr_id must be a UUID4",
|
|
172
|
+
}
|
|
173
|
+
original = CCRStore.get_instance().retrieve(ccr_id)
|
|
174
|
+
if original is None:
|
|
175
|
+
return {
|
|
176
|
+
"ok": False, "content": None, "size_bytes": 0,
|
|
177
|
+
"error": "not found (expired / never stored / wrong id)",
|
|
178
|
+
}
|
|
179
|
+
size = len(original)
|
|
180
|
+
try:
|
|
181
|
+
text = original.decode("utf-8")
|
|
182
|
+
except UnicodeDecodeError:
|
|
183
|
+
text = original.decode("latin-1")
|
|
184
|
+
return {"ok": True, "content": text, "size_bytes": size, "error": None}
|
|
185
|
+
|
|
186
|
+
except Exception as exc:
|
|
187
|
+
logger.error("slm_retrieve failed (fail-open): %s", exc)
|
|
188
|
+
return {
|
|
189
|
+
"ok": False, "content": None, "size_bytes": 0,
|
|
190
|
+
"error": f"internal error: {exc}",
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
@server.tool(annotations=ToolAnnotations(readOnlyHint=False, destructiveHint=False))
|
|
194
|
+
async def slm_cache_set(key: str, value: str, ttl_seconds: int = 86400) -> dict:
|
|
195
|
+
"""Cache a result you want to reuse (tool output, file read, search result).
|
|
196
|
+
|
|
197
|
+
This caches results the agent explicitly routes through SLM — NOT the
|
|
198
|
+
Claude conversation turn (impossible without a proxy).
|
|
199
|
+
|
|
200
|
+
Do not cache secrets, credentials, or ccr_ids via this tool.
|
|
201
|
+
|
|
202
|
+
Args:
|
|
203
|
+
key: Cache key (max 512 chars). Namespaced per agent automatically.
|
|
204
|
+
value: Value to store as string (max 1MB).
|
|
205
|
+
ttl_seconds: Time-to-live in seconds (default 24h).
|
|
206
|
+
"""
|
|
207
|
+
try:
|
|
208
|
+
if not key or len(key) > _MAX_KV_KEY_CHARS:
|
|
209
|
+
return {
|
|
210
|
+
"ok": False, "stored": False,
|
|
211
|
+
"note": f"key must be 1–{_MAX_KV_KEY_CHARS} chars",
|
|
212
|
+
}
|
|
213
|
+
value_bytes = value.encode("utf-8")
|
|
214
|
+
if len(value_bytes) > _MAX_KV_VALUE_BYTES:
|
|
215
|
+
return {"ok": False, "stored": False, "note": "value exceeds 1MB limit"}
|
|
216
|
+
|
|
217
|
+
tenant = _tenant()
|
|
218
|
+
cache_key = hashlib.sha256(f"mcpkv:{tenant}:{key}".encode()).hexdigest()
|
|
219
|
+
norm_tid = _normalize_tenant_id(tenant)
|
|
220
|
+
ttl_exp = time.time() + ttl_seconds
|
|
221
|
+
|
|
222
|
+
CacheDB.get_default().set(
|
|
223
|
+
cache_key, norm_tid, value_bytes,
|
|
224
|
+
model="mcp-kv", ttl_expires=ttl_exp, tags=["mcp-kv"],
|
|
225
|
+
)
|
|
226
|
+
return {"ok": True, "stored": True, "note": None}
|
|
227
|
+
|
|
228
|
+
except Exception as exc:
|
|
229
|
+
logger.error("slm_cache_set failed (fail-open): %s", exc)
|
|
230
|
+
return {"ok": False, "stored": False, "note": f"internal error: {exc}"}
|
|
231
|
+
|
|
232
|
+
@server.tool(annotations=ToolAnnotations(readOnlyHint=True))
|
|
233
|
+
async def slm_cache_get(key: str) -> dict:
|
|
234
|
+
"""Retrieve a previously cached result.
|
|
235
|
+
|
|
236
|
+
Returns hit:True + value if the key exists and has not expired.
|
|
237
|
+
Returns hit:False (never raises) on miss, expiry, or any error.
|
|
238
|
+
|
|
239
|
+
Args:
|
|
240
|
+
key: Cache key used in slm_cache_set.
|
|
241
|
+
"""
|
|
242
|
+
global _kv_hits, _kv_misses
|
|
243
|
+
try:
|
|
244
|
+
if not key or len(key) > _MAX_KV_KEY_CHARS:
|
|
245
|
+
return {
|
|
246
|
+
"ok": False, "hit": False, "value": None,
|
|
247
|
+
"note": f"key must be 1–{_MAX_KV_KEY_CHARS} chars",
|
|
248
|
+
}
|
|
249
|
+
tenant = _tenant()
|
|
250
|
+
cache_key = hashlib.sha256(f"mcpkv:{tenant}:{key}".encode()).hexdigest()
|
|
251
|
+
norm_tid = _normalize_tenant_id(tenant)
|
|
252
|
+
|
|
253
|
+
blob = CacheDB.get_default().get_value(cache_key, norm_tid)
|
|
254
|
+
if blob is None:
|
|
255
|
+
with _kv_lock:
|
|
256
|
+
_kv_misses += 1
|
|
257
|
+
return {"ok": True, "hit": False, "value": None, "note": None}
|
|
258
|
+
with _kv_lock:
|
|
259
|
+
_kv_hits += 1
|
|
260
|
+
return {"ok": True, "hit": True, "value": blob.decode("utf-8"), "note": None}
|
|
261
|
+
|
|
262
|
+
except Exception as exc:
|
|
263
|
+
logger.error("slm_cache_get failed (fail-open): %s", exc)
|
|
264
|
+
return {
|
|
265
|
+
"ok": False, "hit": False, "value": None,
|
|
266
|
+
"note": f"internal error: {exc}",
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
@server.tool(annotations=ToolAnnotations(readOnlyHint=True))
|
|
270
|
+
async def slm_optimize_stats() -> dict:
|
|
271
|
+
"""Return compression and cache statistics.
|
|
272
|
+
|
|
273
|
+
Proxy/compress stats are daemon-persisted (accurate across restarts).
|
|
274
|
+
KV stats are in-module counters for this MCP process session only.
|
|
275
|
+
"""
|
|
276
|
+
try:
|
|
277
|
+
snap = CacheDB.get_default().metrics_load()
|
|
278
|
+
with _kv_lock:
|
|
279
|
+
kv_h = _kv_hits
|
|
280
|
+
kv_m = _kv_misses
|
|
281
|
+
return {
|
|
282
|
+
"ok": True,
|
|
283
|
+
"compress_runs": snap.compress_runs,
|
|
284
|
+
"tokens_saved_compress": snap.tokens_saved_compress,
|
|
285
|
+
"cache_proxy_hits": snap.hits,
|
|
286
|
+
"cache_proxy_misses": snap.misses,
|
|
287
|
+
"cache_kv_hits": kv_h,
|
|
288
|
+
"cache_kv_misses": kv_m,
|
|
289
|
+
"ccr_note": (
|
|
290
|
+
"CCR entry count not tracked per-session; "
|
|
291
|
+
"see daemon /api/v1/metrics"
|
|
292
|
+
),
|
|
293
|
+
"note": "proxy stats are daemon-persisted; kv stats are this session only",
|
|
294
|
+
}
|
|
295
|
+
except Exception as exc:
|
|
296
|
+
logger.error("slm_optimize_stats failed (fail-open): %s", exc)
|
|
297
|
+
return {
|
|
298
|
+
"ok": False,
|
|
299
|
+
"compress_runs": 0, "tokens_saved_compress": 0,
|
|
300
|
+
"cache_proxy_hits": 0, "cache_proxy_misses": 0,
|
|
301
|
+
"cache_kv_hits": 0, "cache_kv_misses": 0,
|
|
302
|
+
"ccr_note": None,
|
|
303
|
+
"note": f"internal error: {exc}",
|
|
304
|
+
}
|
|
@@ -281,10 +281,13 @@ class MeshBroker:
|
|
|
281
281
|
try:
|
|
282
282
|
now = datetime.now(timezone.utc).isoformat()
|
|
283
283
|
# Direct messages to this peer
|
|
284
|
+
# v3.6.12 (mesh-3): only UNREAD direct messages — was returning read
|
|
285
|
+
# ones too, so every poll re-listed already-read messages until the
|
|
286
|
+
# 24h cleanup (broadcast/project already filter unread via mesh_reads).
|
|
284
287
|
direct = conn.execute(
|
|
285
288
|
"SELECT id, from_peer, to_peer, msg_type, content, read, created_at, "
|
|
286
289
|
"target_type, project_path FROM mesh_messages "
|
|
287
|
-
"WHERE to_peer=? AND target_type='peer' "
|
|
290
|
+
"WHERE to_peer=? AND target_type='peer' AND COALESCE(read, 0) = 0 "
|
|
288
291
|
"AND (expires_at IS NULL OR expires_at > ?) "
|
|
289
292
|
"ORDER BY created_at DESC LIMIT 100",
|
|
290
293
|
(peer_id, now),
|
|
@@ -411,10 +414,18 @@ class MeshBroker:
|
|
|
411
414
|
return {"ok": True, "action": "acquired"}
|
|
412
415
|
|
|
413
416
|
elif action == "release":
|
|
414
|
-
|
|
415
|
-
|
|
417
|
+
# v3.6.12 (mesh-2): report whether we actually released. The
|
|
418
|
+
# DELETE is correctly owner-scoped, but it previously returned
|
|
419
|
+
# released=ok:true even when a NON-owner released nothing.
|
|
420
|
+
cur = conn.execute(
|
|
421
|
+
"DELETE FROM mesh_locks WHERE file_path=? AND locked_by=?",
|
|
422
|
+
(file_path, locked_by),
|
|
423
|
+
)
|
|
416
424
|
conn.commit()
|
|
417
|
-
|
|
425
|
+
if cur.rowcount and cur.rowcount > 0:
|
|
426
|
+
return {"ok": True, "action": "released"}
|
|
427
|
+
return {"ok": False, "action": "not_released",
|
|
428
|
+
"error": "no lock held by this peer for that file"}
|
|
418
429
|
|
|
419
430
|
elif action == "query":
|
|
420
431
|
row = conn.execute(
|
|
@@ -317,11 +317,16 @@ class CompressRouter:
|
|
|
317
317
|
|
|
318
318
|
@staticmethod
|
|
319
319
|
def _normalize_whitespace(text: str) -> str:
|
|
320
|
-
"""Layer 1
|
|
320
|
+
"""Layer 1 safe: collapse runs of 3+ blank lines to a single blank line.
|
|
321
|
+
|
|
322
|
+
v3.6.12 (normalize-1): no longer rstrips trailing spaces per line — that
|
|
323
|
+
is LOSSY for Markdown hard breaks (two trailing spaces) and padded string
|
|
324
|
+
literals, which broke the 'lossless/safe' guarantee that callers (incl.
|
|
325
|
+
slm_compress mode=normalize) rely on. Only collapsing excess blank lines
|
|
326
|
+
remains, which is semantically safe.
|
|
327
|
+
"""
|
|
321
328
|
import re
|
|
322
|
-
|
|
323
|
-
lines = [line.rstrip() for line in text.split("\n")]
|
|
324
|
-
return "\n".join(lines)
|
|
329
|
+
return re.sub(r"\n{3,}", "\n\n", text)
|
|
325
330
|
|
|
326
331
|
# ── Lazy loaders ─────────────────────────────────────────────────────
|
|
327
332
|
|
|
@@ -14,7 +14,12 @@ REUSE: DatabaseManager from superlocalmemory/src/superlocalmemory/storage/databa
|
|
|
14
14
|
ENCRYPTION (resolves SEC-C-01 / CWE-312, NEW-M-01, NEW-M-02):
|
|
15
15
|
- All value BLOBs (llmcache_entries.value_blob) are AES-256-GCM encrypted.
|
|
16
16
|
- CCR original_blob is ALSO AES-256-GCM encrypted.
|
|
17
|
-
- Key
|
|
17
|
+
- Key storage: a single MACHINE-WIDE key file (~/.superlocalmemory/opt-key.bin,
|
|
18
|
+
0o600) is generated once and reused for all cache DBs on the machine. (The
|
|
19
|
+
per-DB salt below is persisted for provenance but does NOT make the AES key
|
|
20
|
+
per-DB — a single install has one llmcache.db, so a machine-wide key is the
|
|
21
|
+
intended model. A tampered/rotated key now degrades to a cache MISS, not a
|
|
22
|
+
crash — see _decrypt fail-open, v3.6.12 cache-1.)
|
|
18
23
|
- Salt: os.urandom(32) generated ONCE at DB creation, stored in
|
|
19
24
|
llmcache_schema_version.description='salt:<hex>'. NO hardcoded salt.
|
|
20
25
|
- Nonce (12 bytes random) prepended to each ciphertext.
|
|
@@ -43,6 +48,7 @@ from dataclasses import dataclass, field
|
|
|
43
48
|
from pathlib import Path
|
|
44
49
|
from typing import Any
|
|
45
50
|
|
|
51
|
+
from cryptography.exceptions import InvalidTag
|
|
46
52
|
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
|
47
53
|
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
|
|
48
54
|
from cryptography.hazmat.primitives import hashes
|
|
@@ -368,7 +374,15 @@ class CacheDB:
|
|
|
368
374
|
nonce = blob[:_AES_NONCE_BYTES]
|
|
369
375
|
ciphertext = blob[_AES_NONCE_BYTES:]
|
|
370
376
|
aesgcm = AESGCM(self._aes_key)
|
|
371
|
-
|
|
377
|
+
# v3.6.12 (cache-1): AES-GCM raises cryptography.exceptions.InvalidTag
|
|
378
|
+
# (NOT a ValueError subclass) on a tampered/wrong-key blob. Every caller
|
|
379
|
+
# catches ValueError to fail-open; convert InvalidTag -> ValueError here
|
|
380
|
+
# at the single chokepoint so a corrupt/rotated-key cache entry degrades
|
|
381
|
+
# to a miss instead of raising out of get()/get_value()/ccr_get().
|
|
382
|
+
try:
|
|
383
|
+
return aesgcm.decrypt(nonce, ciphertext, associated_data=None)
|
|
384
|
+
except InvalidTag as exc:
|
|
385
|
+
raise ValueError(f"AES-GCM authentication failed: {exc}") from exc
|
|
372
386
|
|
|
373
387
|
# ---- assertion ----
|
|
374
388
|
|
|
@@ -421,6 +435,30 @@ class CacheDB:
|
|
|
421
435
|
logger.warning("CacheDB.get failed (cache miss): %s", exc)
|
|
422
436
|
return None
|
|
423
437
|
|
|
438
|
+
def get_value(self, cache_key: str, tenant_id: str) -> bytes | None:
|
|
439
|
+
"""Pure value lookup — no hit_count increment (unlike get()).
|
|
440
|
+
|
|
441
|
+
Used by MCP KV tools which manage their own hit/miss counters.
|
|
442
|
+
Caller must have already normalized tenant_id. Fail-open: returns None on error.
|
|
443
|
+
"""
|
|
444
|
+
try:
|
|
445
|
+
rows = self._db.execute(
|
|
446
|
+
"SELECT value_blob, compressed FROM llmcache_entries "
|
|
447
|
+
"WHERE cache_key = ? AND tenant_id = ? "
|
|
448
|
+
"AND (ttl_expires IS NULL OR ttl_expires > ?) LIMIT 1",
|
|
449
|
+
(cache_key, tenant_id, time.time()),
|
|
450
|
+
)
|
|
451
|
+
if not rows:
|
|
452
|
+
return None
|
|
453
|
+
row = dict(rows[0])
|
|
454
|
+
plaintext = self._decrypt(row["value_blob"])
|
|
455
|
+
if row.get("compressed", 0):
|
|
456
|
+
plaintext = zlib.decompress(plaintext)
|
|
457
|
+
return plaintext
|
|
458
|
+
except (sqlite3.Error, ValueError, zlib.error) as exc:
|
|
459
|
+
logger.warning("CacheDB.get_value failed (fail-open): %s", exc)
|
|
460
|
+
return None
|
|
461
|
+
|
|
424
462
|
def set(
|
|
425
463
|
self,
|
|
426
464
|
key: str,
|
|
@@ -108,12 +108,20 @@ def create_app() -> FastAPI:
|
|
|
108
108
|
# Rate limiting (graceful)
|
|
109
109
|
try:
|
|
110
110
|
from superlocalmemory.infra.rate_limiter import RateLimiter
|
|
111
|
-
|
|
112
|
-
|
|
111
|
+
from superlocalmemory.core.remote_mode import (
|
|
112
|
+
rate_limit_config,
|
|
113
|
+
is_rate_limit_exempt,
|
|
114
|
+
)
|
|
115
|
+
# v3.6.12 (issue #40): env-tunable thresholds (defaults unchanged).
|
|
116
|
+
_rl_write, _rl_read, _rl_window = rate_limit_config()
|
|
117
|
+
_write_limiter = RateLimiter(max_requests=_rl_write, window_seconds=_rl_window)
|
|
118
|
+
_read_limiter = RateLimiter(max_requests=_rl_read, window_seconds=_rl_window)
|
|
113
119
|
|
|
114
120
|
@application.middleware("http")
|
|
115
121
|
async def rate_limit_middleware(request, call_next):
|
|
116
122
|
client_ip = request.client.host if request.client else "unknown"
|
|
123
|
+
if is_rate_limit_exempt(client_ip):
|
|
124
|
+
return await call_next(request)
|
|
117
125
|
is_write = request.method in ("POST", "PUT", "DELETE", "PATCH")
|
|
118
126
|
limiter = _write_limiter if is_write else _read_limiter
|
|
119
127
|
allowed, remaining = limiter.is_allowed(client_ip)
|
|
@@ -122,7 +130,7 @@ def create_app() -> FastAPI:
|
|
|
122
130
|
return JSONResponse(
|
|
123
131
|
status_code=429,
|
|
124
132
|
content={"error": "Too many requests."},
|
|
125
|
-
headers={"Retry-After": str(limiter.
|
|
133
|
+
headers={"Retry-After": str(limiter.window)},
|
|
126
134
|
)
|
|
127
135
|
response = await call_next(request)
|
|
128
136
|
response.headers["X-RateLimit-Remaining"] = str(remaining)
|
|
@@ -77,6 +77,19 @@ def _get_broker(request: Request):
|
|
|
77
77
|
config = getattr(request.app.state, 'config', None)
|
|
78
78
|
if config and not getattr(config, 'mesh_enabled', True):
|
|
79
79
|
raise HTTPException(503, detail="Mesh disabled in config")
|
|
80
|
+
# v3.6.12 (mesh-1 security): SLM_MESH_SHARED_SECRET was read by the broker but
|
|
81
|
+
# never verified on inbound mesh HTTP calls. When a secret is configured,
|
|
82
|
+
# require it (constant-time) from NON-loopback callers via X-Mesh-Secret.
|
|
83
|
+
# The local MCP client always calls over loopback and is exempt, so this is
|
|
84
|
+
# zero-change for single-machine use and closes the LAN mesh auth bypass.
|
|
85
|
+
secret = getattr(broker, "_shared_secret", None)
|
|
86
|
+
if secret:
|
|
87
|
+
client_host = request.client.host if request.client else ""
|
|
88
|
+
if client_host not in ("127.0.0.1", "::1", "localhost"):
|
|
89
|
+
import hmac
|
|
90
|
+
presented = request.headers.get("x-mesh-secret", "")
|
|
91
|
+
if not hmac.compare_digest(presented, secret):
|
|
92
|
+
raise HTTPException(401, detail="invalid or missing mesh secret")
|
|
80
93
|
return broker
|
|
81
94
|
|
|
82
95
|
|
|
@@ -33,8 +33,11 @@ router = APIRouter(tags=["internal"])
|
|
|
33
33
|
|
|
34
34
|
_ALLOWED_ORIGIN_PREFIXES = (
|
|
35
35
|
"http://127.0.0.1",
|
|
36
|
+
"https://127.0.0.1",
|
|
36
37
|
"http://localhost",
|
|
38
|
+
"https://localhost",
|
|
37
39
|
"http://[::1]",
|
|
40
|
+
"https://[::1]",
|
|
38
41
|
)
|
|
39
42
|
|
|
40
43
|
|
|
@@ -57,13 +60,22 @@ async def get_token(request: Request) -> JSONResponse:
|
|
|
57
60
|
logger.debug("token: primitives unimportable: %s", exc)
|
|
58
61
|
return JSONResponse({"error": "server_error"}, status_code=500)
|
|
59
62
|
|
|
63
|
+
# v3.6.12 (issue #39): in SLM_REMOTE mode, also serve the token to
|
|
64
|
+
# explicitly-allowlisted LAN clients so a remote-browser dashboard can load
|
|
65
|
+
# the Brain page. Default stays loopback-only — remote_mode helpers return
|
|
66
|
+
# False unless SLM_REMOTE=1 AND the client IP is in SLM_MCP_ALLOWED_HOSTS.
|
|
67
|
+
from superlocalmemory.core.remote_mode import (
|
|
68
|
+
is_lan_client_allowed,
|
|
69
|
+
is_remote_origin_allowed,
|
|
70
|
+
)
|
|
71
|
+
|
|
60
72
|
client_host = request.client.host if request.client else ""
|
|
61
|
-
if not is_loopback(client_host):
|
|
73
|
+
if not is_loopback(client_host) and not is_lan_client_allowed(client_host):
|
|
62
74
|
return JSONResponse({"error": "loopback only"}, status_code=403)
|
|
63
75
|
|
|
64
76
|
headers = {k.lower(): v for k, v in request.headers.items()}
|
|
65
77
|
origin = headers.get("origin", "")
|
|
66
|
-
if not _origin_is_loopback(origin):
|
|
78
|
+
if not _origin_is_loopback(origin) and not is_remote_origin_allowed(origin):
|
|
67
79
|
return JSONResponse(
|
|
68
80
|
{"error": "origin not allowed"}, status_code=403,
|
|
69
81
|
)
|