superlocalmemory 3.6.5 → 3.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +56 -1
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +30 -10
- package/src/superlocalmemory/core/block_hygiene.py +147 -0
- package/src/superlocalmemory/core/config.py +36 -0
- package/src/superlocalmemory/core/consolidation_engine.py +13 -10
- package/src/superlocalmemory/core/engine.py +24 -5
- package/src/superlocalmemory/core/ingest_gate.py +133 -0
- package/src/superlocalmemory/core/injection.py +27 -0
- package/src/superlocalmemory/core/maintenance_scheduler.py +9 -0
- package/src/superlocalmemory/core/recall_pipeline.py +9 -0
- package/src/superlocalmemory/core/recall_worker.py +12 -20
- package/src/superlocalmemory/mcp/server.py +19 -6
- package/src/superlocalmemory/mcp/tools_core.py +2 -0
- package/src/superlocalmemory/retrieval/engine.py +58 -0
- package/src/superlocalmemory/server/recall_serializer.py +225 -0
- package/src/superlocalmemory/server/unified_daemon.py +85 -56
- package/src/superlocalmemory/storage/models.py +3 -0
- package/src/superlocalmemory.egg-info/PKG-INFO +57 -2
- package/src/superlocalmemory.egg-info/SOURCES.txt +3 -0
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 — Ingest Gate (v3.6.6)
|
|
4
|
+
|
|
5
|
+
"""Ingest gate for the remember/store write path (v3.6.6 F-4).
|
|
6
|
+
|
|
7
|
+
Responsibilities:
|
|
8
|
+
1. Hard reject content > 1MB (nobody's memory is a megabyte).
|
|
9
|
+
2. Clamp content > 24000 chars to head 70% + tail 30% + truncation marker.
|
|
10
|
+
Head+tail (not head-only) preserves OPEN ITEMS that session-close facts put
|
|
11
|
+
at the END. Only pathological pastes (167KB JSON, 50KB logs) are touched;
|
|
12
|
+
normal dense memories (6-15K session handoffs) pass through intact.
|
|
13
|
+
Full original preserved in result.full_content for storage in memories table.
|
|
14
|
+
3. Prompt-template firewall: reject content matching _PROMPT_TEMPLATE_PATTERNS
|
|
15
|
+
(extends v3.6.4 remember-write-02 quality gate).
|
|
16
|
+
|
|
17
|
+
Kill-switch: SLM_INGEST_NO_GATE=1 bypasses rules 2 and 3 (but NOT the 1MB hard cap).
|
|
18
|
+
|
|
19
|
+
Usage::
|
|
20
|
+
from superlocalmemory.core.ingest_gate import apply_ingest_gate
|
|
21
|
+
result = apply_ingest_gate(content)
|
|
22
|
+
if result.rejected:
|
|
23
|
+
return {"success": False, "error": result.rejection_reason}
|
|
24
|
+
store(result.fact_content, full_content=result.full_content)
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import os
|
|
30
|
+
from dataclasses import dataclass, field
|
|
31
|
+
|
|
32
|
+
# v3.6.6: clamp only pathological monsters; preserve normal dense memories.
|
|
33
|
+
# 24K chars ≈ 6K tokens — well beyond any legitimate single memory.
|
|
34
|
+
_MAX_VERBATIM_CHARS = 24000
|
|
35
|
+
_HEAD_FRACTION = 0.70 # head 70% + tail 30% so OPEN ITEMS at the end survive
|
|
36
|
+
_MAX_INGEST_BYTES = 1_048_576 # 1MB — hard cap, NOT bypassed by kill-switch
|
|
37
|
+
_TRUNCATION_MARKER = "\n…[content truncated at ingest; full text in source memory]…\n"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class IngestGateResult:
|
|
42
|
+
"""Result of applying the ingest gate to content.
|
|
43
|
+
|
|
44
|
+
fact_content: Content to store in atomic_facts.content (may be truncated).
|
|
45
|
+
full_content: Original unmodified content (for memories table storage).
|
|
46
|
+
rejected: True if content should be rejected outright.
|
|
47
|
+
rejection_reason: Human-readable reason for rejection (when rejected=True).
|
|
48
|
+
truncated: True if fact_content was head-sliced.
|
|
49
|
+
"""
|
|
50
|
+
fact_content: str
|
|
51
|
+
full_content: str
|
|
52
|
+
rejected: bool = False
|
|
53
|
+
rejection_reason: str = ""
|
|
54
|
+
truncated: bool = False
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def apply_ingest_gate(
|
|
58
|
+
content: str,
|
|
59
|
+
max_verbatim_chars: int = _MAX_VERBATIM_CHARS,
|
|
60
|
+
max_ingest_bytes: int = _MAX_INGEST_BYTES,
|
|
61
|
+
) -> IngestGateResult:
|
|
62
|
+
"""Apply the ingest quality gate to content before storing.
|
|
63
|
+
|
|
64
|
+
Returns IngestGateResult. Callers MUST check result.rejected before
|
|
65
|
+
proceeding with storage.
|
|
66
|
+
|
|
67
|
+
Kill-switch SLM_INGEST_NO_GATE=1 bypasses the verbatim size clamp and
|
|
68
|
+
template firewall but NOT the 1MB hard cap (safety boundary).
|
|
69
|
+
"""
|
|
70
|
+
gate_active = os.environ.get("SLM_INGEST_NO_GATE", "0") != "1"
|
|
71
|
+
|
|
72
|
+
# --- Hard cap: 1MB regardless of kill-switch ---
|
|
73
|
+
try:
|
|
74
|
+
byte_len = len(content.encode("utf-8", errors="replace"))
|
|
75
|
+
except Exception:
|
|
76
|
+
byte_len = len(content)
|
|
77
|
+
if byte_len > max_ingest_bytes:
|
|
78
|
+
return IngestGateResult(
|
|
79
|
+
fact_content=content,
|
|
80
|
+
full_content=content,
|
|
81
|
+
rejected=True,
|
|
82
|
+
rejection_reason=(
|
|
83
|
+
f"Content size {byte_len} bytes exceeds maximum "
|
|
84
|
+
f"{max_ingest_bytes} bytes (1MB). "
|
|
85
|
+
"Nobody's memory is a megabyte."
|
|
86
|
+
),
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
# --- Gate bypassed ---
|
|
90
|
+
if not gate_active:
|
|
91
|
+
return IngestGateResult(
|
|
92
|
+
fact_content=content,
|
|
93
|
+
full_content=content,
|
|
94
|
+
rejected=False,
|
|
95
|
+
truncated=False,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
# --- Prompt-template firewall ---
|
|
99
|
+
# Extends the v3.6.4 remember-write-02 quality gate.
|
|
100
|
+
try:
|
|
101
|
+
from superlocalmemory.core.injection import is_prompt_template
|
|
102
|
+
if is_prompt_template(content):
|
|
103
|
+
return IngestGateResult(
|
|
104
|
+
fact_content=content,
|
|
105
|
+
full_content=content,
|
|
106
|
+
rejected=True,
|
|
107
|
+
rejection_reason=(
|
|
108
|
+
"Content matches internal prompt-template patterns "
|
|
109
|
+
"(low-quality gate). Prompt machinery must not be stored as memory."
|
|
110
|
+
),
|
|
111
|
+
)
|
|
112
|
+
except Exception:
|
|
113
|
+
pass # Defensive: gate failure must never block a store
|
|
114
|
+
|
|
115
|
+
# --- Verbatim size clamp (head 70% + tail 30%) ---
|
|
116
|
+
if len(content) > max_verbatim_chars:
|
|
117
|
+
budget = max_verbatim_chars - len(_TRUNCATION_MARKER)
|
|
118
|
+
head_len = int(budget * _HEAD_FRACTION)
|
|
119
|
+
tail_len = budget - head_len
|
|
120
|
+
fact_content = content[:head_len] + _TRUNCATION_MARKER + content[-tail_len:]
|
|
121
|
+
return IngestGateResult(
|
|
122
|
+
fact_content=fact_content,
|
|
123
|
+
full_content=content,
|
|
124
|
+
rejected=False,
|
|
125
|
+
truncated=True,
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
return IngestGateResult(
|
|
129
|
+
fact_content=content,
|
|
130
|
+
full_content=content,
|
|
131
|
+
rejected=False,
|
|
132
|
+
truncated=False,
|
|
133
|
+
)
|
|
@@ -35,6 +35,33 @@ _LOW_QUALITY_PATTERNS = (
|
|
|
35
35
|
re.compile(r"the first line must be exactly", re.IGNORECASE),
|
|
36
36
|
)
|
|
37
37
|
|
|
38
|
+
# v3.6.6: Prompt-template firewall patterns (F-3, F-4).
|
|
39
|
+
# These patterns identify content that is internal LLM prompt machinery,
|
|
40
|
+
# not genuine user memories. Used at ingest (F-4) and recall serialization (F-3).
|
|
41
|
+
# Stdlib-only (re module) so hooks import chain stays light.
|
|
42
|
+
_PROMPT_TEMPLATE_PATTERNS = (
|
|
43
|
+
re.compile(r"you are summarizing a claude code session", re.IGNORECASE),
|
|
44
|
+
re.compile(r"you are a memory consolidation agent", re.IGNORECASE),
|
|
45
|
+
re.compile(r"apply maximum non-destructive compression", re.IGNORECASE),
|
|
46
|
+
re.compile(r"<task-notification\b", re.IGNORECASE),
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def is_prompt_template(content: str) -> bool:
|
|
51
|
+
"""True if *content* matches internal prompt-template patterns.
|
|
52
|
+
|
|
53
|
+
Used by:
|
|
54
|
+
- F-3: drop source_content that is prompt machinery
|
|
55
|
+
- F-4: reject template content at ingest (extends v3.6.4 quality gate)
|
|
56
|
+
|
|
57
|
+
Separate from is_low_quality() so the firewall can be applied without
|
|
58
|
+
the full quality-check heuristics (e.g. at raw ingest before any
|
|
59
|
+
other processing).
|
|
60
|
+
"""
|
|
61
|
+
if not content:
|
|
62
|
+
return False
|
|
63
|
+
return any(pat.search(content) for pat in _PROMPT_TEMPLATE_PATTERNS)
|
|
64
|
+
|
|
38
65
|
# Leading category tag like "[active_decisions] " / "[learned_preferences] ".
|
|
39
66
|
_CATEGORY_TAG_RE = re.compile(r"^\s*\[[a-z0-9_]+\]\s*", re.IGNORECASE)
|
|
40
67
|
|
|
@@ -124,6 +124,15 @@ class MaintenanceScheduler:
|
|
|
124
124
|
except Exception as exc:
|
|
125
125
|
logger.debug("Pending cleanup skipped: %s", exc)
|
|
126
126
|
|
|
127
|
+
# v3.6.6 F-5: Daily core-block recompile with hygiene (dedup + char cap).
|
|
128
|
+
# Ensures blocks stay clean even when purge or new facts arrive between
|
|
129
|
+
# session-init recompiles.
|
|
130
|
+
try:
|
|
131
|
+
from superlocalmemory.core.block_hygiene import _recompile_core_blocks
|
|
132
|
+
_recompile_core_blocks(self._db, self._config, self._profile_id)
|
|
133
|
+
except Exception as exc:
|
|
134
|
+
logger.debug("Core-block recompile skipped: %s", exc)
|
|
135
|
+
|
|
127
136
|
self._schedule_next()
|
|
128
137
|
|
|
129
138
|
def _sync_cloud_destinations(self, manager: object) -> None:
|
|
@@ -324,6 +324,8 @@ def apply_adaptive_ranking(
|
|
|
324
324
|
channel_weights=response.channel_weights,
|
|
325
325
|
total_candidates=response.total_candidates,
|
|
326
326
|
retrieval_time_ms=response.retrieval_time_ms,
|
|
327
|
+
# v3.6.6: preserve evidence-floor signal across reranking rebuilds.
|
|
328
|
+
no_confident_match=(len(new_results) == 0) and response.no_confident_match,
|
|
327
329
|
)
|
|
328
330
|
|
|
329
331
|
|
|
@@ -426,6 +428,8 @@ def apply_v2_adaptive_ranking(
|
|
|
426
428
|
channel_weights=response.channel_weights,
|
|
427
429
|
total_candidates=response.total_candidates,
|
|
428
430
|
retrieval_time_ms=response.retrieval_time_ms,
|
|
431
|
+
# v3.6.6: preserve evidence-floor signal across reranking rebuilds.
|
|
432
|
+
no_confident_match=(len(new_results) == 0) and response.no_confident_match,
|
|
429
433
|
)
|
|
430
434
|
except Exception as exc: # pragma: no cover — defensive
|
|
431
435
|
logger.debug("apply_v2_adaptive_ranking skipped: %s", exc)
|
|
@@ -554,6 +558,8 @@ def apply_v2_bandit_ensemble(
|
|
|
554
558
|
channel_weights=response.channel_weights,
|
|
555
559
|
total_candidates=response.total_candidates,
|
|
556
560
|
retrieval_time_ms=response.retrieval_time_ms,
|
|
561
|
+
# v3.6.6: preserve evidence-floor signal across ensemble rebuilds.
|
|
562
|
+
no_confident_match=(len(final_results) == 0) and response.no_confident_match,
|
|
557
563
|
)
|
|
558
564
|
except Exception as exc: # pragma: no cover — defensive top-level
|
|
559
565
|
logger.debug("apply_v2_bandit_ensemble skipped: %s", exc)
|
|
@@ -675,6 +681,9 @@ def run_recall(
|
|
|
675
681
|
channel_weights=response.channel_weights,
|
|
676
682
|
total_candidates=response.total_candidates + len(enhanced_facts),
|
|
677
683
|
retrieval_time_ms=response.retrieval_time_ms,
|
|
684
|
+
# v3.6.6: agentic round-2 may add facts; recompute flag.
|
|
685
|
+
no_confident_match=(len(enhanced_results[:limit]) == 0)
|
|
686
|
+
and response.no_confident_match,
|
|
678
687
|
)
|
|
679
688
|
except Exception as exc:
|
|
680
689
|
logger.debug("Agentic sufficiency skipped: %s", exc)
|
|
@@ -71,26 +71,17 @@ def _handle_recall(
|
|
|
71
71
|
memory_ids = list({r.fact.memory_id for r in response.results[:limit] if r.fact.memory_id})
|
|
72
72
|
memory_map = engine._db.get_memory_content_batch(memory_ids) if memory_ids else {}
|
|
73
73
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
"trust_score": round(r.trust_score, 4),
|
|
86
|
-
"channel_scores": {
|
|
87
|
-
k: round(v, 4) for k, v in (r.channel_scores or {}).items()
|
|
88
|
-
},
|
|
89
|
-
"fact_type": fact_type.value if fact_type and hasattr(fact_type, "value") else "",
|
|
90
|
-
"lifecycle": lifecycle.value if lifecycle and hasattr(lifecycle, "value") else "",
|
|
91
|
-
"access_count": getattr(r.fact, "access_count", 0),
|
|
92
|
-
"evidence_chain": list(getattr(r, "evidence_chain", []) or []),
|
|
93
|
-
})
|
|
74
|
+
# v3.6.6: same shared chokepoint as the daemon HTTP route + CLI fallback,
|
|
75
|
+
# so the MCP WorkerPool subprocess path returns identical budgeted output.
|
|
76
|
+
from superlocalmemory.server.recall_serializer import serialize_recall_response
|
|
77
|
+
_rc = getattr(engine._config, "retrieval", None)
|
|
78
|
+
results, no_confident_match = serialize_recall_response(
|
|
79
|
+
response,
|
|
80
|
+
limit=limit,
|
|
81
|
+
memory_map=memory_map,
|
|
82
|
+
per_fact_max=getattr(_rc, "recall_per_fact_max_chars", 2400),
|
|
83
|
+
total_max=getattr(_rc, "recall_total_max_chars", 12000),
|
|
84
|
+
)
|
|
94
85
|
return {
|
|
95
86
|
"ok": True,
|
|
96
87
|
"query": query,
|
|
@@ -102,6 +93,7 @@ def _handle_recall(
|
|
|
102
93
|
},
|
|
103
94
|
"total_candidates": getattr(response, "total_candidates", 0),
|
|
104
95
|
"results": results,
|
|
96
|
+
"no_confident_match": no_confident_match,
|
|
105
97
|
}
|
|
106
98
|
|
|
107
99
|
|
|
@@ -237,8 +237,19 @@ def _eager_warmup() -> None:
|
|
|
237
237
|
_logger.warning("Mesh auto-register failed: %s", exc)
|
|
238
238
|
|
|
239
239
|
import threading
|
|
240
|
-
|
|
241
|
-
|
|
240
|
+
|
|
241
|
+
# v3.6.7: Suppress standalone-process behaviours when the MCP server is
|
|
242
|
+
# imported inside the daemon (SLM_MCP_EMBEDDED=1). Three threads are safe
|
|
243
|
+
# to run in a dedicated `slm mcp` subprocess but harmful inside the daemon:
|
|
244
|
+
# mcp-warmup — creates a LIGHT engine duplicate; daemon has a FULL one.
|
|
245
|
+
# parent-watchdog — calls os._exit(0) if its parent IDE quits, which would
|
|
246
|
+
# kill the daemon along with it.
|
|
247
|
+
# stdin-eof-monitor — monitors stdin pipe; meaningless inside the daemon.
|
|
248
|
+
_embedded_in_daemon = _os.environ.get("SLM_MCP_EMBEDDED") == "1"
|
|
249
|
+
|
|
250
|
+
if not _embedded_in_daemon:
|
|
251
|
+
_warmup_thread = threading.Thread(target=_eager_warmup, daemon=True, name="mcp-warmup")
|
|
252
|
+
_warmup_thread.start()
|
|
242
253
|
|
|
243
254
|
|
|
244
255
|
# V3.4.57: Parent watchdog — self-terminate when the IDE/Claude session dies.
|
|
@@ -274,8 +285,9 @@ def _parent_watchdog() -> None:
|
|
|
274
285
|
pass # Transient errors — keep watching
|
|
275
286
|
|
|
276
287
|
|
|
277
|
-
|
|
278
|
-
_watchdog_thread.
|
|
288
|
+
if not _embedded_in_daemon:
|
|
289
|
+
_watchdog_thread = threading.Thread(target=_parent_watchdog, daemon=True, name="parent-watchdog")
|
|
290
|
+
_watchdog_thread.start()
|
|
279
291
|
|
|
280
292
|
|
|
281
293
|
# V3.5.9: Stdin EOF monitor — complements the parent watchdog for the case where
|
|
@@ -337,8 +349,9 @@ def _stdin_eof_monitor() -> None:
|
|
|
337
349
|
_mlog.debug("stdin EOF monitor error: %s — watchdog will cover", exc)
|
|
338
350
|
|
|
339
351
|
|
|
340
|
-
|
|
341
|
-
_stdin_monitor_thread.
|
|
352
|
+
if not _embedded_in_daemon:
|
|
353
|
+
_stdin_monitor_thread = threading.Thread(target=_stdin_eof_monitor, daemon=True, name="stdin-eof-monitor")
|
|
354
|
+
_stdin_monitor_thread.start()
|
|
342
355
|
|
|
343
356
|
|
|
344
357
|
if __name__ == "__main__":
|
|
@@ -236,6 +236,8 @@ def register_core_tools(server, get_engine: Callable) -> None:
|
|
|
236
236
|
"query_type": result.get("query_type", "unknown"),
|
|
237
237
|
"channel_weights": result.get("channel_weights", {}),
|
|
238
238
|
"retrieval_time_ms": result.get("retrieval_time_ms", 0),
|
|
239
|
+
# v3.6.6: surface evidence-floor signal to MCP clients.
|
|
240
|
+
"no_confident_match": result.get("no_confident_match", False),
|
|
239
241
|
}
|
|
240
242
|
return {"success": False, "error": result.get("error", "Recall failed")}
|
|
241
243
|
except Exception as exc:
|
|
@@ -295,15 +295,73 @@ class RetrievalEngine:
|
|
|
295
295
|
if len(final_top) > len(top[:effective_limit]):
|
|
296
296
|
facts = self._load_facts(final_top, profile_id)
|
|
297
297
|
|
|
298
|
+
# v3.6.6: Evidence floor — gate on per-channel scores (NOT fused/RRF score).
|
|
299
|
+
# Nonsense queries fuse at 0.75-0.78 because RRF is rank-derived and
|
|
300
|
+
# uncalibrated. The discriminator is EARNED CHANNEL EVIDENCE:
|
|
301
|
+
# semantic >= min_semantic_evidence (0.60) OR bm25 > 0
|
|
302
|
+
# OR entity_graph > 0 OR temporal > 0 OR fact is pinned.
|
|
303
|
+
# spreading_activation and hopfield do NOT count — they are associative
|
|
304
|
+
# amplifiers that fabricated the nonsense results in calibration tests.
|
|
305
|
+
# Kill-switch: SLM_RECALL_NO_FLOOR=1 bypasses the floor.
|
|
306
|
+
import os as _os_floor
|
|
307
|
+
floor_enabled = (
|
|
308
|
+
getattr(self._config, "evidence_floor_enabled", True)
|
|
309
|
+
and _os_floor.environ.get("SLM_RECALL_NO_FLOOR", "0") != "1"
|
|
310
|
+
)
|
|
311
|
+
if floor_enabled:
|
|
312
|
+
min_sem = getattr(self._config, "min_semantic_evidence", 0.60)
|
|
313
|
+
final_top = self._apply_evidence_floor(final_top, facts, min_sem)
|
|
314
|
+
# Trim facts dict to match filtered final_top
|
|
315
|
+
filtered_ids = {fr.fact_id for fr in final_top}
|
|
316
|
+
facts = {fid: f for fid, f in facts.items() if fid in filtered_ids}
|
|
317
|
+
|
|
298
318
|
# 6. Build response
|
|
299
319
|
results = self._build_results(final_top, facts, strat)
|
|
300
320
|
ms = (time.monotonic() - t0) * 1000.0
|
|
321
|
+
no_match = floor_enabled and len(results) == 0
|
|
301
322
|
return RecallResponse(
|
|
302
323
|
query=query, mode=mode, results=results,
|
|
303
324
|
query_type=strat.query_type, channel_weights=strat.weights,
|
|
304
325
|
total_candidates=total, retrieval_time_ms=ms,
|
|
326
|
+
no_confident_match=no_match,
|
|
305
327
|
)
|
|
306
328
|
|
|
329
|
+
# -- Evidence floor (v3.6.6) -------------------------------------------
|
|
330
|
+
|
|
331
|
+
@staticmethod
|
|
332
|
+
def _apply_evidence_floor(
|
|
333
|
+
final_top: list[FusionResult],
|
|
334
|
+
facts: dict[str, AtomicFact],
|
|
335
|
+
min_semantic: float,
|
|
336
|
+
) -> list[FusionResult]:
|
|
337
|
+
"""Filter results that earned no channel evidence.
|
|
338
|
+
|
|
339
|
+
Keep a result only if it earned:
|
|
340
|
+
- semantic cosine >= min_semantic (default 0.60), OR
|
|
341
|
+
- bm25 > 0, OR entity_graph > 0, OR temporal > 0, OR
|
|
342
|
+
- the underlying fact is pinned.
|
|
343
|
+
|
|
344
|
+
spreading_activation and hopfield do NOT count as primary evidence.
|
|
345
|
+
Empty result after filtering is a success (no_confident_match=True).
|
|
346
|
+
"""
|
|
347
|
+
kept: list[FusionResult] = []
|
|
348
|
+
for fr in final_top:
|
|
349
|
+
cs = fr.channel_scores or {}
|
|
350
|
+
# Primary channel evidence check
|
|
351
|
+
if (
|
|
352
|
+
cs.get("semantic", 0.0) >= min_semantic
|
|
353
|
+
or cs.get("bm25", 0.0) > 0.0
|
|
354
|
+
or cs.get("entity_graph", 0.0) > 0.0
|
|
355
|
+
or cs.get("temporal", 0.0) > 0.0
|
|
356
|
+
):
|
|
357
|
+
kept.append(fr)
|
|
358
|
+
continue
|
|
359
|
+
# Pinned fact bypass — always pass regardless of channel scores
|
|
360
|
+
fact = facts.get(fr.fact_id)
|
|
361
|
+
if fact is not None and getattr(fact, "pinned", False):
|
|
362
|
+
kept.append(fr)
|
|
363
|
+
return kept
|
|
364
|
+
|
|
307
365
|
# -- Cross-channel intersection boost -----------------------------------
|
|
308
366
|
|
|
309
367
|
@staticmethod
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 — Recall Serializer (v3.6.6)
|
|
4
|
+
|
|
5
|
+
"""Recall output budget and source_content discipline helpers (v3.6.6).
|
|
6
|
+
|
|
7
|
+
F-2: Per-fact content clamp + total budget stubs.
|
|
8
|
+
F-3: source_content preview + template firewall.
|
|
9
|
+
|
|
10
|
+
THE single shared serialization chokepoint. Every surface that turns a
|
|
11
|
+
RecallResponse into transport dicts goes through ``serialize_recall_response``
|
|
12
|
+
so MCP, CLI, the daemon HTTP route, the in-process queue adapter, and the
|
|
13
|
+
WorkerPool fallback all return byte-for-byte identical output (parity across
|
|
14
|
+
surfaces AND modes A/B). The evidence floor lives upstream in
|
|
15
|
+
RetrievalEngine.recall (also shared); this layer owns presentation only.
|
|
16
|
+
|
|
17
|
+
Pure functions — no side effects, no DB access. Stdlib-only at import
|
|
18
|
+
(hooks import chain must stay light).
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import re
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# ---------------------------------------------------------------------------
|
|
28
|
+
# F-2: Per-fact content clamp
|
|
29
|
+
# ---------------------------------------------------------------------------
|
|
30
|
+
|
|
31
|
+
def clamp_fact_content(
|
|
32
|
+
content: str,
|
|
33
|
+
max_chars: int = 2400,
|
|
34
|
+
) -> tuple[str, bool]:
|
|
35
|
+
"""Clamp a single fact's content to max_chars.
|
|
36
|
+
|
|
37
|
+
Strategy: head 70% + "\\n…[truncated N chars]…\\n" + tail 30%.
|
|
38
|
+
The tail is kept because session-close facts put OPEN ITEMS at the end.
|
|
39
|
+
|
|
40
|
+
Returns:
|
|
41
|
+
(clamped_content, was_truncated)
|
|
42
|
+
"""
|
|
43
|
+
if not content or len(content) <= max_chars:
|
|
44
|
+
return content, False
|
|
45
|
+
|
|
46
|
+
head_len = int(max_chars * 0.70)
|
|
47
|
+
tail_len = max_chars - head_len
|
|
48
|
+
dropped = len(content) - max_chars
|
|
49
|
+
marker = f"\n…[truncated {dropped} chars]…\n"
|
|
50
|
+
|
|
51
|
+
result = content[:head_len] + marker + content[-tail_len:]
|
|
52
|
+
return result, True
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def apply_recall_budget(
|
|
56
|
+
results: list[dict],
|
|
57
|
+
per_fact_max: int = 2400,
|
|
58
|
+
total_max: int = 12000,
|
|
59
|
+
full: bool = False,
|
|
60
|
+
) -> list[dict]:
|
|
61
|
+
"""Apply per-fact clamp and total budget to a list of result dicts.
|
|
62
|
+
|
|
63
|
+
Args:
|
|
64
|
+
results: List of result dicts (must have at minimum 'fact_id',
|
|
65
|
+
'score', 'content' keys).
|
|
66
|
+
per_fact_max: Maximum chars for a single fact's content.
|
|
67
|
+
total_max: Maximum total content chars before remaining results
|
|
68
|
+
become stubs.
|
|
69
|
+
full: If True, bypasses all clamping (escape hatch for tools/CLI
|
|
70
|
+
that need full content — additive backward-compat param).
|
|
71
|
+
|
|
72
|
+
Returns:
|
|
73
|
+
New list of result dicts with potentially clamped/stubbed content.
|
|
74
|
+
Mutates nothing — returns new dicts.
|
|
75
|
+
"""
|
|
76
|
+
if not results:
|
|
77
|
+
return []
|
|
78
|
+
|
|
79
|
+
if full:
|
|
80
|
+
# full=True: return everything as-is, no clamping, no stubs
|
|
81
|
+
return [dict(r) for r in results]
|
|
82
|
+
|
|
83
|
+
out: list[dict] = []
|
|
84
|
+
cumulative_chars = 0
|
|
85
|
+
|
|
86
|
+
for r in results:
|
|
87
|
+
content = r.get("content", "") or ""
|
|
88
|
+
|
|
89
|
+
# Check if we're already over total budget
|
|
90
|
+
if cumulative_chars >= total_max:
|
|
91
|
+
# Emit stub: fact_id, score, first 120 chars + "…"
|
|
92
|
+
stub_content = content[:120] + ("…" if len(content) > 120 else "")
|
|
93
|
+
stub = {k: v for k, v in r.items() if k not in ("content",)}
|
|
94
|
+
stub["content"] = stub_content
|
|
95
|
+
stub["stub"] = True
|
|
96
|
+
out.append(stub)
|
|
97
|
+
continue
|
|
98
|
+
|
|
99
|
+
# Per-fact clamp
|
|
100
|
+
clamped, was_truncated = clamp_fact_content(content, max_chars=per_fact_max)
|
|
101
|
+
new_r = dict(r)
|
|
102
|
+
new_r["content"] = clamped
|
|
103
|
+
if was_truncated:
|
|
104
|
+
new_r["truncated"] = True
|
|
105
|
+
|
|
106
|
+
cumulative_chars += len(clamped)
|
|
107
|
+
out.append(new_r)
|
|
108
|
+
|
|
109
|
+
return out
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# ---------------------------------------------------------------------------
|
|
113
|
+
# F-3: source_content discipline
|
|
114
|
+
# ---------------------------------------------------------------------------
|
|
115
|
+
|
|
116
|
+
def apply_source_content_discipline(
|
|
117
|
+
result: dict,
|
|
118
|
+
include_source: bool = False,
|
|
119
|
+
) -> dict:
|
|
120
|
+
"""Apply source_content discipline to a single result dict.
|
|
121
|
+
|
|
122
|
+
Default behavior:
|
|
123
|
+
- Trim source_content to ≤ 280 chars
|
|
124
|
+
- Drop entirely if it matches prompt-template patterns
|
|
125
|
+
|
|
126
|
+
include_source=True:
|
|
127
|
+
- Returns full source_content (unless it's a template, always dropped)
|
|
128
|
+
|
|
129
|
+
Returns a new dict — never mutates input.
|
|
130
|
+
"""
|
|
131
|
+
from superlocalmemory.core.injection import is_prompt_template
|
|
132
|
+
|
|
133
|
+
if "source_content" not in result:
|
|
134
|
+
return dict(result)
|
|
135
|
+
|
|
136
|
+
src = result.get("source_content") or ""
|
|
137
|
+
|
|
138
|
+
# Template firewall: drop regardless of include_source
|
|
139
|
+
if src and is_prompt_template(src):
|
|
140
|
+
new_r = dict(result)
|
|
141
|
+
new_r["source_content"] = ""
|
|
142
|
+
return new_r
|
|
143
|
+
|
|
144
|
+
# Empty source: return unchanged
|
|
145
|
+
if not src:
|
|
146
|
+
return dict(result)
|
|
147
|
+
|
|
148
|
+
if include_source:
|
|
149
|
+
return dict(result)
|
|
150
|
+
|
|
151
|
+
# Default: preview ≤ 280 chars
|
|
152
|
+
new_r = dict(result)
|
|
153
|
+
new_r["source_content"] = src[:280]
|
|
154
|
+
return new_r
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
# ---------------------------------------------------------------------------
|
|
158
|
+
# THE shared chokepoint: RecallResponse -> transport dicts (all surfaces)
|
|
159
|
+
# ---------------------------------------------------------------------------
|
|
160
|
+
|
|
161
|
+
def serialize_recall_response(
|
|
162
|
+
response: Any,
|
|
163
|
+
*,
|
|
164
|
+
limit: int = 10,
|
|
165
|
+
memory_map: dict[str, str] | None = None,
|
|
166
|
+
per_fact_max: int = 2400,
|
|
167
|
+
total_max: int = 12000,
|
|
168
|
+
full: bool = False,
|
|
169
|
+
include_source: bool = False,
|
|
170
|
+
) -> tuple[list[dict], bool]:
|
|
171
|
+
"""Convert a RecallResponse into budgeted, source-disciplined dicts.
|
|
172
|
+
|
|
173
|
+
This is the ONE function every recall surface calls (daemon HTTP route,
|
|
174
|
+
in-process queue adapter, CLI direct-fallback, WorkerPool). Guarantees
|
|
175
|
+
identical output regardless of surface or mode.
|
|
176
|
+
|
|
177
|
+
Args:
|
|
178
|
+
response: A RecallResponse (engine result objects in .results).
|
|
179
|
+
limit: Max results to serialize.
|
|
180
|
+
memory_map: fact.memory_id -> source memory content (optional).
|
|
181
|
+
per_fact_max: Per-fact content char cap (config-driven).
|
|
182
|
+
total_max: Total content char budget before stubs (config-driven).
|
|
183
|
+
full: Bypass clamping/stubs (additive escape hatch).
|
|
184
|
+
include_source: Return full source_content (else ≤280-char preview).
|
|
185
|
+
|
|
186
|
+
Returns:
|
|
187
|
+
(results, no_confident_match) — results is a list of dicts; the bool
|
|
188
|
+
is the evidence-floor signal lifted from the response (additive).
|
|
189
|
+
"""
|
|
190
|
+
memory_map = memory_map or {}
|
|
191
|
+
raw: list[dict] = []
|
|
192
|
+
for r in (response.results or [])[:limit]:
|
|
193
|
+
fact = r.fact
|
|
194
|
+
fact_type = getattr(fact, "fact_type", None)
|
|
195
|
+
lifecycle = getattr(fact, "lifecycle", None)
|
|
196
|
+
raw.append({
|
|
197
|
+
"fact_id": fact.fact_id,
|
|
198
|
+
"memory_id": fact.memory_id,
|
|
199
|
+
"content": fact.content or "",
|
|
200
|
+
"source_content": memory_map.get(fact.memory_id, "") or "",
|
|
201
|
+
"score": round(r.score, 4),
|
|
202
|
+
"confidence": round(getattr(r, "confidence", 0.0), 4),
|
|
203
|
+
"trust_score": round(getattr(r, "trust_score", 0.0), 4),
|
|
204
|
+
"channel_scores": {
|
|
205
|
+
k: round(v, 4) for k, v in (getattr(r, "channel_scores", None) or {}).items()
|
|
206
|
+
},
|
|
207
|
+
"fact_type": fact_type.value
|
|
208
|
+
if fact_type is not None and hasattr(fact_type, "value")
|
|
209
|
+
else (getattr(fact, "fact_type", "") or ""),
|
|
210
|
+
"lifecycle": lifecycle.value
|
|
211
|
+
if lifecycle is not None and hasattr(lifecycle, "value")
|
|
212
|
+
else (lifecycle or ""),
|
|
213
|
+
"access_count": getattr(fact, "access_count", 0),
|
|
214
|
+
"created_at": getattr(fact, "created_at", "") or "",
|
|
215
|
+
"evidence_chain": list(getattr(r, "evidence_chain", []) or []),
|
|
216
|
+
})
|
|
217
|
+
|
|
218
|
+
# F-3 source discipline, then F-2 budget — order matters (discipline first
|
|
219
|
+
# so the template firewall runs before any preview slicing).
|
|
220
|
+
disciplined = [apply_source_content_discipline(d, include_source=include_source) for d in raw]
|
|
221
|
+
budgeted = apply_recall_budget(
|
|
222
|
+
disciplined, per_fact_max=per_fact_max, total_max=total_max, full=full,
|
|
223
|
+
)
|
|
224
|
+
no_confident_match = bool(getattr(response, "no_confident_match", False))
|
|
225
|
+
return budgeted, no_confident_match
|