superlocalmemory 4.0.10 → 4.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/.claude-plugin/marketplace.json +12 -2
  2. package/CHANGELOG.md +244 -0
  3. package/README.md +40 -75
  4. package/package.json +6 -3
  5. package/plugin/.claude-plugin/plugin.json +2 -2
  6. package/plugin/CLAUDE.md +3 -3
  7. package/plugin/agents/slm-governance-advisor.md +1 -1
  8. package/plugin/agents/slm-loop-runner.md +4 -4
  9. package/plugin/agents/slm-memory-advisor.md +1 -1
  10. package/plugin/agents/slm-optimize-advisor.md +1 -1
  11. package/plugin/requirements.txt +1 -1
  12. package/plugin/skills/slm-cache/SKILL.md +1 -1
  13. package/plugin/skills/slm-compress/SKILL.md +1 -1
  14. package/plugin/skills/slm-governance/SKILL.md +1 -1
  15. package/plugin/skills/slm-graph/SKILL.md +1 -1
  16. package/plugin/skills/slm-loop/SKILL.md +2 -2
  17. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  18. package/plugin/skills/slm-profile/SKILL.md +5 -5
  19. package/plugin/skills/slm-recall/SKILL.md +102 -15
  20. package/plugin/skills/slm-remember/SKILL.md +35 -3
  21. package/plugin/skills/slm-scope/SKILL.md +1 -1
  22. package/plugin/skills/slm-session/SKILL.md +29 -3
  23. package/plugin/skills/slm-status/SKILL.md +1 -1
  24. package/plugin-src/rules/AGENTS.md +16 -8
  25. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  26. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-governance/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  29. package/plugin-src/skills/slm-loop/SKILL.md +2 -2
  30. package/plugin-src/skills/slm-mesh/SKILL.md +1 -1
  31. package/plugin-src/skills/slm-profile/SKILL.md +5 -5
  32. package/plugin-src/skills/slm-recall/SKILL.md +102 -15
  33. package/plugin-src/skills/slm-remember/SKILL.md +35 -3
  34. package/plugin-src/skills/slm-scope/SKILL.md +1 -1
  35. package/plugin-src/skills/slm-session/SKILL.md +29 -3
  36. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  37. package/pyproject.toml +1 -1
  38. package/src/superlocalmemory/__init__.py +1 -1
  39. package/src/superlocalmemory/cli/commands.py +357 -18
  40. package/src/superlocalmemory/cli/daemon.py +30 -0
  41. package/src/superlocalmemory/cli/db_migrate.py +71 -1
  42. package/src/superlocalmemory/cli/gdpr_cmd.py +15 -2
  43. package/src/superlocalmemory/cli/main.py +24 -2
  44. package/src/superlocalmemory/code_graph/database.py +44 -0
  45. package/src/superlocalmemory/compliance/gdpr.py +449 -39
  46. package/src/superlocalmemory/core/admission.py +231 -11
  47. package/src/superlocalmemory/core/backend_orchestrator.py +190 -84
  48. package/src/superlocalmemory/core/config.py +90 -11
  49. package/src/superlocalmemory/core/consolidation_engine.py +34 -0
  50. package/src/superlocalmemory/core/engine.py +140 -11
  51. package/src/superlocalmemory/core/graph_analyzer.py +76 -112
  52. package/src/superlocalmemory/core/graph_metrics.py +597 -0
  53. package/src/superlocalmemory/core/graph_pruner.py +121 -0
  54. package/src/superlocalmemory/core/maintenance_scheduler.py +205 -0
  55. package/src/superlocalmemory/core/mode_capability.py +111 -0
  56. package/src/superlocalmemory/core/ollama_validator.py +315 -0
  57. package/src/superlocalmemory/core/projection_drain.py +380 -0
  58. package/src/superlocalmemory/core/recall_pipeline.py +390 -3
  59. package/src/superlocalmemory/core/recall_worker.py +6 -3
  60. package/src/superlocalmemory/core/scale_autopromote.py +196 -0
  61. package/src/superlocalmemory/core/scale_engine.py +16 -2
  62. package/src/superlocalmemory/core/score_contract.py +21 -1
  63. package/src/superlocalmemory/core/session_identity.py +85 -0
  64. package/src/superlocalmemory/core/status_contract.py +108 -0
  65. package/src/superlocalmemory/core/worker_pool.py +4 -4
  66. package/src/superlocalmemory/core/working_memory.py +288 -0
  67. package/src/superlocalmemory/encoding/cognitive_consolidator.py +36 -6
  68. package/src/superlocalmemory/encoding/context_generator.py +1 -1
  69. package/src/superlocalmemory/encoding/entity_resolver.py +38 -0
  70. package/src/superlocalmemory/encoding/fact_extractor.py +18 -14
  71. package/src/superlocalmemory/encoding/prospective_markers.py +262 -0
  72. package/src/superlocalmemory/encoding/type_router.py +12 -12
  73. package/src/superlocalmemory/evolution/mutation_generator.py +30 -4
  74. package/src/superlocalmemory/graph/cozo_adjacency.py +122 -0
  75. package/src/superlocalmemory/graph/cozo_backend.py +103 -138
  76. package/src/superlocalmemory/hooks/portable_kit.py +10 -2
  77. package/src/superlocalmemory/learning/bandit.py +43 -0
  78. package/src/superlocalmemory/learning/consolidation_worker.py +54 -0
  79. package/src/superlocalmemory/learning/database.py +60 -3
  80. package/src/superlocalmemory/learning/entity_compiler.py +21 -58
  81. package/src/superlocalmemory/learning/feedback.py +3 -1
  82. package/src/superlocalmemory/learning/outcomes.py +47 -16
  83. package/src/superlocalmemory/learning/pattern_miner.py +28 -3
  84. package/src/superlocalmemory/learning/pattern_miner_constants.py +43 -0
  85. package/src/superlocalmemory/learning/pcos.py +291 -0
  86. package/src/superlocalmemory/learning/reward_from_outcomes.py +365 -0
  87. package/src/superlocalmemory/learning/reward_proxy.py +100 -10
  88. package/src/superlocalmemory/learning/signal_kinds.py +79 -0
  89. package/src/superlocalmemory/mcp/profiles.py +14 -2
  90. package/src/superlocalmemory/mcp/tools_active.py +2 -1
  91. package/src/superlocalmemory/mcp/tools_core.py +31 -3
  92. package/src/superlocalmemory/mcp/tools_v28.py +20 -1
  93. package/src/superlocalmemory/parameterization/pattern_extractor.py +14 -1
  94. package/src/superlocalmemory/parameterization/soft_prompt_generator.py +98 -0
  95. package/src/superlocalmemory/retrieval/bm25_channel.py +64 -3
  96. package/src/superlocalmemory/retrieval/channel_status.py +117 -0
  97. package/src/superlocalmemory/retrieval/engine.py +106 -11
  98. package/src/superlocalmemory/retrieval/entity_channel.py +210 -256
  99. package/src/superlocalmemory/retrieval/graph_adjacency.py +219 -0
  100. package/src/superlocalmemory/retrieval/scope_policy.py +20 -0
  101. package/src/superlocalmemory/retrieval/semantic_channel.py +47 -5
  102. package/src/superlocalmemory/retrieval/spreading.py +288 -0
  103. package/src/superlocalmemory/server/api.py +24 -5
  104. package/src/superlocalmemory/server/bandit_loops.py +17 -1
  105. package/src/superlocalmemory/server/rbac_enforce.py +26 -6
  106. package/src/superlocalmemory/server/recall_health.py +87 -10
  107. package/src/superlocalmemory/server/recall_serializer.py +9 -0
  108. package/src/superlocalmemory/server/routes/behavioral.py +75 -10
  109. package/src/superlocalmemory/server/routes/compliance.py +98 -18
  110. package/src/superlocalmemory/server/routes/config_api.py +186 -4
  111. package/src/superlocalmemory/server/routes/evolution.py +178 -0
  112. package/src/superlocalmemory/server/routes/ingest.py +8 -0
  113. package/src/superlocalmemory/server/routes/learning_telemetry.py +2 -1
  114. package/src/superlocalmemory/server/routes/memories.py +49 -7
  115. package/src/superlocalmemory/server/routes/timeline.py +4 -0
  116. package/src/superlocalmemory/server/routes/v3_api.py +191 -15
  117. package/src/superlocalmemory/server/ui.py +20 -4
  118. package/src/superlocalmemory/server/unified_daemon.py +241 -7
  119. package/src/superlocalmemory/storage/_migration_internals.py +54 -2
  120. package/src/superlocalmemory/storage/_schema_version.py +24 -3
  121. package/src/superlocalmemory/storage/database.py +477 -59
  122. package/src/superlocalmemory/storage/embedding_codec.py +71 -0
  123. package/src/superlocalmemory/storage/lineage_retention.py +236 -0
  124. package/src/superlocalmemory/storage/logical_edges.py +43 -2
  125. package/src/superlocalmemory/storage/migration_runner.py +119 -0
  126. package/src/superlocalmemory/storage/migrations/M043_quarantine_display_summaries.py +60 -36
  127. package/src/superlocalmemory/storage/migrations/M044_play_carries_its_own_evidence.py +127 -0
  128. package/src/superlocalmemory/storage/migrations/M045_fact_outcome_score.py +158 -0
  129. package/src/superlocalmemory/storage/migrations/M046_prospective_memory_has_its_own_name.py +620 -0
  130. package/src/superlocalmemory/storage/migrations/M047_fisher_vectors_are_stored_like_every_other_vector.py +306 -0
  131. package/src/superlocalmemory/storage/migrations/M048_upcoming_holds_only_what_is_upcoming.py +207 -0
  132. package/src/superlocalmemory/storage/migrations/M049_a_schema_version_marker_is_one_row.py +201 -0
  133. package/src/superlocalmemory/storage/migrations.py +18 -2
  134. package/src/superlocalmemory/storage/models.py +40 -1
  135. package/src/superlocalmemory/storage/projection_outbox.py +346 -0
  136. package/src/superlocalmemory/storage/retention_policy.py +860 -0
  137. package/src/superlocalmemory/storage/schema.py +35 -1
  138. package/src/superlocalmemory/storage/write_coordinator.py +19 -2
  139. package/src/superlocalmemory/trust/scorer.py +43 -1
  140. package/src/superlocalmemory/ui/index.html +9 -18
  141. package/src/superlocalmemory/ui/js/event-delegation.js +12 -1
  142. package/src/superlocalmemory/ui/js/od-health.js +28 -6
  143. package/src/superlocalmemory/ui/js/od-memories.js +19 -0
  144. package/src/superlocalmemory/ui/js/od-settings.js +87 -1
  145. package/src/superlocalmemory/ui/js/recall-lab.js +78 -3
@@ -0,0 +1,219 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later - see LICENSE file
3
+ # Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
4
+
5
+ """One shape for the graph the entity walk reads, whoever stored it.
6
+
7
+ WHY THIS EXISTS
8
+ ---------------
9
+ The graph was stored twice — in SQLite and, once promoted, in CozoDB — and the
10
+ *walk over it* was written twice with it. The two implementations did not compute
11
+ the same function: the SQLite one multiplies activation by a PageRank factor at
12
+ every hop and the Cozo one had no PageRank at all. Measured on a copy of the
13
+ author's store, 3,567 of 3,667 shared facts came out with different scores, the
14
+ top-20 sets differed, and the projected path therefore failed its shadow
15
+ comparison on **every** query and fell back to SQLite. The projection was correct
16
+ data that nothing could use.
17
+
18
+ The lesson is not "fix the second walk". It is that a storage backend must supply
19
+ **data, not behaviour**. So this module defines the one shape the walk consumes,
20
+ and each store gets an adapter that produces it. Adding a third store later is
21
+ one adapter and no algorithm; there is no second implementation to keep in step,
22
+ and nothing to shadow-compare, because there is only one answer to compare.
23
+
24
+ WHY IT IS ARRAYS AND NOT DICTS
25
+ ------------------------------
26
+ The walk's cost was never the storage engine or the size of the graph — 7,460
27
+ nodes over 849k edges is small. It was doing 3.4 million relaxations as
28
+ interpreted dictionary lookups (cProfile counted 4,367,932 ``dict.get`` calls in
29
+ one recall). Held as CSR arrays the same relaxation is a handful of vectorised
30
+ passes. See :mod:`superlocalmemory.retrieval.spreading`.
31
+
32
+ The node space is fact ids. Entity ids are deliberately a separate namespace:
33
+ treating them as graph nodes produces a healthy-looking and semantically wrong
34
+ graph, which is why the projection keeps ``fact_entity`` as an explicit bridge.
35
+ """
36
+
37
+ from __future__ import annotations
38
+
39
+ import logging
40
+ from dataclasses import dataclass, field
41
+ from typing import Any, Iterable, Mapping, Protocol, Sequence
42
+
43
+ import numpy as np
44
+
45
+ logger = logging.getLogger(__name__)
46
+
47
+ #: Cap on the PageRank multiplier, matching the walk that defined it.
48
+ PAGERANK_BOOST_CAP = 2.0
49
+
50
+
51
+ @dataclass(frozen=True)
52
+ class AdjacencySnapshot:
53
+ """An immutable view of one profile's fact graph, ready to walk.
54
+
55
+ Frozen because the walk must not be able to change the graph underneath a
56
+ concurrent reader — the channel serialises *loading* behind a lock, and a
57
+ snapshot handed out after that is safe to read from anywhere.
58
+ """
59
+
60
+ node_ids: tuple[str, ...]
61
+ node_index: Mapping[str, int]
62
+ #: CSR over an UNDIRECTED graph: every edge appears in both endpoints' rows
63
+ #: with the same weight, so "incoming to j" and "outgoing from j" are the
64
+ #: same list. The walk relies on that to take a segment maximum per row.
65
+ indptr: np.ndarray
66
+ indices: np.ndarray
67
+ weights: np.ndarray
68
+ entity_to_facts: Mapping[str, tuple[int, ...]]
69
+ fact_to_entities: tuple[tuple[str, ...], ...]
70
+ #: Per-node PageRank and community, dense so the walk never branches on
71
+ #: presence. 0.0 and -1 mean "not measured", which is what the dict-based
72
+ #: walk expressed as a missing key.
73
+ pagerank: np.ndarray
74
+ community: np.ndarray
75
+ has_metrics: bool
76
+ #: Provenance, so a surface can say which store answered and an operator can
77
+ #: tell a fast path from a fallback rather than guessing.
78
+ source: str = "sqlite"
79
+ edge_count: int = 0
80
+ fact_count: int = 0
81
+ profile_id: str = "default"
82
+ extras: dict[str, Any] = field(default_factory=dict)
83
+
84
+ @property
85
+ def node_count(self) -> int:
86
+ return len(self.node_ids)
87
+
88
+ def index_of(self, fact_id: str) -> int | None:
89
+ return self.node_index.get(fact_id)
90
+
91
+ def peak_propagation_factor(self, decay: float) -> float:
92
+ """Largest ``decay * weight * pagerank_boost`` any edge can apply.
93
+
94
+ At or above 1.0 the walk can amplify along a path, and a bounded
95
+ iteration is then the only well-defined reading of it — which is what
96
+ :mod:`spreading` implements, and why it does not matter that the
97
+ dict-based walk was order-dependent in that regime.
98
+
99
+ Measured across four real workspaces: 0.7056 and 0.7100 on the two
100
+ large ones, 0.8497 on a 24-fact one, and **1.1061 on a five-fact one**,
101
+ which amplifies. Small graphs concentrate rank by construction, so the
102
+ amplifying regime is not exotic — it is what every workspace looks like
103
+ on its first day.
104
+ """
105
+ if self.weights.size == 0:
106
+ return 0.0
107
+ boost = np.minimum(1.0 + self.pagerank * 2.0, PAGERANK_BOOST_CAP)
108
+ if not self.has_metrics:
109
+ return float(decay)
110
+ return float(np.max(decay * self.weights * boost[self.indices]))
111
+
112
+
113
+ def snapshot_from_maps(
114
+ adjacency: Mapping[str, Sequence[tuple[str, float]]],
115
+ entity_to_facts: Mapping[str, Iterable[str]],
116
+ fact_to_entities: Mapping[str, Iterable[str]],
117
+ graph_metrics: Mapping[str, Mapping[str, Any]] | None,
118
+ *,
119
+ source: str,
120
+ profile_id: str,
121
+ nodes: Iterable[str] | None = None,
122
+ fact_count: int = 0,
123
+ ) -> AdjacencySnapshot:
124
+ """Build a snapshot from the dict form both adapters produce.
125
+
126
+ Node order is the sorted fact ids, not insertion order. That is what makes a
127
+ snapshot reproducible: two loads of the same graph produce the same arrays,
128
+ so a score computed from them is the same number and not a function of which
129
+ row a database happened to return first.
130
+
131
+ ``nodes`` IS THE VISIBLE FACT CORPUS, NOT THE FACTS THAT HAVE EDGES.
132
+ An earlier version derived the node space from the adjacency keys, which
133
+ silently excluded every fact with no edge yet — and ingestion is
134
+ queryable-first, so that is exactly the set a user has just added. Those
135
+ facts are reachable through their entities and the walk seeds them at 1.0;
136
+ leaving them out of the node space scored them zero instead. Caught on a real
137
+ store: four candidates, one of them the highest-scoring result for its query.
138
+ An edgeless fact is a node with an empty CSR row, which the walk's
139
+ segment-maximum already reads as zero incoming activation.
140
+ """
141
+ node_ids = tuple(sorted(set(nodes) if nodes is not None else set(adjacency)))
142
+ node_index = {fid: i for i, fid in enumerate(node_ids)}
143
+ n = len(node_ids)
144
+
145
+ indptr = np.zeros(n + 1, dtype=np.int64)
146
+ flat_indices: list[int] = []
147
+ flat_weights: list[float] = []
148
+ for i, fid in enumerate(node_ids):
149
+ # Sorted within the row for the same reason the rows are sorted.
150
+ neighbours = sorted(
151
+ (node_index[nid], float(w))
152
+ for nid, w in adjacency.get(fid, ())
153
+ if nid in node_index
154
+ )
155
+ for j, w in neighbours:
156
+ flat_indices.append(j)
157
+ flat_weights.append(w)
158
+ indptr[i + 1] = len(flat_indices)
159
+
160
+ indices = np.asarray(flat_indices, dtype=np.int64)
161
+ weights = np.asarray(flat_weights, dtype=np.float64)
162
+
163
+ metrics = graph_metrics or {}
164
+ pagerank = np.zeros(n, dtype=np.float64)
165
+ community = np.full(n, -1, dtype=np.int64)
166
+ for fid, i in node_index.items():
167
+ entry = metrics.get(fid)
168
+ if not entry:
169
+ continue
170
+ pagerank[i] = float(entry.get("pagerank_score", 0.0) or 0.0)
171
+ comm = entry.get("community_id")
172
+ if comm is not None:
173
+ try:
174
+ community[i] = int(comm)
175
+ except (TypeError, ValueError):
176
+ pass
177
+
178
+ e2f = {
179
+ eid: tuple(sorted(node_index[f] for f in facts if f in node_index))
180
+ for eid, facts in entity_to_facts.items()
181
+ }
182
+ f2e = tuple(
183
+ tuple(fact_to_entities.get(fid, ()) or ()) for fid in node_ids
184
+ )
185
+
186
+ return AdjacencySnapshot(
187
+ node_ids=node_ids,
188
+ node_index=node_index,
189
+ indptr=indptr,
190
+ indices=indices,
191
+ weights=weights,
192
+ entity_to_facts=e2f,
193
+ fact_to_entities=f2e,
194
+ pagerank=pagerank,
195
+ community=community,
196
+ has_metrics=bool(metrics),
197
+ source=source,
198
+ edge_count=int(indices.size // 2),
199
+ fact_count=fact_count or n,
200
+ profile_id=profile_id,
201
+ )
202
+
203
+
204
+ class AdjacencySource(Protocol):
205
+ """A store that can hand over one profile's fact graph.
206
+
207
+ Data only. A source that also implemented the walk is the defect this
208
+ interface exists to prevent.
209
+ """
210
+
211
+ name: str
212
+
213
+ def load(
214
+ self,
215
+ profile_id: str,
216
+ *,
217
+ include_global: bool = False,
218
+ include_shared: bool = False,
219
+ ) -> AdjacencySnapshot: ...
@@ -32,6 +32,26 @@ def authorized_fact_ids(
32
32
  unique_ids = list(dict.fromkeys(fact_ids))
33
33
  if not unique_ids:
34
34
  return set()
35
+ # Ask for the ids, not the memories. The hydrating call below answers the
36
+ # same question by decoding a 768-float embedding and two Fisher vectors per
37
+ # candidate — measured at 374 ms of a 430 ms recall on the author's store,
38
+ # to authorise 3,659 candidates for a 20-result page. Both build their
39
+ # predicate from the same two calls, so they cannot disagree; a test asserts
40
+ # it. Kept as a probe rather than a hard requirement because the lightweight
41
+ # DB wrappers in maintenance paths do not implement every method.
42
+ id_only = getattr(db, "visible_fact_ids", None)
43
+ if callable(id_only):
44
+ try:
45
+ allowed = id_only(
46
+ unique_ids,
47
+ profile_id,
48
+ include_global=bool(include_global),
49
+ include_shared=bool(include_shared),
50
+ )
51
+ if isinstance(allowed, (set, frozenset)):
52
+ return set(allowed)
53
+ except Exception:
54
+ pass
35
55
  try:
36
56
  facts = db.get_facts_by_ids(
37
57
  unique_ids,
@@ -30,6 +30,12 @@ if TYPE_CHECKING:
30
30
 
31
31
  logger = logging.getLogger(__name__)
32
32
 
33
+ #: How often the Lance projection is checked against the index it replaces: one
34
+ #: in this many searches. A comparison costs a second full search, so checking
35
+ #: every one doubled the channel, and a projection that has genuinely diverged is
36
+ #: caught inside this many queries either way.
37
+ SCALE_SHADOW_SAMPLE_EVERY = 50
38
+
33
39
  # Minimum variance floor to prevent division-by-zero in Fisher distance
34
40
  _VARIANCE_FLOOR: float = 1e-6
35
41
 
@@ -112,6 +118,8 @@ class SemanticChannel:
112
118
  self._scale_vector_backend: Any | None = None
113
119
  self._scale_shadow_checks = 0
114
120
  self._scale_shadow_mismatches = 0
121
+ self._scale_shadow_errors = 0
122
+ self._scale_searches = 0
115
123
  # V3.3.19: TurboQuant 3-tier search (stateless, optional)
116
124
  self._qas = quantization_aware_search
117
125
 
@@ -166,10 +174,37 @@ class SemanticChannel:
166
174
  and not include_global
167
175
  and not include_shared
168
176
  ):
169
- projected = self._search_via_lance(
170
- query_embedding, q_vec, profile_id, top_k,
171
- include_global=include_global, include_shared=include_shared,
172
- )
177
+ self._scale_searches += 1
178
+ # Shadowing SAMPLES rather than running on every query. Comparing
179
+ # means running both engines, and they cost the same -- 18.3 ms
180
+ # against 18.7 ms over 5,324 vectors -- so shadowing every search
181
+ # doubled this channel for an answer already known to match.
182
+ # Verified on a copy of the author's store: 18 checks, 0 mismatches.
183
+ #
184
+ # What guarantees the projection is not this comparison. It is the
185
+ # outbox, which commits the intent to project in the same SQLite
186
+ # transaction as the memory, plus a parity gate holding the
187
+ # projection to the index it stands in for. A per-query re-run of the
188
+ # canonical path is a development instrument, and leaving one in
189
+ # production is how a permanent 2x cost gets mistaken for safety.
190
+ sampled = (self._scale_searches % SCALE_SHADOW_SAMPLE_EVERY) == 1
191
+ try:
192
+ projected = self._search_via_lance(
193
+ query_embedding, q_vec, profile_id, top_k,
194
+ include_global=include_global, include_shared=include_shared,
195
+ )
196
+ except Exception as exc:
197
+ # The Cozo path this mirrors has always caught its failures; this
198
+ # one did not, so a Lance error propagated out of recall instead
199
+ # of degrading to the index that was still sitting there.
200
+ self._scale_shadow_errors += 1
201
+ logger.warning("Lance semantic projection failed, using SQLite: %s", exc)
202
+ return self._search_without_lance(
203
+ query_embedding, q_vec, profile_id, top_k,
204
+ include_global=include_global, include_shared=include_shared,
205
+ )
206
+ if not sampled:
207
+ return projected
173
208
  canonical = self._search_without_lance(
174
209
  query_embedding, q_vec, profile_id, top_k,
175
210
  include_global=include_global, include_shared=include_shared,
@@ -178,7 +213,11 @@ class SemanticChannel:
178
213
  if {fid for fid, _ in projected} == {fid for fid, _ in canonical}:
179
214
  return projected
180
215
  self._scale_shadow_mismatches += 1
181
- logger.warning("Lance semantic projection diverged from SQLite; using SQLite")
216
+ logger.warning(
217
+ "Lance semantic projection diverged from SQLite on sampled check "
218
+ "%d of %d searches; using SQLite",
219
+ self._scale_shadow_checks, self._scale_searches,
220
+ )
182
221
  return canonical
183
222
 
184
223
  # --- FAST PATH: sqlite-vec KNN ---
@@ -203,8 +242,11 @@ class SemanticChannel:
203
242
 
204
243
  def scale_projection_telemetry(self) -> dict[str, int]:
205
244
  return {
245
+ "searches": self._scale_searches,
206
246
  "shadow_checks": self._scale_shadow_checks,
207
247
  "shadow_mismatches": self._scale_shadow_mismatches,
248
+ "shadow_errors": self._scale_shadow_errors,
249
+ "sample_every": SCALE_SHADOW_SAMPLE_EVERY,
208
250
  }
209
251
 
210
252
  def _search_via_lance(
@@ -0,0 +1,288 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later - see LICENSE file
3
+ # Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
4
+
5
+ """The spreading-activation walk. One implementation, no storage in it.
6
+
7
+ A pure function of an :class:`~.graph_adjacency.AdjacencySnapshot`. It opens no
8
+ connection and knows no backend, which is the point: the same walk answers
9
+ whether the graph came out of SQLite or CozoDB, so the two can never disagree and
10
+ there is nothing to shadow-compare.
11
+
12
+ WHAT IT COMPUTES
13
+ ----------------
14
+ Activation starts at 1.0 on every fact linked to a query's entities and spreads
15
+ outward for ``max_hops``, multiplied each hop by ``decay``, by the edge weight,
16
+ and by a PageRank factor on the receiving fact. A fact's score is the best path
17
+ that reaches it — a max, not a sum, so a fact reached twice is not thereby more
18
+ relevant. Two enrichments follow the spread (a community bonus and a
19
+ contradiction penalty), then scores are normalised to [0, 1].
20
+
21
+ SYNCHRONOUS, AND WHY THAT IS A FIX
22
+ ----------------------------------
23
+ Each hop is computed from the previous hop's values only. The dict-based walk
24
+ this replaces read ``activation[fid]`` while iterating the frontier, so a fact
25
+ updated earlier in a hop propagated again within that same hop — meaning the
26
+ walk could reach further than ``max_hops`` allowed, by an amount that depended
27
+ on set iteration order.
28
+
29
+ That is benign exactly while ``decay * weight * pagerank_boost < 1`` everywhere,
30
+ because then the max-product has a unique fixpoint and order only changes how
31
+ fast it is reached.
32
+
33
+ **It is not benign in general, and it is not benign here.** The boost is capped
34
+ at 2.0 and ``decay`` is 0.7, so any graph whose peak rank reaches 0.215
35
+ amplifies. Measured across four real workspaces after the ranking was repaired:
36
+
37
+ workspace facts peak rank peak factor
38
+ a large one 12,078 0.003984 0.7056
39
+ another one 4,038 0.007155 0.7100
40
+ a small one 24 0.106919 0.8497
41
+ a very small one 5 0.290068 1.1061 ← amplifies
42
+
43
+ The last row is the point. A small graph concentrates rank by construction — a
44
+ five-fact workspace on a first day of use is not a corner case, it is every
45
+ workspace's first day — and there the old walk's answer depended on dictionary
46
+ ordering. A bounded synchronous iteration is well-defined in both regimes.
47
+
48
+ (These numbers replace an earlier note citing a peak of 0.1 and a factor of
49
+ 0.84. That reading came from a ranking table whose scores summed to 1.9999 and
50
+ 3.3150 on two real stores rather than to 1, so the peak it reported was an
51
+ artefact of the table being wrong, not a property of any graph.)
52
+ :meth:`AdjacencySnapshot.peak_propagation_factor` is how a caller can see which
53
+ regime a store is in.
54
+
55
+ WHY ARRAYS
56
+ ----------
57
+ The relaxation is a max-times sparse product. Held as CSR, one hop is one
58
+ multiply and one segment-maximum over the edge array, in place of a Python loop
59
+ that did 3.4 million dictionary lookups to produce a hundred numbers.
60
+ """
61
+
62
+ from __future__ import annotations
63
+
64
+ import logging
65
+ from dataclasses import dataclass
66
+ from typing import Iterable, Mapping, Sequence
67
+
68
+ import numpy as np
69
+
70
+ from superlocalmemory.retrieval.graph_adjacency import (
71
+ PAGERANK_BOOST_CAP,
72
+ AdjacencySnapshot,
73
+ )
74
+
75
+ logger = logging.getLogger(__name__)
76
+
77
+ #: Community-bonus ceiling and the penalty for a fact outside every seed
78
+ #: community. Carried over unchanged from the walk that introduced them.
79
+ COMMUNITY_BONUS_SCALE = 0.15
80
+ COMMUNITY_BONUS_CAP = 1.3
81
+ COMMUNITY_OUTSIDER_PENALTY = 0.9
82
+
83
+
84
+ @dataclass(frozen=True)
85
+ class ActivationResult:
86
+ """Activation over the snapshot's node space, plus how it was reached."""
87
+
88
+ scores: np.ndarray
89
+ hops_run: int
90
+ seeded: int
91
+ normalised_by: float
92
+
93
+ def as_mapping(
94
+ self, snapshot: AdjacencySnapshot, *, threshold: float
95
+ ) -> dict[str, float]:
96
+ """The scores at or above ``threshold``, keyed by fact id."""
97
+ keep = np.flatnonzero(self.scores >= threshold)
98
+ return {snapshot.node_ids[i]: float(self.scores[i]) for i in keep}
99
+
100
+
101
+ def _boost(snapshot: AdjacencySnapshot) -> np.ndarray:
102
+ """Per-node PageRank multiplier, or ones when no metrics were measured.
103
+
104
+ Without metrics the walk deliberately drops the edge weight too. That is not
105
+ an oversight: weighting without a compensating boost dampens propagation by
106
+ about 14% and measurably lowered retrieval quality, so the two arrived
107
+ together and have to leave together.
108
+ """
109
+ if not snapshot.has_metrics:
110
+ return np.ones(snapshot.node_count, dtype=np.float64)
111
+ return np.minimum(1.0 + snapshot.pagerank * 2.0, PAGERANK_BOOST_CAP)
112
+
113
+
114
+ def _segment_max(values: np.ndarray, indptr: np.ndarray, rows: int) -> np.ndarray:
115
+ """Maximum of ``values`` within each CSR row; 0.0 for an empty row.
116
+
117
+ ``np.maximum.reduceat`` returns the element at the start offset for a
118
+ zero-length segment rather than an identity, so empty rows would inherit
119
+ whichever neighbour happened to sit at that offset — a silent wrong answer
120
+ for exactly the isolated facts a graph channel should score at zero.
121
+ """
122
+ out = np.zeros(rows, dtype=np.float64)
123
+ if values.size == 0:
124
+ return out
125
+ starts = indptr[:-1]
126
+ non_empty = starts < indptr[1:]
127
+ if not non_empty.any():
128
+ return out
129
+ reduced = np.maximum.reduceat(values, starts[non_empty])
130
+ out[non_empty] = reduced
131
+ return out
132
+
133
+
134
+ def activate(
135
+ snapshot: AdjacencySnapshot,
136
+ seed_entity_ids: Sequence[str],
137
+ *,
138
+ decay: float,
139
+ threshold: float,
140
+ max_hops: int,
141
+ ) -> ActivationResult:
142
+ """Spread activation from a query's entities across the fact graph."""
143
+ n = snapshot.node_count
144
+ scores = np.zeros(n, dtype=np.float64)
145
+ if n == 0 or not seed_entity_ids:
146
+ return ActivationResult(scores, 0, 0, 1.0)
147
+
148
+ seeded_nodes: list[int] = []
149
+ for entity_id in seed_entity_ids:
150
+ seeded_nodes.extend(snapshot.entity_to_facts.get(entity_id, ()))
151
+ if seeded_nodes:
152
+ scores[np.asarray(seeded_nodes, dtype=np.int64)] = 1.0
153
+
154
+ visited_entities = set(seed_entity_ids)
155
+ frontier = np.flatnonzero(scores > 0.0)
156
+ boost = _boost(snapshot)
157
+ use_weights = snapshot.has_metrics
158
+ hops_run = 0
159
+
160
+ for hop in range(1, max_hops):
161
+ hop_decay = decay**hop
162
+ if hop_decay < threshold:
163
+ break
164
+ if frontier.size == 0:
165
+ break
166
+ hops_run = hop
167
+
168
+ # --- edge propagation, one vectorised pass over the edge array -------
169
+ # Every edge sits in both endpoints' rows with the same weight, so the
170
+ # maximum over a row is the maximum over that node's incoming values.
171
+ contributions = scores[snapshot.indices] * decay
172
+ if use_weights:
173
+ contributions = contributions * snapshot.weights
174
+ incoming = _segment_max(contributions, snapshot.indptr, n)
175
+ if use_weights:
176
+ incoming = incoming * boost
177
+ # A hop only ever raises a score, and only above the threshold.
178
+ candidate = np.where(incoming >= threshold, incoming, 0.0)
179
+ improved = candidate > scores
180
+ next_nodes = np.flatnonzero(improved)
181
+ scores[improved] = candidate[improved]
182
+
183
+ # --- entity hop: facts reached through a newly seen entity ----------
184
+ # Stateful and cheap, so it stays a loop. It reads the frontier as it
185
+ # was at the start of this hop, which is the same order of operations
186
+ # the dict walk used.
187
+ newly_seen: list[str] = []
188
+ for node in frontier.tolist():
189
+ for entity_id in snapshot.fact_to_entities[node]:
190
+ if entity_id not in visited_entities:
191
+ visited_entities.add(entity_id)
192
+ newly_seen.append(entity_id)
193
+ entity_nodes: list[int] = []
194
+ for entity_id in newly_seen:
195
+ entity_nodes.extend(snapshot.entity_to_facts.get(entity_id, ()))
196
+ if entity_nodes:
197
+ reached = np.asarray(sorted(set(entity_nodes)), dtype=np.int64)
198
+ lifts = reached[hop_decay > scores[reached]]
199
+ if lifts.size:
200
+ scores[lifts] = hop_decay
201
+ next_nodes = np.union1d(next_nodes, lifts)
202
+
203
+ frontier = next_nodes
204
+
205
+ return ActivationResult(scores, hops_run, len(seeded_nodes), 1.0)
206
+
207
+
208
+ def apply_community_bias(
209
+ scores: np.ndarray,
210
+ snapshot: AdjacencySnapshot,
211
+ seed_entity_ids: Iterable[str],
212
+ *,
213
+ penalise_outsiders: bool = True,
214
+ ) -> None:
215
+ """Favour facts sharing a community with the query's seeds. In place.
216
+
217
+ A no-op without metrics, which is also when ``community`` is all -1.
218
+
219
+ ``penalise_outsiders`` exists because the two callers genuinely differ and
220
+ the difference is not cosmetic. Search damps a fact belonging to no seed
221
+ community by 0.9; candidate scoring does not, because it re-scores a set
222
+ another channel already chose and a graph signal has no business vetoing
223
+ that channel's find. Collapsing the two into one behaviour would silently
224
+ change one caller's results, so the asymmetry is a parameter and this
225
+ paragraph is why.
226
+ """
227
+ if not snapshot.has_metrics or scores.size == 0:
228
+ return
229
+ seed_nodes: list[int] = []
230
+ for entity_id in seed_entity_ids:
231
+ seed_nodes.extend(snapshot.entity_to_facts.get(entity_id, ()))
232
+ if not seed_nodes:
233
+ return
234
+ seed_communities = snapshot.community[np.asarray(seed_nodes, dtype=np.int64)]
235
+ seed_communities = seed_communities[seed_communities >= 0]
236
+ if seed_communities.size == 0:
237
+ return
238
+ labels, counts = np.unique(seed_communities, return_counts=True)
239
+ total = float(counts.sum())
240
+ share = dict(zip(labels.tolist(), (counts / total).tolist()))
241
+
242
+ known = snapshot.community >= 0
243
+ multiplier = np.ones(scores.size, dtype=np.float64)
244
+ for label, fraction in share.items():
245
+ in_seed = known & (snapshot.community == label)
246
+ multiplier[in_seed] = min(
247
+ 1.0 + COMMUNITY_BONUS_SCALE * fraction, COMMUNITY_BONUS_CAP
248
+ )
249
+ if penalise_outsiders:
250
+ outsider = known & ~np.isin(snapshot.community, labels)
251
+ multiplier[outsider] = COMMUNITY_OUTSIDER_PENALTY
252
+ scores *= multiplier
253
+
254
+
255
+ def normalise(scores: np.ndarray, *, threshold: float) -> float:
256
+ """Scale scores so the best is 1.0. Returns the divisor used. In place.
257
+
258
+ Reported rather than hidden because it is the only reason a fact seeded at
259
+ exactly 1.0 can come back as something else, which is otherwise a confusing
260
+ thing to see in a trace.
261
+ """
262
+ if scores.size == 0:
263
+ return 1.0
264
+ kept = scores[scores >= threshold]
265
+ if kept.size == 0:
266
+ return 1.0
267
+ peak = float(kept.max())
268
+ if peak > 0.0:
269
+ scores /= peak
270
+ return peak
271
+ return 1.0
272
+
273
+
274
+ def ranked(
275
+ scores: np.ndarray, snapshot: AdjacencySnapshot, *, threshold: float
276
+ ) -> list[tuple[str, float]]:
277
+ """Facts at or above ``threshold``, best first, ties broken on fact id.
278
+
279
+ The tie-break is part of the contract, not a detail. An entity-seeded walk
280
+ puts every directly-linked fact at exactly the same score, so a large group
281
+ arrives at the cut-off together and which of them survives ``top_k`` is
282
+ decided entirely by how ties are ordered. Two implementations that agreed on
283
+ every score but not on this returned different results for the same query.
284
+ """
285
+ keep = np.flatnonzero(scores >= threshold)
286
+ pairs = [(snapshot.node_ids[i], float(scores[i])) for i in keep]
287
+ pairs.sort(key=lambda item: (-item[1], item[0]))
288
+ return pairs
@@ -39,6 +39,7 @@ from fastapi.middleware.gzip import GZipMiddleware
39
39
  from pydantic import BaseModel
40
40
  import uvicorn
41
41
 
42
+ from superlocalmemory.core.config import CANONICAL_RECALL_LIMIT
42
43
  from superlocalmemory.server.security_middleware import SecurityHeadersMiddleware
43
44
  from superlocalmemory.server.routes.helpers import SLM_VERSION
44
45
  from superlocalmemory.infra.data_root import DynamicStatePath
@@ -60,7 +61,7 @@ UI_DIR = Path(__file__).resolve().parent.parent / "ui"
60
61
 
61
62
  class SearchRequest(BaseModel):
62
63
  query: str
63
- limit: int = 10
64
+ limit: int = CANONICAL_RECALL_LIMIT
64
65
  min_score: float = 0.3
65
66
 
66
67
 
@@ -246,12 +247,30 @@ def create_app() -> FastAPI:
246
247
  "<p><a href='/docs'>API Documentation</a></p>"
247
248
  "</body></html>"
248
249
  )
249
- from superlocalmemory.server.asset_versions import render_index
250
250
  from superlocalmemory import __version__ as _v
251
251
 
252
- return render_index(
253
- index_path, UI_DIR, substitutions={"__SLM_VERSION__": _v},
254
- )
252
+ # __SLM_VERSION__ was substituted only by the unified daemon, so the
253
+ # dashboard's upgrade detector did nothing when served from here.
254
+ # Asset versioning is cosmetic. It must never be why this page 500s.
255
+ #
256
+ # The import is deferred (house style, keeps startup lean), which means
257
+ # it resolves at REQUEST time — so when `pip install -e .` replaced the
258
+ # installed package underneath a running daemon, this route began
259
+ # answering "Internal Server Error" on the dashboard while every other
260
+ # endpoint was fine. A stale hand-written version string is a trifle; a
261
+ # blank page is not. Fall back to the file as written.
262
+ try:
263
+ from superlocalmemory.server.asset_versions import render_index
264
+
265
+ return render_index(
266
+ index_path, UI_DIR, substitutions={"__SLM_VERSION__": _v},
267
+ )
268
+ except Exception as exc: # noqa: BLE001 — serve the page regardless
269
+ logger.warning(
270
+ "asset version rewrite unavailable, serving index.html as "
271
+ "written: %s: %s", type(exc).__name__, exc,
272
+ )
273
+ return index_path.read_text().replace("__SLM_VERSION__", _v)
255
274
 
256
275
  @application.get("/health")
257
276
  async def health_check():