superlocalmemory 4.0.9 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/CHANGELOG.md +245 -0
  3. package/README.md +7 -7
  4. package/package.json +4 -2
  5. package/plugin/.claude-plugin/plugin.json +2 -2
  6. package/plugin/CLAUDE.md +3 -3
  7. package/plugin/agents/slm-governance-advisor.md +1 -1
  8. package/plugin/agents/slm-loop-runner.md +4 -4
  9. package/plugin/agents/slm-memory-advisor.md +1 -1
  10. package/plugin/agents/slm-optimize-advisor.md +1 -1
  11. package/plugin/requirements.txt +1 -1
  12. package/plugin/skills/slm-cache/SKILL.md +1 -1
  13. package/plugin/skills/slm-compress/SKILL.md +1 -1
  14. package/plugin/skills/slm-governance/SKILL.md +1 -1
  15. package/plugin/skills/slm-graph/SKILL.md +1 -1
  16. package/plugin/skills/slm-loop/SKILL.md +2 -2
  17. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  18. package/plugin/skills/slm-profile/SKILL.md +5 -5
  19. package/plugin/skills/slm-recall/SKILL.md +102 -15
  20. package/plugin/skills/slm-remember/SKILL.md +35 -3
  21. package/plugin/skills/slm-scope/SKILL.md +1 -1
  22. package/plugin/skills/slm-session/SKILL.md +29 -3
  23. package/plugin/skills/slm-status/SKILL.md +1 -1
  24. package/plugin-src/rules/AGENTS.md +16 -8
  25. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  26. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-governance/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  29. package/plugin-src/skills/slm-loop/SKILL.md +2 -2
  30. package/plugin-src/skills/slm-mesh/SKILL.md +1 -1
  31. package/plugin-src/skills/slm-profile/SKILL.md +5 -5
  32. package/plugin-src/skills/slm-recall/SKILL.md +102 -15
  33. package/plugin-src/skills/slm-remember/SKILL.md +35 -3
  34. package/plugin-src/skills/slm-scope/SKILL.md +1 -1
  35. package/plugin-src/skills/slm-session/SKILL.md +29 -3
  36. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  37. package/pyproject.toml +1 -1
  38. package/src/superlocalmemory/__init__.py +1 -1
  39. package/src/superlocalmemory/cli/commands.py +308 -20
  40. package/src/superlocalmemory/cli/daemon.py +30 -0
  41. package/src/superlocalmemory/cli/db_migrate.py +71 -1
  42. package/src/superlocalmemory/cli/gdpr_cmd.py +15 -2
  43. package/src/superlocalmemory/cli/main.py +26 -4
  44. package/src/superlocalmemory/code_graph/bridge/maintenance.py +8 -0
  45. package/src/superlocalmemory/code_graph/database.py +44 -0
  46. package/src/superlocalmemory/compliance/gdpr.py +449 -39
  47. package/src/superlocalmemory/core/admission.py +231 -11
  48. package/src/superlocalmemory/core/backend_orchestrator.py +190 -84
  49. package/src/superlocalmemory/core/config.py +90 -11
  50. package/src/superlocalmemory/core/consolidation_engine.py +34 -0
  51. package/src/superlocalmemory/core/engine.py +140 -11
  52. package/src/superlocalmemory/core/fact_consolidator.py +316 -125
  53. package/src/superlocalmemory/core/graph_analyzer.py +76 -112
  54. package/src/superlocalmemory/core/graph_metrics.py +597 -0
  55. package/src/superlocalmemory/core/graph_pruner.py +121 -0
  56. package/src/superlocalmemory/core/maintenance.py +44 -6
  57. package/src/superlocalmemory/core/maintenance_scheduler.py +205 -0
  58. package/src/superlocalmemory/core/memory_health.py +266 -0
  59. package/src/superlocalmemory/core/mode_capability.py +111 -0
  60. package/src/superlocalmemory/core/ollama_validator.py +315 -0
  61. package/src/superlocalmemory/core/operation_policy_registry.py +1 -1
  62. package/src/superlocalmemory/core/operation_request.py +1 -1
  63. package/src/superlocalmemory/core/ops_remediation.py +2 -2
  64. package/src/superlocalmemory/core/projection_drain.py +380 -0
  65. package/src/superlocalmemory/core/recall_pipeline.py +390 -3
  66. package/src/superlocalmemory/core/recall_worker.py +6 -3
  67. package/src/superlocalmemory/core/scale_autopromote.py +196 -0
  68. package/src/superlocalmemory/core/scale_engine.py +16 -2
  69. package/src/superlocalmemory/core/score_contract.py +21 -1
  70. package/src/superlocalmemory/core/session_identity.py +85 -0
  71. package/src/superlocalmemory/core/status_contract.py +108 -0
  72. package/src/superlocalmemory/core/store_pipeline.py +78 -3
  73. package/src/superlocalmemory/core/worker_pool.py +4 -4
  74. package/src/superlocalmemory/core/working_memory.py +288 -0
  75. package/src/superlocalmemory/encoding/cognitive_consolidator.py +51 -7
  76. package/src/superlocalmemory/encoding/context_generator.py +1 -1
  77. package/src/superlocalmemory/encoding/entity_resolver.py +38 -0
  78. package/src/superlocalmemory/encoding/fact_extractor.py +18 -14
  79. package/src/superlocalmemory/encoding/prospective_markers.py +262 -0
  80. package/src/superlocalmemory/encoding/type_router.py +12 -12
  81. package/src/superlocalmemory/evolution/mutation_generator.py +30 -4
  82. package/src/superlocalmemory/graph/cozo_adjacency.py +122 -0
  83. package/src/superlocalmemory/graph/cozo_backend.py +103 -138
  84. package/src/superlocalmemory/hooks/portable_kit.py +10 -2
  85. package/src/superlocalmemory/learning/bandit.py +43 -0
  86. package/src/superlocalmemory/learning/consolidation_worker.py +54 -0
  87. package/src/superlocalmemory/learning/database.py +60 -3
  88. package/src/superlocalmemory/learning/entity_compiler.py +21 -58
  89. package/src/superlocalmemory/learning/feedback.py +3 -1
  90. package/src/superlocalmemory/learning/outcomes.py +47 -16
  91. package/src/superlocalmemory/learning/pattern_miner.py +28 -3
  92. package/src/superlocalmemory/learning/pattern_miner_constants.py +43 -0
  93. package/src/superlocalmemory/learning/pcos.py +291 -0
  94. package/src/superlocalmemory/learning/reward_from_outcomes.py +365 -0
  95. package/src/superlocalmemory/learning/reward_proxy.py +100 -10
  96. package/src/superlocalmemory/learning/signal_kinds.py +79 -0
  97. package/src/superlocalmemory/mcp/profiles.py +14 -2
  98. package/src/superlocalmemory/mcp/server.py +1 -1
  99. package/src/superlocalmemory/mcp/session_binding.py +92 -0
  100. package/src/superlocalmemory/mcp/tools_active.py +2 -1
  101. package/src/superlocalmemory/mcp/tools_core.py +71 -42
  102. package/src/superlocalmemory/mcp/tools_ops.py +2 -2
  103. package/src/superlocalmemory/mcp/tools_v28.py +20 -1
  104. package/src/superlocalmemory/parameterization/pattern_extractor.py +14 -1
  105. package/src/superlocalmemory/parameterization/soft_prompt_generator.py +98 -0
  106. package/src/superlocalmemory/retrieval/bm25_channel.py +68 -11
  107. package/src/superlocalmemory/retrieval/channel_status.py +117 -0
  108. package/src/superlocalmemory/retrieval/engine.py +106 -11
  109. package/src/superlocalmemory/retrieval/entity_channel.py +217 -257
  110. package/src/superlocalmemory/retrieval/graph_adjacency.py +219 -0
  111. package/src/superlocalmemory/retrieval/scope_policy.py +42 -1
  112. package/src/superlocalmemory/retrieval/semantic_channel.py +47 -5
  113. package/src/superlocalmemory/retrieval/spreading.py +288 -0
  114. package/src/superlocalmemory/retrieval/temporal_channel.py +13 -1
  115. package/src/superlocalmemory/retrieval/vector_store.py +63 -0
  116. package/src/superlocalmemory/server/api.py +26 -2
  117. package/src/superlocalmemory/server/asset_versions.py +171 -0
  118. package/src/superlocalmemory/server/bandit_loops.py +17 -1
  119. package/src/superlocalmemory/server/rbac_enforce.py +26 -6
  120. package/src/superlocalmemory/server/recall_serializer.py +9 -0
  121. package/src/superlocalmemory/server/routes/abstraction.py +201 -0
  122. package/src/superlocalmemory/server/routes/behavioral.py +75 -10
  123. package/src/superlocalmemory/server/routes/compliance.py +98 -18
  124. package/src/superlocalmemory/server/routes/config_api.py +186 -4
  125. package/src/superlocalmemory/server/routes/data_io.py +29 -1
  126. package/src/superlocalmemory/server/routes/entity.py +13 -1
  127. package/src/superlocalmemory/server/routes/evolution.py +178 -0
  128. package/src/superlocalmemory/server/routes/ingest.py +8 -0
  129. package/src/superlocalmemory/server/routes/learning_telemetry.py +2 -1
  130. package/src/superlocalmemory/server/routes/memories.py +49 -7
  131. package/src/superlocalmemory/server/routes/mesh.py +1 -1
  132. package/src/superlocalmemory/server/routes/timeline.py +4 -0
  133. package/src/superlocalmemory/server/routes/v3_api.py +193 -17
  134. package/src/superlocalmemory/server/ui.py +24 -1
  135. package/src/superlocalmemory/server/unified_daemon.py +292 -9
  136. package/src/superlocalmemory/storage/_migration_internals.py +35 -0
  137. package/src/superlocalmemory/storage/_schema_version.py +24 -3
  138. package/src/superlocalmemory/storage/database.py +598 -82
  139. package/src/superlocalmemory/storage/embedding_codec.py +71 -0
  140. package/src/superlocalmemory/storage/lineage_retention.py +236 -0
  141. package/src/superlocalmemory/storage/logical_edges.py +43 -2
  142. package/src/superlocalmemory/storage/migration_runner.py +130 -0
  143. package/src/superlocalmemory/storage/migrations/M043_quarantine_display_summaries.py +488 -0
  144. package/src/superlocalmemory/storage/migrations/M044_play_carries_its_own_evidence.py +127 -0
  145. package/src/superlocalmemory/storage/migrations/M045_fact_outcome_score.py +158 -0
  146. package/src/superlocalmemory/storage/migrations/M046_prospective_memory_has_its_own_name.py +620 -0
  147. package/src/superlocalmemory/storage/migrations/M047_fisher_vectors_are_stored_like_every_other_vector.py +306 -0
  148. package/src/superlocalmemory/storage/migrations/M048_upcoming_holds_only_what_is_upcoming.py +207 -0
  149. package/src/superlocalmemory/storage/migrations/M049_a_schema_version_marker_is_one_row.py +201 -0
  150. package/src/superlocalmemory/storage/migrations.py +18 -2
  151. package/src/superlocalmemory/storage/models.py +40 -1
  152. package/src/superlocalmemory/storage/projection_outbox.py +346 -0
  153. package/src/superlocalmemory/storage/retention_policy.py +860 -0
  154. package/src/superlocalmemory/storage/schema.py +110 -1
  155. package/src/superlocalmemory/storage/write_coordinator.py +19 -2
  156. package/src/superlocalmemory/summaries/base.py +1 -1
  157. package/src/superlocalmemory/summaries/non_answer.py +223 -0
  158. package/src/superlocalmemory/trust/scorer.py +43 -1
  159. package/src/superlocalmemory/ui/index.html +10 -19
  160. package/src/superlocalmemory/ui/js/event-delegation.js +12 -1
  161. package/src/superlocalmemory/ui/js/od-health.js +28 -6
  162. package/src/superlocalmemory/ui/js/od-memories.js +209 -1
  163. package/src/superlocalmemory/ui/js/od-ops-health.js +1 -1
  164. package/src/superlocalmemory/ui/js/od-settings.js +87 -1
  165. package/src/superlocalmemory/ui/js/recall-lab.js +78 -3
@@ -0,0 +1,266 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later - see LICENSE file
3
+
4
+ """Tell the owner, in their own words, whether their memory works.
5
+
6
+ Until now the only way to learn that 43.7% of a store could not be found by
7
+ asking a question was to write the SQL yourself. One machine sat in exactly
8
+ that state for months while every status line it showed said the system was
9
+ healthy, because nothing measured reachability and nothing reported it.
10
+
11
+ So this module answers four questions a non-engineer can act on:
12
+
13
+ * How many memories do I have?
14
+ * How many can actually be found by asking a question?
15
+ * How many were withheld because a model wrote them, not me?
16
+ * Is anything still being repaired?
17
+
18
+ Read-only, and every query is bounded. Fail-soft by construction: a missing
19
+ table or column yields ``None`` for that line rather than an exception, because
20
+ a health report that crashes on an old store is worse than one that says "not
21
+ known yet".
22
+
23
+ Consumed by ``slm doctor``, ``GET /api/v3/memory-health``, and the dashboard.
24
+ One implementation so the three cannot disagree with each other.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import logging
30
+ import re
31
+ import sqlite3
32
+ from dataclasses import dataclass, field
33
+ from pathlib import Path
34
+
35
+ logger = logging.getLogger(__name__)
36
+
37
+ __all__ = ["MemoryHealth", "measure", "describe"]
38
+
39
+
40
+ @dataclass(frozen=True)
41
+ class MemoryHealth:
42
+ """A store's answer-ability, counted rather than assumed."""
43
+
44
+ #: Memories that recall is allowed to return.
45
+ live_facts: int = 0
46
+ #: Of those, how many have a vector projection, i.e. can be found by
47
+ #: meaning rather than only by matching words.
48
+ findable_by_meaning: int = 0
49
+ #: Memories with no vector at all. These are reachable by keyword only.
50
+ missing_vector: int = 0
51
+ #: Machine-written summaries withheld from recall and kept for display.
52
+ withheld_summaries: int = 0
53
+ #: Summaries preserved in the display table.
54
+ display_summaries: int = 0
55
+ #: Memories hidden by the retention system, excluding the withheld ones.
56
+ hidden_by_forgetting: int = 0
57
+ #: Rows whose retention zone contradicts their retention score, i.e. hidden
58
+ #: while the maths says to keep them. Should be zero after repair.
59
+ inconsistently_hidden: int = 0
60
+ #: Present only when a table or column was absent.
61
+ unavailable: tuple[str, ...] = field(default_factory=tuple)
62
+
63
+ @property
64
+ def reachability(self) -> float:
65
+ """Share of live memories findable by meaning, 0.0-1.0."""
66
+ if self.live_facts <= 0:
67
+ return 1.0
68
+ return self.findable_by_meaning / self.live_facts
69
+
70
+ @property
71
+ def healthy(self) -> bool:
72
+ """Whether anything here warrants telling the owner about."""
73
+ return (
74
+ self.reachability >= 0.99
75
+ and self.missing_vector == 0
76
+ and self.inconsistently_hidden == 0
77
+ )
78
+
79
+
80
+ def measure(db_path: str | Path) -> MemoryHealth:
81
+ """Count the store's answer-ability. Read-only; never raises."""
82
+ unavailable: list[str] = []
83
+ try:
84
+ conn = sqlite3.connect(f"file:{Path(db_path)}?mode=ro", uri=True)
85
+ except sqlite3.Error as exc:
86
+ logger.debug("memory health: cannot open %s: %s", db_path, exc)
87
+ return MemoryHealth(unavailable=("database",))
88
+
89
+ try:
90
+ if not _table_exists(conn, "atomic_facts"):
91
+ return MemoryHealth(unavailable=("atomic_facts",))
92
+
93
+ # Quarantine came in 4.0.10. On an older store every fact is "live",
94
+ # which is the honest reading of a store that has no withheld rows.
95
+ has_q = _has_column(conn, "atomic_facts", "quarantined")
96
+ if not has_q:
97
+ unavailable.append("quarantined")
98
+ live_clause = "COALESCE(quarantined, 0) = 0" if has_q else "1=1"
99
+
100
+ live = _count(conn, f"SELECT COUNT(*) FROM atomic_facts WHERE {live_clause}")
101
+ withheld = (
102
+ _count(conn, "SELECT COUNT(*) FROM atomic_facts WHERE quarantined = 1")
103
+ if has_q else 0
104
+ )
105
+ missing_vec = _count(
106
+ conn,
107
+ f"SELECT COUNT(*) FROM atomic_facts "
108
+ f"WHERE embedding IS NULL AND {live_clause}",
109
+ )
110
+
111
+ if _table_exists(conn, "embedding_metadata"):
112
+ findable = _count(
113
+ conn,
114
+ "SELECT COUNT(*) FROM embedding_metadata em "
115
+ "JOIN atomic_facts af ON af.fact_id = em.fact_id "
116
+ f"WHERE {_prefixed(live_clause, 'af')}",
117
+ )
118
+ else:
119
+ unavailable.append("embedding_metadata")
120
+ findable = 0
121
+
122
+ display = (
123
+ _count(conn, "SELECT COUNT(*) FROM consolidated_summaries")
124
+ if _table_exists(conn, "consolidated_summaries") else 0
125
+ )
126
+ if not _table_exists(conn, "consolidated_summaries"):
127
+ unavailable.append("consolidated_summaries")
128
+
129
+ hidden = inconsistent = 0
130
+ if _table_exists(conn, "fact_retention"):
131
+ hidden = _count(
132
+ conn,
133
+ "SELECT COUNT(*) FROM fact_retention r "
134
+ "JOIN atomic_facts af ON af.fact_id = r.fact_id "
135
+ "WHERE r.lifecycle_zone IN ('archive', 'forgotten') "
136
+ f" AND {_prefixed(live_clause, 'af')}",
137
+ )
138
+ # The contradiction M043 repairs: hidden, yet scored to keep.
139
+ inconsistent = _count(
140
+ conn,
141
+ "SELECT COUNT(*) FROM fact_retention r "
142
+ "JOIN atomic_facts af ON af.fact_id = r.fact_id "
143
+ "WHERE r.lifecycle_zone IN ('archive', 'forgotten') "
144
+ " AND r.retention_score > 0.8 "
145
+ f" AND {_prefixed(live_clause, 'af')}",
146
+ )
147
+ else:
148
+ unavailable.append("fact_retention")
149
+
150
+ return MemoryHealth(
151
+ live_facts=live,
152
+ findable_by_meaning=findable,
153
+ missing_vector=missing_vec,
154
+ withheld_summaries=withheld,
155
+ display_summaries=display,
156
+ hidden_by_forgetting=hidden,
157
+ inconsistently_hidden=inconsistent,
158
+ unavailable=tuple(unavailable),
159
+ )
160
+ except sqlite3.Error as exc:
161
+ logger.debug("memory health measurement failed: %s", exc)
162
+ return MemoryHealth(unavailable=(*unavailable, "query_failed"))
163
+ finally:
164
+ conn.close()
165
+
166
+
167
+ def describe(health: MemoryHealth) -> list[str]:
168
+ """Plain-language lines for a reader who does not write SQL.
169
+
170
+ No percentages without the counts behind them, and no jargon: "findable by
171
+ asking a question" rather than "vector coverage", because the person who
172
+ needs this line is the one who would not know what a vector is.
173
+ """
174
+ lines: list[str] = []
175
+ if "atomic_facts" in health.unavailable or "database" in health.unavailable:
176
+ return ["Memory store not readable yet."]
177
+
178
+ lines.append(f"You have {health.live_facts:,} memories.")
179
+
180
+ if "embedding_metadata" in health.unavailable:
181
+ lines.append(
182
+ "Whether they can be found by asking a question is not known yet — "
183
+ "the search index has not been built."
184
+ )
185
+ elif health.live_facts:
186
+ pct = 100.0 * health.reachability
187
+ if health.findable_by_meaning >= health.live_facts:
188
+ # "All" only when the counts actually agree. The threshold used to
189
+ # be reachability >= 0.99, which printed "All of them can be found
190
+ # by asking a question (5,199 indexed)" on a store of 5,205 — a
191
+ # claim of all, contradicted by the number beside it. This module
192
+ # exists to be believed; it cannot round in its own favour.
193
+ lines.append(
194
+ f"All {health.live_facts:,} of them can be found by asking a "
195
+ f"question."
196
+ )
197
+ elif health.reachability >= 0.99:
198
+ gap = health.live_facts - health.findable_by_meaning
199
+ lines.append(
200
+ f"{health.findable_by_meaning:,} of them can be found by asking "
201
+ f"a question. The other {gap:,} can only be found by matching "
202
+ f"words. That is a small enough share to be normal — a memory "
203
+ f"written moments ago, or one the model could not read."
204
+ )
205
+ else:
206
+ gap = health.live_facts - health.findable_by_meaning
207
+ lines.append(
208
+ f"{health.findable_by_meaning:,} of them ({pct:.0f}%) can be "
209
+ f"found by asking a question. The other {gap:,} can only be "
210
+ f"found by matching words, so a question phrased differently "
211
+ f"will miss them. This repairs itself as the service runs; if "
212
+ f"it does not, the embedding model is unavailable."
213
+ )
214
+
215
+ if health.withheld_summaries:
216
+ lines.append(
217
+ f"{health.withheld_summaries:,} machine-written summaries are kept "
218
+ f"out of your answers and shown on the dashboard instead. They were "
219
+ f"written by the summarizer, not by you, and they used to be "
220
+ f"returned as if they were your own notes."
221
+ )
222
+
223
+ if health.inconsistently_hidden:
224
+ lines.append(
225
+ f"{health.inconsistently_hidden:,} memories are hidden even though "
226
+ f"they are marked worth keeping. This is a fault and it is repaired "
227
+ f"automatically the next time the service starts."
228
+ )
229
+
230
+ if health.hidden_by_forgetting:
231
+ lines.append(
232
+ f"{health.hidden_by_forgetting:,} older memories are set aside by "
233
+ f"the forgetting curve. They are not deleted and a deep search "
234
+ f"still reaches them."
235
+ )
236
+
237
+ return lines
238
+
239
+
240
+ def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
241
+ return conn.execute(
242
+ "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,),
243
+ ).fetchone() is not None
244
+
245
+
246
+ def _has_column(conn: sqlite3.Connection, table: str, column: str) -> bool:
247
+ return any(
248
+ row[1] == column for row in conn.execute(f"PRAGMA table_info({table})")
249
+ )
250
+
251
+
252
+ def _prefixed(clause: str, prefix: str) -> str:
253
+ """Qualify a bare column reference for use in a joined query.
254
+
255
+ Word-bounded, so a future column named ``quarantined_at`` is not silently
256
+ rewritten to ``af.quarantined_at`` by a substring match. No such column
257
+ exists today; the point is that the failure would be a wrong count rather
258
+ than an error, and a wrong count in a health report is the one thing this
259
+ module must not produce.
260
+ """
261
+ return re.sub(r"\bquarantined\b", f"{prefix}.quarantined", clause)
262
+
263
+
264
+ def _count(conn: sqlite3.Connection, sql: str) -> int:
265
+ row = conn.execute(sql).fetchone()
266
+ return int(row[0]) if row else 0
@@ -0,0 +1,111 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later - see LICENSE file
3
+ # Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
4
+
5
+ """What the running mode can do with a model, in words a user can act on.
6
+
7
+ WHY THIS EXISTS
8
+
9
+ Mode A runs with no language model at all. That is the point of it: nothing on
10
+ the store or recall path calls out to anything, and a summary is assembled
11
+ directly from the user's own notes rather than written.
12
+
13
+ The surfaces already reported *what* they did -- a summary came back labelled
14
+ "assembled directly from your own notes" -- but never *why*, and never what to do
15
+ about it. Someone looking at a plainer summary than they expected had no way to
16
+ learn that a written one needs a model and which modes have one. They would
17
+ reasonably conclude the feature was broken.
18
+
19
+ So this returns the mode, whether a model is available, and one sentence naming
20
+ the next step. Every model-backed surface returns the same block, so the
21
+ explanation is identical wherever it appears rather than reworded per pane.
22
+
23
+ WHAT IT IS NOT
24
+
25
+ It is not a capability gate. Nothing here decides whether a call is made; the
26
+ mode's configuration does that. This only describes the decision to the person
27
+ looking at the result.
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import logging
33
+ from typing import Any
34
+
35
+ logger = logging.getLogger(__name__)
36
+
37
+ #: Named so the message can say which modes to consider without hard-coding the
38
+ #: sentence at each call site.
39
+ _MODE_WITH_LOCAL_MODEL = "B"
40
+ _MODE_WITH_CLOUD_MODEL = "C"
41
+
42
+ _NO_MODEL_MESSAGE = (
43
+ "This mode runs entirely on your machine with no language model, so "
44
+ "summaries are assembled from your own notes rather than written. For "
45
+ f"written summaries and the other model-backed features, switch to Mode "
46
+ f"{_MODE_WITH_LOCAL_MODEL} (a local model) or Mode {_MODE_WITH_CLOUD_MODEL} "
47
+ "(your own cloud model) in Settings and connect one."
48
+ )
49
+
50
+ _MODEL_CONFIGURED_BUT_ABSENT = (
51
+ "This mode uses a language model, but none is reachable right now, so "
52
+ "results fall back to being assembled from your own notes. Check the model "
53
+ "settings, and that the local model server is running if you are using one."
54
+ )
55
+
56
+ _CLOUD_MODEL_HAS_NO_KEY = (
57
+ "This mode uses a hosted language model and no key has been set for it, so "
58
+ "summaries are assembled from your own notes rather than written. Run "
59
+ "`slm provider set` and supply your key, or switch to Mode "
60
+ f"{_MODE_WITH_LOCAL_MODEL} to use a model on this machine."
61
+ )
62
+
63
+ #: Providers that run on the machine and need no credential. Anything else is
64
+ #: a hosted service, and naming one without a key is not a configured model --
65
+ #: which is what this used to report, so the surfaces said everything was fine
66
+ #: while every model-backed feature was about to fall back.
67
+ _LOCAL_PROVIDERS = frozenset({
68
+ "ollama", "lmstudio", "llamacpp", "llama.cpp", "local", "vllm", "none", "",
69
+ })
70
+
71
+
72
+ def llm_capability(config: Any, *, llm_reachable: bool | None = None) -> dict:
73
+ """Describe this mode's model support for a user-facing surface.
74
+
75
+ ``llm_reachable`` lets a caller that already knows the answer pass it in --
76
+ a summary route has just tried and knows whether it worked, and asking again
77
+ would mean a second connection attempt to say something it already knows.
78
+ Left as None, availability is taken from configuration alone.
79
+ """
80
+ mode = ""
81
+ provider = ""
82
+ api_key = ""
83
+ try:
84
+ mode = str(getattr(getattr(config, "mode", None), "value", "") or "").upper()
85
+ llm = getattr(config, "llm", None)
86
+ provider = str(getattr(llm, "provider", "") or "")
87
+ api_key = str(getattr(llm, "api_key", "") or "").strip()
88
+ except Exception as exc: # noqa: BLE001 -- a description must not raise
89
+ logger.debug("mode capability: cannot read config: %s", exc)
90
+
91
+ mode_has_model = mode in (_MODE_WITH_LOCAL_MODEL, _MODE_WITH_CLOUD_MODEL)
92
+ needs_a_key = provider.lower() not in _LOCAL_PROVIDERS
93
+ missing_key = mode_has_model and bool(provider) and needs_a_key and not api_key
94
+ configured = bool(provider) and mode_has_model and not missing_key
95
+ available = configured if llm_reachable is None else bool(llm_reachable)
96
+
97
+ if not mode_has_model:
98
+ message = _NO_MODEL_MESSAGE
99
+ elif missing_key and llm_reachable is not True:
100
+ message = _CLOUD_MODEL_HAS_NO_KEY
101
+ elif not available:
102
+ message = _MODEL_CONFIGURED_BUT_ABSENT
103
+ else:
104
+ message = ""
105
+
106
+ return {
107
+ "mode": mode or "?",
108
+ "llm_available": bool(available),
109
+ "llm_provider": provider,
110
+ "message": message,
111
+ }
@@ -0,0 +1,315 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later - see LICENSE file
3
+ # Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
4
+
5
+ """Ask the local model server whether a model exists, before trusting it.
6
+
7
+ Mode B runs on Ollama, and a user picking a model there picks it by typing a
8
+ name. Two things then go wrong silently.
9
+
10
+ **The name is wrong.** Nothing happens at the moment of the mistake; the config
11
+ saves, the daemon starts, and every write quietly falls back to a worse path.
12
+ The error a user eventually sees is unrelated to what they did.
13
+
14
+ **The name is right and the shape is different.** Every embedding model emits a
15
+ vector of a fixed width. `nomic-embed-text` emits 768 numbers,
16
+ `mxbai-embed-large` emits 1024. Vectors of different widths cannot be compared,
17
+ so a store holding both answers similarity questions with noise — and it does so
18
+ without failing, because nothing in a similarity search knows the difference
19
+ between a bad answer and a good one. **This is the single silent-and-catastrophic
20
+ change a user can make**, so it is refused rather than warned about, and the
21
+ refusal says the one command that would make it safe.
22
+
23
+ There are two Ollama roles and they are separate models with separate names:
24
+ the embedding model (`embedding.ollama_model`) turns text into vectors, and the
25
+ generation model (`llm.model` when the provider is Ollama) writes summaries.
26
+ They are validated independently because a machine can perfectly well have one
27
+ and not the other.
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import logging
33
+ import sqlite3
34
+ from dataclasses import dataclass
35
+ from pathlib import Path
36
+
37
+ __all__ = [
38
+ "EMBEDDING",
39
+ "GENERATION",
40
+ "OllamaProbe",
41
+ "DimensionChange",
42
+ "validate_ollama_model",
43
+ "stored_embedding_dimension",
44
+ "check_embedding_model_change",
45
+ "same_embedding_model",
46
+ ]
47
+
48
+ logger = logging.getLogger(__name__)
49
+
50
+ EMBEDDING = "embedding"
51
+ GENERATION = "generation"
52
+
53
+ DEFAULT_BASE_URL = "http://localhost:11434"
54
+
55
+ #: Long enough for a cold model load, short enough that a wedged server is not
56
+ #: mistaken for a slow one.
57
+ _CONNECT_TIMEOUT = 3.0
58
+ _RESPONSE_TIMEOUT = 60.0
59
+
60
+ _PROBE_TEXT = "superlocalmemory model probe"
61
+ _PROBE_PROMPT = "Reply with the single word: ok"
62
+
63
+
64
+ @dataclass(frozen=True)
65
+ class OllamaProbe:
66
+ """What the server said when asked about one model."""
67
+
68
+ ok: bool
69
+ message: str
70
+ dimension: int | None = None
71
+
72
+ def __bool__(self) -> bool: # pragma: no cover - convenience only
73
+ return self.ok
74
+
75
+
76
+ @dataclass(frozen=True)
77
+ class DimensionChange:
78
+ """A model change that would put two vector widths in one store."""
79
+
80
+ allowed: bool
81
+ message: str
82
+ stored_dimension: int | None = None
83
+ new_dimension: int | None = None
84
+
85
+
86
+ def same_embedding_model(left: str, right: str) -> bool:
87
+ """True when two names are two spellings of one model.
88
+
89
+ ``nomic-ai/nomic-embed-text-v1.5`` is the HuggingFace name and
90
+ ``nomic-embed-text`` is the Ollama pull name for the same weights. Warning
91
+ about a dimension change between them would be a false alarm, and a false
92
+ alarm about the one thing that is genuinely dangerous is how a real warning
93
+ gets ignored.
94
+ """
95
+ def base(name: str) -> str:
96
+ stem = name.strip().lower().rsplit("/", 1)[-1]
97
+ # An Ollama tag is PART OF THE IDENTITY, not packaging.
98
+ # ``qwen3-embedding:0.6b`` emits 1024 numbers and ``qwen3-embedding:8b``
99
+ # emits 4096, and stripping the tag made this function call them the
100
+ # same model — which then skipped the width check that exists to stop
101
+ # exactly that swap. Only ``:latest`` means "whatever the bare name
102
+ # means", so only that one is dropped.
103
+ if stem.endswith(":latest"):
104
+ stem = stem[: -len(":latest")]
105
+ # A HuggingFace revision suffix on an otherwise identical name is
106
+ # packaging; ``nomic-embed-text-v1.5`` and ``nomic-embed-text`` are the
107
+ # same weights under two registries' conventions.
108
+ if ":" not in stem:
109
+ for suffix in ("-v1.5", "-v1", "-v2"):
110
+ if stem.endswith(suffix):
111
+ stem = stem[: -len(suffix)]
112
+ return stem
113
+
114
+ return base(left) == base(right)
115
+
116
+
117
+ def validate_ollama_model(
118
+ model_name: str,
119
+ role: str = EMBEDDING,
120
+ *,
121
+ base_url: str = DEFAULT_BASE_URL,
122
+ timeout: float = _RESPONSE_TIMEOUT,
123
+ ) -> OllamaProbe:
124
+ """Ask the server to actually use the model, and report what happened.
125
+
126
+ Listing the installed models is not enough: a name can be present and the
127
+ model still fail to load. So this runs the smallest real request for the
128
+ role and reads the answer, which is also the only way to learn an embedding
129
+ model's width.
130
+ """
131
+ if role not in (EMBEDDING, GENERATION):
132
+ raise ValueError(f"role must be {EMBEDDING!r} or {GENERATION!r}, got {role!r}")
133
+ name = (model_name or "").strip()
134
+ if not name:
135
+ return OllamaProbe(False, "No model name given.")
136
+
137
+ try:
138
+ import httpx
139
+ except ImportError: # pragma: no cover - httpx is a hard dependency
140
+ return OllamaProbe(False, "httpx is not installed, so Ollama cannot be reached.")
141
+
142
+ url = base_url.rstrip("/")
143
+ request_timeout = httpx.Timeout(timeout, connect=_CONNECT_TIMEOUT)
144
+
145
+ try:
146
+ if role == EMBEDDING:
147
+ response = httpx.post(
148
+ f"{url}/api/embed",
149
+ json={"model": name, "input": [_PROBE_TEXT]},
150
+ timeout=request_timeout,
151
+ )
152
+ else:
153
+ response = httpx.post(
154
+ f"{url}/api/generate",
155
+ json={"model": name, "prompt": _PROBE_PROMPT, "stream": False},
156
+ timeout=request_timeout,
157
+ )
158
+ except httpx.ConnectError:
159
+ return OllamaProbe(
160
+ False,
161
+ f"Ollama is not running at {url}. Start it with: ollama serve",
162
+ )
163
+ except httpx.TimeoutException:
164
+ return OllamaProbe(
165
+ False,
166
+ f"Ollama did not answer within {timeout:.0f}s. The model may still be "
167
+ f"downloading — check with: ollama list",
168
+ )
169
+ except Exception as exc: # noqa: BLE001 - the message is the product here
170
+ return OllamaProbe(False, f"Could not reach Ollama at {url}: {exc}")
171
+
172
+ if response.status_code == 404 or _looks_missing(response):
173
+ return OllamaProbe(
174
+ False,
175
+ f"Ollama has no model called {name!r}. Run: ollama pull {name}",
176
+ )
177
+ if role == EMBEDDING and _refuses_embeddings(response):
178
+ return OllamaProbe(
179
+ False,
180
+ f"{name!r} is not an embedding model — the server refused the "
181
+ f"request. Pick a model built for embeddings, such as "
182
+ f"nomic-embed-text.",
183
+ )
184
+ if response.status_code != 200:
185
+ return OllamaProbe(
186
+ False,
187
+ f"Ollama answered {response.status_code} for {name!r}: "
188
+ f"{response.text[:200]}",
189
+ )
190
+
191
+ try:
192
+ payload = response.json()
193
+ except ValueError:
194
+ return OllamaProbe(False, f"Ollama returned something that is not JSON for {name!r}.")
195
+
196
+ if role == EMBEDDING:
197
+ vectors = payload.get("embeddings") or []
198
+ if not vectors or not isinstance(vectors[0], list) or not vectors[0]:
199
+ return OllamaProbe(
200
+ False,
201
+ f"{name!r} answered but returned no vector, so it is not an "
202
+ f"embedding model. Pick one built for embeddings, such as "
203
+ f"nomic-embed-text.",
204
+ )
205
+ width = len(vectors[0])
206
+ return OllamaProbe(True, f"{name} emits {width}-dimensional vectors.", width)
207
+
208
+ text = str(payload.get("response") or "").strip()
209
+ if not text:
210
+ return OllamaProbe(
211
+ False,
212
+ f"{name!r} answered but wrote nothing, so it cannot be used to "
213
+ f"generate summaries.",
214
+ )
215
+ return OllamaProbe(True, f"{name} answered a probe prompt.")
216
+
217
+
218
+ def _refuses_embeddings(response: object) -> bool:
219
+ """A generation-only model answers an embedding request with a refusal.
220
+
221
+ The server's own wording is about its build flags and means nothing to
222
+ somebody who typed a chat model's name into an embedding field.
223
+ """
224
+ status = getattr(response, "status_code", 0)
225
+ try:
226
+ lowered = response.text.lower() # type: ignore[attr-defined]
227
+ except Exception: # pragma: no cover - defensive
228
+ lowered = ""
229
+ if status == 501:
230
+ return True
231
+ return "does not support embed" in lowered or "not support embeddings" in lowered
232
+
233
+
234
+ def _looks_missing(response: object) -> bool:
235
+ """Ollama reports an unknown model as a 400 with a body that says so."""
236
+ try:
237
+ body = response.text # type: ignore[attr-defined]
238
+ except Exception: # pragma: no cover - defensive
239
+ return False
240
+ lowered = body.lower()
241
+ return "not found" in lowered and "model" in lowered
242
+
243
+
244
+ def stored_embedding_dimension(db_path: str | Path) -> int | None:
245
+ """The vector width this store already holds, or None if it holds none."""
246
+ path = Path(db_path)
247
+ if not path.exists():
248
+ return None
249
+ try:
250
+ conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
251
+ except sqlite3.Error:
252
+ return None
253
+ try:
254
+ row = conn.execute(
255
+ "SELECT dimension FROM embedding_metadata "
256
+ "WHERE dimension IS NOT NULL AND dimension > 0 LIMIT 1"
257
+ ).fetchone()
258
+ except sqlite3.Error:
259
+ return None
260
+ finally:
261
+ conn.close()
262
+ return int(row[0]) if row else None
263
+
264
+
265
+ def check_embedding_model_change(
266
+ new_model: str,
267
+ *,
268
+ db_path: str | Path,
269
+ current_model: str = "",
270
+ base_url: str = DEFAULT_BASE_URL,
271
+ ) -> DimensionChange:
272
+ """Decide whether switching the embedding model is safe for this store.
273
+
274
+ Refuses rather than warns. A warning printed into a log is not a decision,
275
+ and the outcome of getting this wrong is a store whose similarity search is
276
+ quietly meaningless.
277
+ """
278
+ # The width is asked for FIRST, always. Recognising two names as the same
279
+ # model may soften the message, but it must never stand in for measuring —
280
+ # a name that merely looks familiar is exactly how a different-width model
281
+ # would get through.
282
+ probe = validate_ollama_model(new_model, EMBEDDING, base_url=base_url)
283
+ if not probe.ok:
284
+ return DimensionChange(False, probe.message)
285
+
286
+ familiar = bool(current_model) and same_embedding_model(current_model, new_model)
287
+ stored = stored_embedding_dimension(db_path)
288
+ if familiar and (stored is None or stored == probe.dimension):
289
+ return DimensionChange(
290
+ True,
291
+ f"{new_model} is another name for the model already in use, and it "
292
+ f"emits the same {probe.dimension}-dimensional vectors.",
293
+ stored,
294
+ probe.dimension,
295
+ )
296
+ if stored is None:
297
+ return DimensionChange(
298
+ True,
299
+ f"{probe.message} This store holds no vectors yet, so nothing has to "
300
+ f"be rebuilt.",
301
+ None,
302
+ probe.dimension,
303
+ )
304
+ if stored == probe.dimension:
305
+ return DimensionChange(True, probe.message, stored, probe.dimension)
306
+
307
+ return DimensionChange(
308
+ False,
309
+ f"{new_model} emits {probe.dimension}-dimensional vectors and this store "
310
+ f"holds {stored}-dimensional ones. Vectors of different widths cannot be "
311
+ f"compared, so every memory already stored would become unfindable by "
312
+ f"meaning. Rebuild them first with: slm db migrate",
313
+ stored,
314
+ probe.dimension,
315
+ )