superlocalmemory 4.0.10 → 4.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/.claude-plugin/marketplace.json +12 -2
  2. package/CHANGELOG.md +244 -0
  3. package/README.md +40 -75
  4. package/package.json +6 -3
  5. package/plugin/.claude-plugin/plugin.json +2 -2
  6. package/plugin/CLAUDE.md +3 -3
  7. package/plugin/agents/slm-governance-advisor.md +1 -1
  8. package/plugin/agents/slm-loop-runner.md +4 -4
  9. package/plugin/agents/slm-memory-advisor.md +1 -1
  10. package/plugin/agents/slm-optimize-advisor.md +1 -1
  11. package/plugin/requirements.txt +1 -1
  12. package/plugin/skills/slm-cache/SKILL.md +1 -1
  13. package/plugin/skills/slm-compress/SKILL.md +1 -1
  14. package/plugin/skills/slm-governance/SKILL.md +1 -1
  15. package/plugin/skills/slm-graph/SKILL.md +1 -1
  16. package/plugin/skills/slm-loop/SKILL.md +2 -2
  17. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  18. package/plugin/skills/slm-profile/SKILL.md +5 -5
  19. package/plugin/skills/slm-recall/SKILL.md +102 -15
  20. package/plugin/skills/slm-remember/SKILL.md +35 -3
  21. package/plugin/skills/slm-scope/SKILL.md +1 -1
  22. package/plugin/skills/slm-session/SKILL.md +29 -3
  23. package/plugin/skills/slm-status/SKILL.md +1 -1
  24. package/plugin-src/rules/AGENTS.md +16 -8
  25. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  26. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-governance/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  29. package/plugin-src/skills/slm-loop/SKILL.md +2 -2
  30. package/plugin-src/skills/slm-mesh/SKILL.md +1 -1
  31. package/plugin-src/skills/slm-profile/SKILL.md +5 -5
  32. package/plugin-src/skills/slm-recall/SKILL.md +102 -15
  33. package/plugin-src/skills/slm-remember/SKILL.md +35 -3
  34. package/plugin-src/skills/slm-scope/SKILL.md +1 -1
  35. package/plugin-src/skills/slm-session/SKILL.md +29 -3
  36. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  37. package/pyproject.toml +1 -1
  38. package/src/superlocalmemory/__init__.py +1 -1
  39. package/src/superlocalmemory/cli/commands.py +357 -18
  40. package/src/superlocalmemory/cli/daemon.py +30 -0
  41. package/src/superlocalmemory/cli/db_migrate.py +71 -1
  42. package/src/superlocalmemory/cli/gdpr_cmd.py +15 -2
  43. package/src/superlocalmemory/cli/main.py +24 -2
  44. package/src/superlocalmemory/code_graph/database.py +44 -0
  45. package/src/superlocalmemory/compliance/gdpr.py +449 -39
  46. package/src/superlocalmemory/core/admission.py +231 -11
  47. package/src/superlocalmemory/core/backend_orchestrator.py +190 -84
  48. package/src/superlocalmemory/core/config.py +90 -11
  49. package/src/superlocalmemory/core/consolidation_engine.py +34 -0
  50. package/src/superlocalmemory/core/engine.py +140 -11
  51. package/src/superlocalmemory/core/graph_analyzer.py +76 -112
  52. package/src/superlocalmemory/core/graph_metrics.py +597 -0
  53. package/src/superlocalmemory/core/graph_pruner.py +121 -0
  54. package/src/superlocalmemory/core/maintenance_scheduler.py +205 -0
  55. package/src/superlocalmemory/core/mode_capability.py +111 -0
  56. package/src/superlocalmemory/core/ollama_validator.py +315 -0
  57. package/src/superlocalmemory/core/projection_drain.py +380 -0
  58. package/src/superlocalmemory/core/recall_pipeline.py +390 -3
  59. package/src/superlocalmemory/core/recall_worker.py +6 -3
  60. package/src/superlocalmemory/core/scale_autopromote.py +196 -0
  61. package/src/superlocalmemory/core/scale_engine.py +16 -2
  62. package/src/superlocalmemory/core/score_contract.py +21 -1
  63. package/src/superlocalmemory/core/session_identity.py +85 -0
  64. package/src/superlocalmemory/core/status_contract.py +108 -0
  65. package/src/superlocalmemory/core/worker_pool.py +4 -4
  66. package/src/superlocalmemory/core/working_memory.py +288 -0
  67. package/src/superlocalmemory/encoding/cognitive_consolidator.py +36 -6
  68. package/src/superlocalmemory/encoding/context_generator.py +1 -1
  69. package/src/superlocalmemory/encoding/entity_resolver.py +38 -0
  70. package/src/superlocalmemory/encoding/fact_extractor.py +18 -14
  71. package/src/superlocalmemory/encoding/prospective_markers.py +262 -0
  72. package/src/superlocalmemory/encoding/type_router.py +12 -12
  73. package/src/superlocalmemory/evolution/mutation_generator.py +30 -4
  74. package/src/superlocalmemory/graph/cozo_adjacency.py +122 -0
  75. package/src/superlocalmemory/graph/cozo_backend.py +103 -138
  76. package/src/superlocalmemory/hooks/portable_kit.py +10 -2
  77. package/src/superlocalmemory/learning/bandit.py +43 -0
  78. package/src/superlocalmemory/learning/consolidation_worker.py +54 -0
  79. package/src/superlocalmemory/learning/database.py +60 -3
  80. package/src/superlocalmemory/learning/entity_compiler.py +21 -58
  81. package/src/superlocalmemory/learning/feedback.py +3 -1
  82. package/src/superlocalmemory/learning/outcomes.py +47 -16
  83. package/src/superlocalmemory/learning/pattern_miner.py +28 -3
  84. package/src/superlocalmemory/learning/pattern_miner_constants.py +43 -0
  85. package/src/superlocalmemory/learning/pcos.py +291 -0
  86. package/src/superlocalmemory/learning/reward_from_outcomes.py +365 -0
  87. package/src/superlocalmemory/learning/reward_proxy.py +100 -10
  88. package/src/superlocalmemory/learning/signal_kinds.py +79 -0
  89. package/src/superlocalmemory/mcp/profiles.py +14 -2
  90. package/src/superlocalmemory/mcp/tools_active.py +2 -1
  91. package/src/superlocalmemory/mcp/tools_core.py +31 -3
  92. package/src/superlocalmemory/mcp/tools_v28.py +20 -1
  93. package/src/superlocalmemory/parameterization/pattern_extractor.py +14 -1
  94. package/src/superlocalmemory/parameterization/soft_prompt_generator.py +98 -0
  95. package/src/superlocalmemory/retrieval/bm25_channel.py +64 -3
  96. package/src/superlocalmemory/retrieval/channel_status.py +117 -0
  97. package/src/superlocalmemory/retrieval/engine.py +106 -11
  98. package/src/superlocalmemory/retrieval/entity_channel.py +210 -256
  99. package/src/superlocalmemory/retrieval/graph_adjacency.py +219 -0
  100. package/src/superlocalmemory/retrieval/scope_policy.py +20 -0
  101. package/src/superlocalmemory/retrieval/semantic_channel.py +47 -5
  102. package/src/superlocalmemory/retrieval/spreading.py +288 -0
  103. package/src/superlocalmemory/server/api.py +24 -5
  104. package/src/superlocalmemory/server/bandit_loops.py +17 -1
  105. package/src/superlocalmemory/server/rbac_enforce.py +26 -6
  106. package/src/superlocalmemory/server/recall_health.py +87 -10
  107. package/src/superlocalmemory/server/recall_serializer.py +9 -0
  108. package/src/superlocalmemory/server/routes/behavioral.py +75 -10
  109. package/src/superlocalmemory/server/routes/compliance.py +98 -18
  110. package/src/superlocalmemory/server/routes/config_api.py +186 -4
  111. package/src/superlocalmemory/server/routes/evolution.py +178 -0
  112. package/src/superlocalmemory/server/routes/ingest.py +8 -0
  113. package/src/superlocalmemory/server/routes/learning_telemetry.py +2 -1
  114. package/src/superlocalmemory/server/routes/memories.py +49 -7
  115. package/src/superlocalmemory/server/routes/timeline.py +4 -0
  116. package/src/superlocalmemory/server/routes/v3_api.py +191 -15
  117. package/src/superlocalmemory/server/ui.py +20 -4
  118. package/src/superlocalmemory/server/unified_daemon.py +241 -7
  119. package/src/superlocalmemory/storage/_migration_internals.py +54 -2
  120. package/src/superlocalmemory/storage/_schema_version.py +24 -3
  121. package/src/superlocalmemory/storage/database.py +477 -59
  122. package/src/superlocalmemory/storage/embedding_codec.py +71 -0
  123. package/src/superlocalmemory/storage/lineage_retention.py +236 -0
  124. package/src/superlocalmemory/storage/logical_edges.py +43 -2
  125. package/src/superlocalmemory/storage/migration_runner.py +119 -0
  126. package/src/superlocalmemory/storage/migrations/M043_quarantine_display_summaries.py +60 -36
  127. package/src/superlocalmemory/storage/migrations/M044_play_carries_its_own_evidence.py +127 -0
  128. package/src/superlocalmemory/storage/migrations/M045_fact_outcome_score.py +158 -0
  129. package/src/superlocalmemory/storage/migrations/M046_prospective_memory_has_its_own_name.py +620 -0
  130. package/src/superlocalmemory/storage/migrations/M047_fisher_vectors_are_stored_like_every_other_vector.py +306 -0
  131. package/src/superlocalmemory/storage/migrations/M048_upcoming_holds_only_what_is_upcoming.py +207 -0
  132. package/src/superlocalmemory/storage/migrations/M049_a_schema_version_marker_is_one_row.py +201 -0
  133. package/src/superlocalmemory/storage/migrations.py +18 -2
  134. package/src/superlocalmemory/storage/models.py +40 -1
  135. package/src/superlocalmemory/storage/projection_outbox.py +346 -0
  136. package/src/superlocalmemory/storage/retention_policy.py +860 -0
  137. package/src/superlocalmemory/storage/schema.py +35 -1
  138. package/src/superlocalmemory/storage/write_coordinator.py +19 -2
  139. package/src/superlocalmemory/trust/scorer.py +43 -1
  140. package/src/superlocalmemory/ui/index.html +9 -18
  141. package/src/superlocalmemory/ui/js/event-delegation.js +12 -1
  142. package/src/superlocalmemory/ui/js/od-health.js +28 -6
  143. package/src/superlocalmemory/ui/js/od-memories.js +19 -0
  144. package/src/superlocalmemory/ui/js/od-settings.js +87 -1
  145. package/src/superlocalmemory/ui/js/recall-lab.js +78 -3
@@ -37,6 +37,8 @@ __all__ = [
37
37
  "EMBEDDING_BYTES",
38
38
  "encode_embedding",
39
39
  "decode_embedding",
40
+ "encode_float_vector",
41
+ "decode_float_vector",
40
42
  ]
41
43
 
42
44
  EMBEDDING_DIM: int = 768
@@ -127,3 +129,72 @@ def decode_embedding(
127
129
  f"Unexpected embedding type {type(raw).__name__!r} for fact {fact_id!r}; "
128
130
  f"expected bytes or str"
129
131
  )
132
+
133
+
134
+ # ---------------------------------------------------------------------------
135
+ # The same treatment for the other float vectors stored on a fact
136
+ # ---------------------------------------------------------------------------
137
+ #
138
+ # A fact carries two more 768-wide vectors besides its embedding: the diagonal
139
+ # Fisher mean and variance that the memory dynamics read. They were written as
140
+ # JSON text, which costs about 17 KB each against 3 KB for the same numbers as
141
+ # float32. Measured on a real 447 MB store: 116.5 MB of Fisher text describing
142
+ # 3.6 MB of memories — thirty-two times the size of the content itself, and
143
+ # more than a quarter of the whole file.
144
+ #
145
+ # The read path accepts both forms for the same reason the embedding one does:
146
+ # a store converts when a migration reaches it, and everything has to keep
147
+ # working in the meantime.
148
+
149
+
150
+ def encode_float_vector(vec: list[float] | None) -> bytes | None:
151
+ """Serialise any float vector to a little-endian float32 BLOB."""
152
+ if vec is None:
153
+ return None
154
+ return np.asarray(vec, dtype=np.float32).tobytes()
155
+
156
+
157
+ def decode_float_vector(
158
+ raw: bytes | str | None,
159
+ *,
160
+ field: str = "vector",
161
+ fact_id: str = "<unknown>",
162
+ ) -> list[float] | None:
163
+ """Read a float vector written as either JSON text or a float32 BLOB.
164
+
165
+ Raises rather than returning ``None`` for a malformed value: a caller
166
+ cannot tell a legitimately absent vector from a lost one, and these feed
167
+ the decay dynamics, where a silently empty vector reads as "no evidence"
168
+ instead of "evidence missing".
169
+ """
170
+ if raw is None or raw == "":
171
+ return None
172
+
173
+ if isinstance(raw, (bytes, bytearray)):
174
+ if len(raw) == 0 or len(raw) % 4 != 0:
175
+ raise ValueError(
176
+ f"Corrupt {field} buffer for fact {fact_id!r}: {len(raw)} bytes "
177
+ f"is not a multiple of 4 (float32)"
178
+ )
179
+ return np.frombuffer(raw, dtype=np.float32).tolist()
180
+
181
+ if isinstance(raw, str):
182
+ try:
183
+ value = json.loads(raw)
184
+ except (json.JSONDecodeError, ValueError) as exc:
185
+ raise ValueError(
186
+ f"Corrupt JSON {field} for fact {fact_id!r}: {exc}"
187
+ ) from exc
188
+ if value is None:
189
+ return None
190
+ if not isinstance(value, list):
191
+ raise ValueError(
192
+ f"{field} for fact {fact_id!r} decoded to "
193
+ f"{type(value).__name__}, expected a list"
194
+ )
195
+ return [float(v) for v in value]
196
+
197
+ raise ValueError(
198
+ f"Unexpected {field} type {type(raw).__name__!r} for fact {fact_id!r}; "
199
+ f"expected bytes or str"
200
+ )
@@ -0,0 +1,236 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later - see LICENSE file
3
+ # Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
4
+
5
+ """Forget the provenance of things that no longer exist.
6
+
7
+ `derivation_lineage` records where each derived object came from — which source
8
+ span produced this fact, this graph edge, this scene. Every ingestion operation
9
+ re-captures lineage for the objects present at that moment, so the table grows
10
+ with use, and nothing has ever deleted from it.
11
+
12
+ Measured on a real 447 MB store:
13
+
14
+ derivation_lineage 256,885 rows, 64 MB, plus 65 MB of indexes
15
+ ... describing a graph edge
16
+ that no longer exists 100,581 rows — 39.2% of the table
17
+ growth ~9,000 rows/day, sustained
18
+
19
+ The graph is pruned. Its lineage was not, so the record of how a deleted edge
20
+ came to exist outlives the edge forever. Those rows answer no question: the
21
+ evidence bundle computes lineage coverage over the objects it exports, and an
22
+ object that is gone is not exported.
23
+
24
+ WHAT IS AND IS NOT DELETED
25
+
26
+ Only a row whose object is provably absent — the type is one this module knows
27
+ how to resolve, the table exists, and no row with that id is in it. An
28
+ unrecognised object type is left alone, because "I do not know what this
29
+ describes" is not evidence that it describes nothing.
30
+
31
+ There is deliberately no age rule. Lineage is what an audit reads to answer
32
+ "where did this come from", and a fact can be years old and still current.
33
+ Deleting provenance for something that still exists would be destroying the
34
+ answer while keeping the question.
35
+ """
36
+
37
+ from __future__ import annotations
38
+
39
+ import logging
40
+ import sqlite3
41
+ from dataclasses import dataclass, field
42
+
43
+ logger = logging.getLogger(__name__)
44
+
45
+ __all__ = ["OBJECT_SOURCES", "LineagePruneReport", "count_orphan_lineage",
46
+ "prune_orphan_lineage"]
47
+
48
+ #: object_type -> (table holding it, column carrying its id).
49
+ #: Mirrors what ``core/derivation_lineage.capture_operation_lineage`` writes.
50
+ #: A type absent from this map is never deleted.
51
+ OBJECT_SOURCES: dict[str, tuple[str, str]] = {
52
+ "fact": ("atomic_facts", "fact_id"),
53
+ "graph_edge": ("graph_edges", "edge_id"),
54
+ "memory_scene": ("memory_scenes", "scene_id"),
55
+ "entity_summary": ("entity_profiles", "profile_entry_id"),
56
+ "index_bm25": ("bm25_tokens", "fact_id"),
57
+ "profile": ("profiles", "profile_id"),
58
+ }
59
+
60
+ #: Rows per transaction. Big enough that commit overhead vanishes, small enough
61
+ #: that an interrupted run has done most of its work and holds no long lock.
62
+ _BATCH = 2_000
63
+
64
+
65
+ class _Rows:
66
+ """Read and write through either a raw connection or the DatabaseManager.
67
+
68
+ The maintenance cycle holds a ``DatabaseManager``, whose lock serialises
69
+ every write; a migration or a test holds a plain ``sqlite3.Connection``.
70
+ Both are legitimate callers, and the difference is two method names.
71
+ """
72
+
73
+ def __init__(self, db: object) -> None:
74
+ self._db = db
75
+ self._managed = hasattr(db, "transaction") and not isinstance(
76
+ db, sqlite3.Connection
77
+ )
78
+
79
+ def query(self, sql: str, params: tuple = ()) -> list:
80
+ rows = self._db.execute(sql, tuple(params))
81
+ return list(rows) if self._managed else rows.fetchall()
82
+
83
+ def write(self, sql: str, params: tuple = ()) -> None:
84
+ if self._managed:
85
+ with self._db.transaction():
86
+ self._db.execute(sql, tuple(params))
87
+ return
88
+ self._db.execute("BEGIN IMMEDIATE")
89
+ try:
90
+ self._db.execute(sql, tuple(params))
91
+ self._db.commit()
92
+ except Exception:
93
+ self._db.rollback()
94
+ raise
95
+
96
+
97
+ @dataclass(frozen=True)
98
+ class LineagePruneReport:
99
+ """What was removed, by object type, and what was left alone."""
100
+
101
+ deleted: dict[str, int] = field(default_factory=dict)
102
+ skipped_types: tuple[str, ...] = ()
103
+
104
+ @property
105
+ def total(self) -> int:
106
+ return sum(self.deleted.values())
107
+
108
+
109
+ def _table_exists(rows: _Rows, table: str) -> bool:
110
+ return bool(rows.query(
111
+ "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
112
+ ))
113
+
114
+
115
+ def _resolvable(rows: _Rows) -> tuple[dict[str, tuple[str, str]], tuple[str, ...]]:
116
+ """Split the types present in the table into resolvable and not."""
117
+ present = {
118
+ str(r[0]) for r in rows.query(
119
+ "SELECT DISTINCT object_type FROM derivation_lineage"
120
+ )
121
+ }
122
+ resolvable: dict[str, tuple[str, str]] = {}
123
+ skipped: list[str] = []
124
+ for object_type in sorted(present):
125
+ source = OBJECT_SOURCES.get(object_type)
126
+ if source is None or not _table_exists(rows, source[0]):
127
+ skipped.append(object_type)
128
+ continue
129
+ resolvable[object_type] = source
130
+ return resolvable, tuple(skipped)
131
+
132
+
133
+ def count_orphan_lineage(
134
+ db: object, *, profile_id: str | None = None,
135
+ ) -> LineagePruneReport:
136
+ """How many rows describe something absent, without deleting anything."""
137
+ rows = _Rows(db)
138
+ if not _table_exists(rows, "derivation_lineage"):
139
+ return LineagePruneReport()
140
+
141
+ resolvable, skipped = _resolvable(rows)
142
+ counts: dict[str, int] = {}
143
+ for object_type, (table, column) in resolvable.items():
144
+ sql = (
145
+ "SELECT COUNT(*) FROM derivation_lineage d WHERE d.object_type = ? "
146
+ f"AND NOT EXISTS (SELECT 1 FROM {table} t WHERE t.{column} = d.object_id)"
147
+ )
148
+ params: list[object] = [object_type]
149
+ if profile_id is not None:
150
+ sql += " AND d.profile_id = ?"
151
+ params.append(profile_id)
152
+ count = int(rows.query(sql, tuple(params))[0][0])
153
+ if count:
154
+ counts[object_type] = count
155
+ return LineagePruneReport(counts, skipped)
156
+
157
+
158
+ def prune_orphan_lineage(
159
+ db: object,
160
+ *,
161
+ profile_id: str | None = None,
162
+ dry_run: bool = False,
163
+ ) -> LineagePruneReport:
164
+ """Delete lineage rows whose object is provably gone.
165
+
166
+ Batched and committed as it goes, so an interrupted run keeps the work it
167
+ already did and the next one continues. Nothing here reads a clock: what is
168
+ deleted depends only on what exists.
169
+ """
170
+ rows = _Rows(db)
171
+ if not _table_exists(rows, "derivation_lineage"):
172
+ return LineagePruneReport()
173
+
174
+ report = count_orphan_lineage(db, profile_id=profile_id)
175
+ if dry_run or not report.total:
176
+ return report
177
+
178
+ resolvable, skipped = _resolvable(rows)
179
+ deleted: dict[str, int] = {}
180
+
181
+ for object_type, (table, column) in resolvable.items():
182
+ if object_type not in report.deleted:
183
+ continue
184
+ removed = 0
185
+ while True:
186
+ sql = (
187
+ "SELECT lineage_id FROM derivation_lineage d "
188
+ "WHERE d.object_type = ? "
189
+ f"AND NOT EXISTS (SELECT 1 FROM {table} t WHERE t.{column} = d.object_id)"
190
+ )
191
+ params: list[object] = [object_type]
192
+ if profile_id is not None:
193
+ sql += " AND d.profile_id = ?"
194
+ params.append(profile_id)
195
+ sql += f" LIMIT {_BATCH}"
196
+
197
+ ids = [r[0] for r in rows.query(sql, tuple(params))]
198
+ if not ids:
199
+ break
200
+ placeholders = ",".join("?" * len(ids))
201
+ # Re-check absence in the DELETE itself. Selecting orphans and then
202
+ # deleting them by id is a check-then-act: another process can
203
+ # recreate the object between the two statements, and the row would
204
+ # be deleted anyway. Repeating the predicate makes the decision and
205
+ # the deletion one statement.
206
+ delete_sql = (
207
+ f"DELETE FROM derivation_lineage WHERE lineage_id IN ({placeholders}) "
208
+ f"AND NOT EXISTS (SELECT 1 FROM {table} t "
209
+ f"WHERE t.{column} = derivation_lineage.object_id)"
210
+ )
211
+ rows.write(delete_sql, tuple(ids))
212
+ # Count what survived the re-check rather than what was offered.
213
+ still = rows.query(
214
+ f"SELECT COUNT(*) FROM derivation_lineage "
215
+ f"WHERE lineage_id IN ({placeholders})",
216
+ tuple(ids),
217
+ )
218
+ remaining_here = int(still[0][0]) if still else 0
219
+ removed += len(ids) - remaining_here
220
+ if remaining_here == len(ids):
221
+ # Every row in this batch was rescued by a concurrent writer.
222
+ # Another pass would select the same rows forever.
223
+ break
224
+ if removed:
225
+ deleted[object_type] = removed
226
+ logger.info(
227
+ "lineage retention: removed %d row(s) describing a %s that no "
228
+ "longer exists", removed, object_type,
229
+ )
230
+
231
+ if skipped:
232
+ logger.info(
233
+ "lineage retention: left %s alone — no table is known to hold them",
234
+ ", ".join(skipped),
235
+ )
236
+ return LineagePruneReport(deleted, skipped)
@@ -20,17 +20,58 @@ _LOGICAL_EDGE_SELECT = """
20
20
  profile_id
21
21
  FROM graph_edges
22
22
  WHERE profile_id = ?
23
+ AND NOT EXISTS (
24
+ SELECT 1 FROM atomic_facts f
25
+ WHERE f.fact_id IN (graph_edges.source_id, graph_edges.target_id)
26
+ AND NOT ({visible})
27
+ )
23
28
  GROUP BY profile_id, source_id, target_id, COALESCE(edge_type, 'related')
24
29
  """
25
30
 
26
31
 
32
+ def _edge_select(conn: sqlite3.Connection) -> str:
33
+ """The logical-edge query, with the withheld-endpoint exclusion resolved.
34
+
35
+ WHY AN EDGE WITH A WITHHELD ENDPOINT IS NOT A LOGICAL EDGE
36
+ ----------------------------------------------------------
37
+ The retrieval channel this projection stands in for does not traverse them.
38
+ It loads ``graph_edges`` by scope and then prunes the result: "Edge scope
39
+ alone cannot authorize an endpoint. Prune both endpoints against the visible
40
+ fact corpus so denied facts cannot influence an allowed candidate indirectly
41
+ through propagation." Its entity map is filtered the same way, for a reason
42
+ it spells out — a withheld row carries its whole cluster's pooled entity
43
+ list, so it out-ranks real memories and then gets discarded at hydration,
44
+ spending the channel's budget on nothing.
45
+
46
+ The export predated that fix and kept the withheld endpoints. Measured on a
47
+ copy of the author's store: Cozo's bridge held 1,257 facts the store may not
48
+ return and its edges touched 805, and the graph search diverged from SQLite
49
+ on **every** query — three shadow checks, three mismatches. One query
50
+ returned 9 results against SQLite's 20, because withheld facts had taken the
51
+ top-k budget. The projection failed closed every time, so recall was correct
52
+ and the projection was dead weight.
53
+
54
+ The predicate is resolved against the passed connection because
55
+ ``archive_status`` and ``quarantined`` each arrive with a migration and may
56
+ be absent on a store the engine has not opened.
57
+ """
58
+ from superlocalmemory.storage.database import visible_fact_clause_for_connection
59
+
60
+ # The helper returns a leading-AND clause for appending; here it is needed as
61
+ # a standalone predicate, so strip the connective and default to "always
62
+ # visible" on a store that has neither column yet.
63
+ clause = visible_fact_clause_for_connection(conn, prefix="f").strip()
64
+ predicate = clause[4:].strip() if clause.upper().startswith("AND ") else clause
65
+ return _LOGICAL_EDGE_SELECT.format(visible=predicate or "1=1")
66
+
67
+
27
68
  def iter_logical_edges(
28
69
  conn: sqlite3.Connection, profile_id: str
29
70
  ) -> Iterator[tuple[Any, ...]]:
30
71
  """Yield normalized graph edges in deterministic fingerprint order."""
31
72
  return iter(
32
73
  conn.execute(
33
- _LOGICAL_EDGE_SELECT + " ORDER BY source_id, target_id, edge_type",
74
+ _edge_select(conn) + " ORDER BY source_id, target_id, edge_type",
34
75
  (profile_id,),
35
76
  )
36
77
  )
@@ -39,7 +80,7 @@ def iter_logical_edges(
39
80
  def count_logical_edges(conn: sqlite3.Connection, profile_id: str) -> int:
40
81
  """Count relationships using the canonical logical identity."""
41
82
  row = conn.execute(
42
- "SELECT COUNT(*) FROM (" + _LOGICAL_EDGE_SELECT + ")",
83
+ "SELECT COUNT(*) FROM (" + _edge_select(conn) + ")",
43
84
  (profile_id,),
44
85
  ).fetchone()
45
86
  return int(row[0] if row else 0)
@@ -159,6 +159,16 @@ from superlocalmemory.storage.migrations import (
159
159
  from superlocalmemory.storage.migrations import (
160
160
  M042_correction_case_ledger as _M042,
161
161
  )
162
+ from superlocalmemory.storage.migrations import (
163
+ M044_play_carries_its_own_evidence as _M044,
164
+ )
165
+ from superlocalmemory.storage.migrations import (
166
+ M045_fact_outcome_score as _M045,
167
+ M046_prospective_memory_has_its_own_name as _M046,
168
+ M047_fisher_vectors_are_stored_like_every_other_vector as _M047,
169
+ M048_upcoming_holds_only_what_is_upcoming as _M048,
170
+ M049_a_schema_version_marker_is_one_row as _M049,
171
+ )
162
172
  from superlocalmemory.storage.migrations import (
163
173
  M043_quarantine_display_summaries as _M043,
164
174
  )
@@ -167,6 +177,7 @@ from superlocalmemory.storage._schema_version import (
167
177
  SchemaVersionError,
168
178
  check_version_or_raise as _check_version_or_raise,
169
179
  ensure_schema_version_table as _ensure_schema_version_table,
180
+ read_schema_version as _read_schema_version,
170
181
  write_schema_version as _write_schema_version,
171
182
  )
172
183
  from superlocalmemory.storage._migration_internals import (
@@ -252,6 +263,13 @@ MIGRATIONS: list[Migration] = [
252
263
  # contains identifiers only and does not alter temporal fact state.
253
264
  Migration(name=_M042.NAME, db_target="memory", ddl=_M042.DDL,
254
265
  dependencies=(_M032.NAME,)),
266
+ # M044 lets a bandit play record which memories it showed, so the reward
267
+ # proxy can settle it from evidence instead of always falling through to
268
+ # the 120-second neutral default. Additive column on M005's bandit_plays,
269
+ # and eager on purpose: nothing bootstraps that table at engine init, so
270
+ # there is no reason to defer it.
271
+ Migration(name=_M044.NAME, db_target="learning", ddl=_M044.DDL,
272
+ dependencies=(_M005.NAME,)),
255
273
  # M006 + M011 are deliberately NOT here — see DEFERRED_MIGRATIONS below.
256
274
  ]
257
275
 
@@ -323,6 +341,39 @@ DEFERRED_MIGRATIONS: list[Migration] = [
323
341
  # store is recoverable.
324
342
  Migration(name=_M043.NAME, db_target="memory", ddl=_M043.DDL,
325
343
  dependencies=(_M011.NAME,)),
344
+ # M045 holds the per-fact outcome score. Deferred because its backfill
345
+ # reads action_outcomes, which engine init bootstraps — the same reason
346
+ # M006 and M011 are deferred. Depends on M006 for the reward column it
347
+ # averages.
348
+ Migration(name=_M045.NAME, db_target="memory", ddl=_M045.DDL,
349
+ dependencies=(_M006.NAME,)),
350
+ # M046 renames the fact type used for planned future events, which means
351
+ # rebuilding atomic_facts to widen a CHECK constraint SQLite cannot alter.
352
+ # Deferred for the same reason as M043: atomic_facts is bootstrapped at
353
+ # engine init, and apply_deferred takes a verified snapshot before the first
354
+ # migration it applies, so a table rebuild has something to fall back to.
355
+ # Depends on M043 so the two never contend for the same table in one pass.
356
+ Migration(name=_M046.NAME, db_target="memory", ddl=_M046.DDL,
357
+ dependencies=(_M043.NAME,)),
358
+ # M047 rewrites the two Fisher vectors on each fact as float32 rather than
359
+ # as decimal text. Deferred because it walks every fact in atomic_facts,
360
+ # which engine init bootstraps. It changes no schema and both forms stay
361
+ # readable, so it is resumable and an interrupted store still works.
362
+ # Depends on M046 so a table rebuild and a full-table update never run in
363
+ # the same pass over the same table.
364
+ Migration(name=_M047.NAME, db_target="memory", ddl=_M047.DDL,
365
+ dependencies=(_M046.NAME,)),
366
+ # M048 finishes what M046 started: M046 renamed the type used for planned
367
+ # events without re-reading a single one of them, so the same wrongly-filed
368
+ # rows now carry a more confident name. Depends on M046 for the rename.
369
+ Migration(name=_M048.NAME, db_target="memory", ddl=_M048.DDL,
370
+ dependencies=(_M046.NAME,)),
371
+ # M049 gives schema_version the unique constraint its six writers all
372
+ # assumed it had. Every one uses INSERT OR IGNORE, which ignores nothing
373
+ # without a constraint, so each appended a duplicate per run: seven distinct
374
+ # versions held as 3,496 rows on one store and 234,348 on another. No
375
+ # dependency -- it touches a bookkeeping table no other migration reads.
376
+ Migration(name=_M049.NAME, db_target="memory", ddl=_M049.DDL),
326
377
  ]
327
378
 
328
379
 
@@ -590,6 +641,60 @@ def _deferred_already_applied(conn: sqlite3.Connection, name: str) -> bool:
590
641
  return False
591
642
 
592
643
 
644
+ def _breaking_floor(learning_db: Path, memory_db: Path) -> int:
645
+ """Highest floor declared by a migration that is recorded complete.
646
+
647
+ A migration declares ``BREAKING_VERSION`` when a store it has touched must
648
+ not be opened by an older build. Only completed ones count: a migration that
649
+ failed has not changed anything an older build would trip over.
650
+ """
651
+ from superlocalmemory.storage._migration_internals import _MODULES
652
+
653
+ logs = {"learning": _read_log(learning_db), "memory": _read_log(memory_db)}
654
+ floor = 0
655
+ for migration in (*MIGRATIONS, *DEFERRED_MIGRATIONS):
656
+ module = _MODULES.get(migration.name)
657
+ declared = getattr(module, "BREAKING_VERSION", 0) if module else 0
658
+ if not declared:
659
+ continue
660
+ if logs.get(migration.db_target, {}).get(migration.name) == "complete":
661
+ floor = max(floor, int(declared))
662
+ return floor
663
+
664
+
665
+ def _stamp_breaking_floor(
666
+ learning_db: Path, memory_db: Path, details: dict[str, str],
667
+ ) -> None:
668
+ """Raise the recorded version to the highest completed breaking floor.
669
+
670
+ Monotonic: never lowers a stored version, so it cannot undo the completion
671
+ certificate on an already-current store. Never fatal — a store that cannot
672
+ be stamped is reported, because failing the whole run here would block an
673
+ upgrade over a guard that only matters to older builds.
674
+ """
675
+ floor = _breaking_floor(learning_db, memory_db)
676
+ if floor <= 0:
677
+ return
678
+ for db_path in (learning_db, memory_db):
679
+ try:
680
+ current = _read_schema_version(db_path)
681
+ if current >= floor:
682
+ continue
683
+ conn = _connect(db_path)
684
+ try:
685
+ _ensure_schema_version_table(conn)
686
+ _write_schema_version(conn, floor)
687
+ finally:
688
+ try:
689
+ conn.close()
690
+ except sqlite3.Error: # pragma: no cover
691
+ pass
692
+ except sqlite3.Error as exc: # pragma: no cover — reported, not fatal
693
+ details["schema_version_floor"] = (
694
+ f"cannot raise the floor on {db_path}: {exc}"
695
+ )
696
+
697
+
593
698
  def apply_deferred(
594
699
  learning_db: Path,
595
700
  memory_db: Path,
@@ -694,6 +799,20 @@ def apply_deferred(
694
799
  except sqlite3.Error: # pragma: no cover
695
800
  pass
696
801
 
802
+ # A migration that makes the store unusable by an older build declares a
803
+ # floor, and that floor is written as soon as the migration is recorded
804
+ # complete — BEFORE and independent of the completion certificate below.
805
+ #
806
+ # The certificate is all-or-nothing across both databases by design. That is
807
+ # right for "is this store fully migrated" and wrong for "may an older build
808
+ # write to it": an unrelated failure on the other database would otherwise
809
+ # leave a rebuilt table guarded by the old ceiling, and the first planned
810
+ # event an older build stored would be rejected by the new constraint and
811
+ # lost. Raising the floor turns that into a refusal to start, which is what
812
+ # the ceiling is for.
813
+ if not dry_run:
814
+ _stamp_breaking_floor(learning_db, memory_db, details)
815
+
697
816
  # The version ceiling is a completion certificate, not an intent marker.
698
817
  # M039 is deferred until engine-owned tables exist, so apply_all must not
699
818
  # stamp version 39. Stamp both stores only after every eager and deferred
@@ -409,45 +409,31 @@ def _sync_lifecycle_mirror(conn: sqlite3.Connection) -> None:
409
409
  """)
410
410
 
411
411
 
412
- def verify(conn: sqlite3.Connection) -> bool:
413
- """Whether the repair's end-state holds.
414
-
415
- Called on every start for an already-complete migration. Returning False
416
- routes to ``repair()``, which makes this a standing guard: if pollution ever
417
- reappears, the next daemon start withholds it without anyone asking.
412
+ def unmet(conn: sqlite3.Connection) -> str:
413
+ """Which check does not hold, named. Empty string when all of them do.
414
+
415
+ ``verify()`` returns a bare boolean, so when a completed migration stops
416
+ verifying the runner can only say "safe repair did not restore M043". This
417
+ checks five separate things, and that sentence names none of them -- a user
418
+ hitting it had to come back and ask which, and so did we. This is the same
419
+ gap that ``migration_failure_reasons`` closed one level up, left open one
420
+ level down.
418
421
  """
419
422
  if not _table_exists(conn, "atomic_facts"):
420
- return True
423
+ return ""
421
424
  if not _has_column(conn, "atomic_facts", "quarantined"):
422
- return False
425
+ return "atomic_facts has no 'quarantined' column"
423
426
  if not _table_exists(conn, "consolidated_summaries"):
424
- return False
425
-
427
+ return "the consolidated_summaries display table is missing"
426
428
  if _table_exists(conn, "fact_consolidations"):
427
- unwithheld = _count(
429
+ n = _count(
428
430
  conn,
429
431
  "SELECT COUNT(*) FROM atomic_facts WHERE COALESCE(quarantined, 0) = 0 "
430
432
  " AND fact_id IN (" + _CONSOLIDATOR_ROWS + ")",
431
433
  )
432
- if unwithheld:
433
- return False
434
-
435
- # Every withheld row must still be visible somewhere, or the repair has
436
- # deleted the owner's view of it rather than moved it.
437
- #
438
- # BY IDENTITY *OR* CONTENT, and the "or" is what makes this an invariant
439
- # rather than a trap. Matching on content alone could never become true
440
- # once a row's content changed after being preserved: the display copy
441
- # keeps the old text, verify stays false, repair() runs apply() again,
442
- # apply() cannot change the past, and the migration is reported failed
443
- # on every start for the rest of the store's life. Matching on identity
444
- # alone fails the other way, because two withheld rows with identical
445
- # text collapse into one display row under the unique triple, leaving the
446
- # second with no row of its own id.
447
- #
448
- # Either match satisfies the guarantee that actually matters: nothing
449
- # the owner could see has stopped being visible.
450
- unpreserved = _count(conn, """
434
+ if n:
435
+ return f"{n} model-written summaries are not withheld from recall"
436
+ n = _count(conn, """
451
437
  SELECT COUNT(*) FROM atomic_facts af
452
438
  WHERE af.fact_id IN (""" + _CONSOLIDATOR_ROWS + """)
453
439
  AND NOT EXISTS (
@@ -457,13 +443,51 @@ def verify(conn: sqlite3.Connection) -> bool:
457
443
  OR cs.content = af.content)
458
444
  )
459
445
  """)
460
- if unpreserved:
461
- return False
462
-
446
+ if n:
447
+ return f"{n} withheld summaries have no display copy"
463
448
  if _table_exists(conn, "fact_retention"):
464
- if _count(conn, "SELECT COUNT(*) FROM (" + _wrongly_hidden(conn) + ")"):
465
- return False
466
- return True
449
+ n = _count(conn, "SELECT COUNT(*) FROM (" + _wrongly_hidden(conn) + ")")
450
+ if n:
451
+ return f"{n} real memories are hidden from recall and should not be"
452
+ return ""
453
+
454
+
455
+ def blocks_serving(conn: sqlite3.Connection) -> bool:
456
+ """Should a daemon refuse to serve while this check does not hold?
457
+
458
+ Only when the SCHEMA is missing. The two schema conditions here -- the
459
+ column and the display table -- mean queries would hit something that is not
460
+ there, so refusing is right. The other three are about DATA: a summary that
461
+ should be withheld is not withheld, or a real memory is hidden. Those make
462
+ some answers worse; they do not stop the store working.
463
+
464
+ The distinction matters because this ``verify()`` is a standing guard over
465
+ data that ordinary use can re-violate -- a consolidation pass hiding one more
466
+ memory is enough. Treating that like a missing table meant one drifted row
467
+ could return 503 on every route indefinitely, with a manual restart the only
468
+ way out. That is an outage caused by a quality check, which is worse than the
469
+ thing the check is for.
470
+
471
+ Reported as #125, where a user's daemon sat unusable on exactly this.
472
+ """
473
+ if not _table_exists(conn, "atomic_facts"):
474
+ return False
475
+ if not _has_column(conn, "atomic_facts", "quarantined"):
476
+ return True
477
+ return not _table_exists(conn, "consolidated_summaries")
478
+
479
+
480
+ def verify(conn: sqlite3.Connection) -> bool:
481
+ """Whether the repair's end-state holds.
482
+
483
+ Called on every start for an already-complete migration. Returning False
484
+ routes to ``repair()``, which makes this a standing guard: if pollution ever
485
+ reappears, the next daemon start withholds it without anyone asking.
486
+
487
+ Thin wrapper over ``unmet()`` so the two can never disagree about what
488
+ "verified" means.
489
+ """
490
+ return not unmet(conn)
467
491
 
468
492
 
469
493
  def repair(conn: sqlite3.Connection) -> None: