superlocalmemory 3.8.14 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (211) hide show
  1. package/ATTRIBUTION.md +4 -4
  2. package/CHANGELOG.md +113 -137
  3. package/README.md +65 -63
  4. package/docs/pi-dev-integration.md +1 -1
  5. package/package.json +6 -1
  6. package/plugin/.claude-plugin/plugin.json +1 -1
  7. package/plugin/CLAUDE.md +3 -3
  8. package/plugin/agents/slm-governance-advisor.md +1 -1
  9. package/plugin/agents/slm-loop-runner.md +1 -1
  10. package/plugin/agents/slm-memory-advisor.md +1 -1
  11. package/plugin/agents/slm-optimize-advisor.md +1 -1
  12. package/plugin/requirements.txt +1 -1
  13. package/plugin/skills/slm-cache/SKILL.md +1 -1
  14. package/plugin/skills/slm-compress/SKILL.md +1 -1
  15. package/plugin/skills/slm-governance/SKILL.md +1 -1
  16. package/plugin/skills/slm-graph/SKILL.md +1 -1
  17. package/plugin/skills/slm-loop/SKILL.md +1 -1
  18. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  19. package/plugin/skills/slm-profile/SKILL.md +1 -1
  20. package/plugin/skills/slm-recall/SKILL.md +1 -1
  21. package/plugin/skills/slm-remember/SKILL.md +1 -1
  22. package/plugin/skills/slm-scope/SKILL.md +1 -1
  23. package/plugin/skills/slm-session/SKILL.md +1 -1
  24. package/plugin/skills/slm-status/SKILL.md +1 -1
  25. package/plugin-src/rules/AGENTS.md +1 -1
  26. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-governance/SKILL.md +248 -0
  29. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  30. package/plugin-src/skills/slm-loop/SKILL.md +99 -0
  31. package/plugin-src/skills/slm-mesh/SKILL.md +282 -0
  32. package/plugin-src/skills/slm-profile/SKILL.md +148 -0
  33. package/plugin-src/skills/slm-recall/SKILL.md +1 -1
  34. package/plugin-src/skills/slm-remember/SKILL.md +1 -1
  35. package/plugin-src/skills/slm-scope/SKILL.md +176 -0
  36. package/plugin-src/skills/slm-session/SKILL.md +1 -1
  37. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  38. package/pyproject.toml +11 -4
  39. package/src/superlocalmemory/__init__.py +1 -1
  40. package/src/superlocalmemory/cli/commands.py +125 -11
  41. package/src/superlocalmemory/cli/daemon.py +5 -1
  42. package/src/superlocalmemory/cli/main.py +35 -2
  43. package/src/superlocalmemory/cli/ops_cmd.py +281 -0
  44. package/src/superlocalmemory/cli/setup_wizard.py +1 -1
  45. package/src/superlocalmemory/compliance/audit.py +65 -0
  46. package/src/superlocalmemory/compliance/eu_ai_act.py +27 -57
  47. package/src/superlocalmemory/compliance/gdpr.py +416 -20
  48. package/src/superlocalmemory/compliance/retention.py +74 -22
  49. package/src/superlocalmemory/compliance/scheduler.py +78 -9
  50. package/src/superlocalmemory/core/actor_context.py +166 -0
  51. package/src/superlocalmemory/core/admission.py +549 -0
  52. package/src/superlocalmemory/core/backend_orchestrator.py +23 -10
  53. package/src/superlocalmemory/core/config.py +202 -24
  54. package/src/superlocalmemory/core/consolidation_engine.py +13 -13
  55. package/src/superlocalmemory/core/context_cache.py +28 -0
  56. package/src/superlocalmemory/core/embeddings.py +64 -2
  57. package/src/superlocalmemory/core/engine.py +7 -2
  58. package/src/superlocalmemory/core/engine_ingestion.py +65 -3
  59. package/src/superlocalmemory/core/engine_wiring.py +32 -5
  60. package/src/superlocalmemory/core/ingest_policy.py +38 -0
  61. package/src/superlocalmemory/core/maintenance.py +255 -0
  62. package/src/superlocalmemory/core/modes.py +40 -13
  63. package/src/superlocalmemory/core/mutations.py +437 -44
  64. package/src/superlocalmemory/core/operation_policy.py +92 -0
  65. package/src/superlocalmemory/core/operation_policy_registry.py +542 -0
  66. package/src/superlocalmemory/core/operation_request.py +127 -0
  67. package/src/superlocalmemory/core/ops_remediation.py +542 -0
  68. package/src/superlocalmemory/core/recall_pipeline.py +7 -0
  69. package/src/superlocalmemory/core/remember_runtime.py +202 -4
  70. package/src/superlocalmemory/core/remote_mode.py +20 -5
  71. package/src/superlocalmemory/core/store_pipeline.py +150 -0
  72. package/src/superlocalmemory/core/topic_signature.py +19 -4
  73. package/src/superlocalmemory/core/transactions/__init__.py +78 -0
  74. package/src/superlocalmemory/core/transactions/concrete_owners.py +597 -0
  75. package/src/superlocalmemory/core/transactions/erasure.py +825 -0
  76. package/src/superlocalmemory/core/transactions/manifest.py +255 -0
  77. package/src/superlocalmemory/core/transactions/manifest_key.py +155 -0
  78. package/src/superlocalmemory/core/transactions/obligations.py +272 -0
  79. package/src/superlocalmemory/core/transactions/owners.py +114 -0
  80. package/src/superlocalmemory/core/transactions/reconciler.py +285 -0
  81. package/src/superlocalmemory/core/transactions/service.py +330 -0
  82. package/src/superlocalmemory/core/worker_pool.py +33 -5
  83. package/src/superlocalmemory/encoding/cognitive_consolidator.py +70 -28
  84. package/src/superlocalmemory/encoding/emotional.py +75 -14
  85. package/src/superlocalmemory/encoding/temporal_parser.py +4 -0
  86. package/src/superlocalmemory/evolution/blind_verifier.py +11 -4
  87. package/src/superlocalmemory/evolution/evolution_store.py +244 -4
  88. package/src/superlocalmemory/evolution/llm_dispatch.py +40 -0
  89. package/src/superlocalmemory/evolution/model_selection.py +18 -3
  90. package/src/superlocalmemory/evolution/mutation_generator.py +3 -0
  91. package/src/superlocalmemory/evolution/skill_activator.py +270 -0
  92. package/src/superlocalmemory/evolution/skill_evolver.py +281 -59
  93. package/src/superlocalmemory/evolution/types.py +30 -8
  94. package/src/superlocalmemory/graph/cozo_backend.py +17 -9
  95. package/src/superlocalmemory/hooks/auto_invoker.py +2 -1
  96. package/src/superlocalmemory/hooks/auto_recall.py +64 -30
  97. package/src/superlocalmemory/hooks/codex_assets.py +14 -1
  98. package/src/superlocalmemory/infra/backup.py +434 -7
  99. package/src/superlocalmemory/infra/process_reaper.py +18 -0
  100. package/src/superlocalmemory/infra/self_heal.py +401 -0
  101. package/src/superlocalmemory/learning/feedback.py +52 -9
  102. package/src/superlocalmemory/loops/engine.py +10 -0
  103. package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
  104. package/src/superlocalmemory/mcp/http_transport.py +30 -331
  105. package/src/superlocalmemory/mcp/profiles.py +5 -0
  106. package/src/superlocalmemory/mcp/resources.py +8 -0
  107. package/src/superlocalmemory/mcp/server.py +51 -4
  108. package/src/superlocalmemory/mcp/shared.py +19 -0
  109. package/src/superlocalmemory/mcp/tools_active.py +25 -4
  110. package/src/superlocalmemory/mcp/tools_code_graph.py +26 -18
  111. package/src/superlocalmemory/mcp/tools_context.py +50 -8
  112. package/src/superlocalmemory/mcp/tools_core.py +69 -21
  113. package/src/superlocalmemory/mcp/tools_evolution.py +9 -2
  114. package/src/superlocalmemory/mcp/tools_learning.py +21 -10
  115. package/src/superlocalmemory/mcp/tools_loops.py +29 -18
  116. package/src/superlocalmemory/mcp/tools_mesh.py +8 -0
  117. package/src/superlocalmemory/mcp/tools_ops.py +115 -0
  118. package/src/superlocalmemory/mcp/tools_optimize.py +4 -0
  119. package/src/superlocalmemory/mcp/tools_v28.py +10 -3
  120. package/src/superlocalmemory/mcp/tools_v3.py +34 -14
  121. package/src/superlocalmemory/mcp/tools_v33.py +18 -33
  122. package/src/superlocalmemory/mesh/broker.py +124 -46
  123. package/src/superlocalmemory/mesh/broker_security.py +470 -0
  124. package/src/superlocalmemory/mesh/discovery.py +365 -0
  125. package/src/superlocalmemory/mesh/lock_protocol.py +313 -0
  126. package/src/superlocalmemory/mesh/node_identity.py +97 -0
  127. package/src/superlocalmemory/mesh/outbox_remote.py +429 -0
  128. package/src/superlocalmemory/mesh/remote_sync.py +511 -28
  129. package/src/superlocalmemory/mesh/state_sync.py +286 -0
  130. package/src/superlocalmemory/optimize/config/store.py +45 -0
  131. package/src/superlocalmemory/parameterization/cross_project.py +12 -0
  132. package/src/superlocalmemory/parameterization/prompt_injector.py +13 -11
  133. package/src/superlocalmemory/parameterization/prompt_lifecycle.py +8 -2
  134. package/src/superlocalmemory/parameterization/workflow_miner.py +17 -0
  135. package/src/superlocalmemory/retrieval/ann_index.py +5 -0
  136. package/src/superlocalmemory/retrieval/bm25_channel.py +49 -2
  137. package/src/superlocalmemory/retrieval/engine.py +19 -4
  138. package/src/superlocalmemory/retrieval/fusion.py +4 -1
  139. package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -3
  140. package/src/superlocalmemory/retrieval/remote_reranker.py +47 -22
  141. package/src/superlocalmemory/retrieval/reranker.py +32 -1
  142. package/src/superlocalmemory/retrieval/temporal_channel.py +16 -3
  143. package/src/superlocalmemory/retrieval/temporal_utils.py +107 -0
  144. package/src/superlocalmemory/retrieval/temporal_validity_filter.py +155 -42
  145. package/src/superlocalmemory/retrieval/vector_store.py +214 -8
  146. package/src/superlocalmemory/server/api.py +5 -5
  147. package/src/superlocalmemory/server/egress_policy.py +258 -0
  148. package/src/superlocalmemory/server/rbac_enforce.py +32 -0
  149. package/src/superlocalmemory/server/route_mutations.py +20 -0
  150. package/src/superlocalmemory/server/routes/compliance.py +153 -7
  151. package/src/superlocalmemory/server/routes/data_io.py +43 -2
  152. package/src/superlocalmemory/server/routes/events.py +15 -0
  153. package/src/superlocalmemory/server/routes/memories.py +56 -3
  154. package/src/superlocalmemory/server/routes/mesh.py +82 -1
  155. package/src/superlocalmemory/server/routes/mesh_lock.py +54 -0
  156. package/src/superlocalmemory/server/routes/mesh_state.py +63 -0
  157. package/src/superlocalmemory/server/routes/v3_api.py +50 -27
  158. package/src/superlocalmemory/server/routes/ws.py +86 -0
  159. package/src/superlocalmemory/server/ui.py +6 -6
  160. package/src/superlocalmemory/server/unified_daemon.py +942 -119
  161. package/src/superlocalmemory/storage/_migration_internals.py +568 -0
  162. package/src/superlocalmemory/storage/_schema_version.py +110 -0
  163. package/src/superlocalmemory/storage/database.py +329 -24
  164. package/src/superlocalmemory/storage/embedding_migrator.py +246 -51
  165. package/src/superlocalmemory/storage/erasure_fence.py +45 -0
  166. package/src/superlocalmemory/storage/generation_fence.py +63 -0
  167. package/src/superlocalmemory/storage/migration_runner.py +140 -424
  168. package/src/superlocalmemory/storage/migrations/M009_model_lineage.py +40 -0
  169. package/src/superlocalmemory/storage/migrations/M033_projection_transactions.py +148 -0
  170. package/src/superlocalmemory/storage/migrations/M034_obligation_integrity.py +58 -0
  171. package/src/superlocalmemory/storage/migrations/M035_erasure_receipts.py +113 -0
  172. package/src/superlocalmemory/storage/migrations/M036_vector_row_map.py +107 -0
  173. package/src/superlocalmemory/storage/migrations/M037_manifest_hmac_version.py +162 -0
  174. package/src/superlocalmemory/storage/migrations/{M033_learning_feedback_channel.py → M038_learning_feedback_channel.py} +3 -3
  175. package/src/superlocalmemory/storage/migrations/{M034_scene_fact_members.py → M039_scene_fact_members.py} +15 -5
  176. package/src/superlocalmemory/storage/migrations/__init__.py +4 -4
  177. package/src/superlocalmemory/storage/schema.py +10 -2
  178. package/src/superlocalmemory/storage/write_coordinator.py +125 -0
  179. package/src/superlocalmemory/trust/scorer.py +28 -4
  180. package/src/superlocalmemory/ui/index.html +14 -3
  181. package/src/superlocalmemory/ui/js/auto-settings.js +12 -1
  182. package/src/superlocalmemory/ui/js/brain.js +6 -4
  183. package/src/superlocalmemory/ui/js/compliance.js +66 -12
  184. package/src/superlocalmemory/ui/js/dashboard.js +13 -3
  185. package/src/superlocalmemory/ui/js/feedback.js +8 -2
  186. package/src/superlocalmemory/ui/js/lifecycle.js +7 -1
  187. package/src/superlocalmemory/ui/js/modal.js +272 -5
  188. package/src/superlocalmemory/ui/js/od-backup.js +9 -2
  189. package/src/superlocalmemory/ui/js/od-compliance-ext.js +301 -0
  190. package/src/superlocalmemory/ui/js/od-operations.js +154 -23
  191. package/src/superlocalmemory/ui/js/od-ops-health.js +417 -0
  192. package/src/superlocalmemory/ui/js/od-optimize.js +35 -21
  193. package/src/superlocalmemory/ui/js/od-team.js +9 -2
  194. package/src/superlocalmemory/ui/js/optimize.js +13 -16
  195. package/src/superlocalmemory/ui/js/profiles.js +7 -3
  196. package/src/superlocalmemory/ui/js/settings.js +7 -1
  197. package/src/superlocalmemory/vector/lancedb_backend.py +19 -9
  198. package/src/superlocalmemory/attribution/mathematical_dna.py +0 -235
  199. package/src/superlocalmemory/cli/post_install.py +0 -114
  200. package/src/superlocalmemory/core/clock_monitor.py +0 -45
  201. package/src/superlocalmemory/core/db_pool.py +0 -80
  202. package/src/superlocalmemory/core/error_catalog.py +0 -113
  203. package/src/superlocalmemory/core/loop_watchdog.py +0 -56
  204. package/src/superlocalmemory/core/priority_queue.py +0 -61
  205. package/src/superlocalmemory/core/pruning_engine.py +0 -216
  206. package/src/superlocalmemory/core/queue_dispatcher.py +0 -73
  207. package/src/superlocalmemory/core/slmignore.py +0 -125
  208. package/src/superlocalmemory/infra/heartbeat_monitor.py +0 -140
  209. package/src/superlocalmemory/infra/webhook_dispatcher.py +0 -247
  210. package/src/superlocalmemory/learning/quantization_scheduler.py +0 -320
  211. package/src/superlocalmemory/storage/access_control.py +0 -182
@@ -24,15 +24,17 @@ Hard rules enforced (LLD-07 §7):
24
24
  - MIG3: a failing migration does NOT prevent the runner from attempting
25
25
  the rest, and does NOT raise to the caller — result comes through the
26
26
  returned stats dict.
27
+
28
+ The private apply-engine (``Migration``, the ``migration_log`` primitives,
29
+ ``_apply_single``, and the name→module registry) lives in
30
+ ``superlocalmemory.storage._migration_internals``; this module owns the ordered
31
+ catalogue and the public orchestration functions.
27
32
  """
28
33
 
29
34
  from __future__ import annotations
30
35
 
31
- import hashlib
32
36
  import logging
33
37
  import sqlite3
34
- from dataclasses import dataclass, field
35
- from datetime import datetime, timezone
36
38
  from pathlib import Path
37
39
 
38
40
  from superlocalmemory.storage.migrations import (
@@ -129,80 +131,46 @@ from superlocalmemory.storage.migrations import (
129
131
  M032_write_coordinator_admission as _M032,
130
132
  )
131
133
  from superlocalmemory.storage.migrations import (
132
- M033_learning_feedback_channel as _M033,
133
- )
134
- from superlocalmemory.storage.migrations import (
135
- M034_scene_fact_members as _M034,
136
- )
137
-
138
- # Map migration name → module (used for the optional ``verify(conn)`` hook
139
- # that lets the runner detect "already applied" state when an idempotent
140
- # retry would otherwise trigger duplicate-column / duplicate-table errors).
141
- _MODULES = {
142
- _M001.NAME: _M001,
143
- _M002.NAME: _M002,
144
- _M003.NAME: _M003,
145
- _M004.NAME: _M004,
146
- _M005.NAME: _M005,
147
- _M006.NAME: _M006,
148
- _M007.NAME: _M007,
149
- _M009.NAME: _M009,
150
- _M010.NAME: _M010,
151
- _M011.NAME: _M011,
152
- _M012.NAME: _M012,
153
- _M013.NAME: _M013,
154
- _M014.NAME: _M014,
155
- _M015.NAME: _M015,
156
- _M016.NAME: _M016,
157
- _M017.NAME: _M017,
158
- _M018.NAME: _M018,
159
- _M019.NAME: _M019,
160
- _M020.NAME: _M020,
161
- _M021.NAME: _M021,
162
- _M022.NAME: _M022,
163
- _M023.NAME: _M023,
164
- _M024.NAME: _M024,
165
- _M025.NAME: _M025,
166
- _M026.NAME: _M026,
167
- _M027.NAME: _M027,
168
- _M028.NAME: _M028,
169
- _M029.NAME: _M029,
170
- _M030.NAME: _M030,
171
- _M031.NAME: _M031,
172
- _M032.NAME: _M032,
173
- _M033.NAME: _M033,
174
- _M034.NAME: _M034,
175
- }
134
+ M033_projection_transactions as _M033,
135
+ )
136
+ from superlocalmemory.storage.migrations import (
137
+ M034_obligation_integrity as _M034,
138
+ )
139
+ from superlocalmemory.storage.migrations import (
140
+ M035_erasure_receipts as _M035,
141
+ )
142
+ from superlocalmemory.storage.migrations import (
143
+ M036_vector_row_map as _M036,
144
+ )
145
+ from superlocalmemory.storage.migrations import (
146
+ M037_manifest_hmac_version as _M037,
147
+ )
148
+ from superlocalmemory.storage.migrations import (
149
+ M038_learning_feedback_channel as _M038,
150
+ )
151
+ from superlocalmemory.storage.migrations import (
152
+ M039_scene_fact_members as _M039,
153
+ )
154
+ from superlocalmemory.storage._schema_version import (
155
+ SUPPORTED_SCHEMA_VERSION,
156
+ SchemaVersionError,
157
+ check_version_or_raise as _check_version_or_raise,
158
+ ensure_schema_version_table as _ensure_schema_version_table,
159
+ write_schema_version as _write_schema_version,
160
+ )
161
+ from superlocalmemory.storage._migration_internals import (
162
+ Migration,
163
+ _MODULES, # noqa: F401 — re-exported for test/introspection compatibility
164
+ _apply_single,
165
+ _connect,
166
+ _db_for,
167
+ _ensure_migration_log,
168
+ _migration_log_exists,
169
+ _read_log,
170
+ )
176
171
 
177
172
  logger = logging.getLogger(__name__)
178
173
 
179
- # Exact historical DDL fingerprints whose resulting schema is intentionally
180
- # accepted by the current migration. Unknown hashes are never reconciled.
181
- _KNOWN_EQUIVALENT_DDL_HASHES: dict[str, frozenset[str]] = {
182
- _M002.NAME: frozenset({
183
- # v3.4.21 hardened copy-forward variant.
184
- "347eeb2ec8aac89f7cbf373da49ac9446be9ed150e6105c382c656cd22426d4b",
185
- # v3.4.22 model_version-default variant shipped through 3.6.x.
186
- "d28666fa1dfa66e6514efd288e6748363513da2255a4cee95d80f233e6728ae7",
187
- }),
188
- _M032.NAME: frozenset({
189
- # Provisional 3.8.6 development ledger: global idempotency_key and
190
- # operation_id uniqueness. Its standalone table is safely rebuilt by
191
- # M032.repair() into the profile-scoped receipt contract.
192
- "e45df41becba3d0c3342eca5ec3bd83aa899eef76943c819d2da73b4ca1625a7",
193
- }),
194
- }
195
-
196
-
197
- @dataclass(frozen=True, slots=True)
198
- class Migration:
199
- """Single migration definition."""
200
-
201
- name: str
202
- db_target: str # 'learning' or 'memory'
203
- ddl: str
204
- dependencies: tuple[str, ...] = field(default_factory=tuple)
205
-
206
174
 
207
175
  # Order matters: M003 creates the log table. The runner handles M003's own
208
176
  # bootstrap (it can't record itself before it exists).
@@ -228,11 +196,6 @@ MIGRATIONS: list[Migration] = [
228
196
  # observations for ShadowTest persistence across daemon restart.
229
197
  Migration(name=_M012.NAME, db_target="learning", ddl=_M012.DDL,
230
198
  dependencies=(_M003.NAME,)),
231
- # M033 adds learning_feedback.channel, which pattern_miner has always
232
- # queried but which no schema ever defined. Its DDL creates the table
233
- # when absent, so it needs no dependency beyond the migration log.
234
- Migration(name=_M033.NAME, db_target="learning", ddl=_M033.DDL,
235
- dependencies=(_M003.NAME,)),
236
199
  Migration(name=_M004.NAME, db_target="memory", ddl=_M004.DDL),
237
200
  # M007 creates pending_outcomes (memory.db, LLD-00 §1.2).
238
201
  Migration(name=_M007.NAME, db_target="memory", ddl=_M007.DDL),
@@ -249,6 +212,20 @@ MIGRATIONS: list[Migration] = [
249
212
  # M032 is standalone and must precede daemon readiness: typed writes use
250
213
  # this append-only receipt ledger for durable idempotency.
251
214
  Migration(name=_M032.NAME, db_target="memory", ddl=_M032.DDL),
215
+ Migration(name=_M033.NAME, db_target="memory", ddl=_M033.DDL),
216
+ Migration(name=_M034.NAME, db_target="memory", ddl=_M034.DDL,
217
+ dependencies=(_M033.NAME,)),
218
+ Migration(name=_M035.NAME, db_target="memory", ddl=_M035.DDL,
219
+ dependencies=(_M033.NAME,)),
220
+ Migration(name=_M036.NAME, db_target="memory", ddl=_M036.DDL,
221
+ dependencies=(_M033.NAME,)),
222
+ Migration(name=_M037.NAME, db_target="memory", ddl=_M037.DDL,
223
+ dependencies=(_M033.NAME, _M035.NAME)),
224
+ # Main-line M033 is renumbered in V4 because V4 already owns M033-M037.
225
+ # It repairs the legacy learning_feedback schema before any reader mines
226
+ # channel patterns.
227
+ Migration(name=_M038.NAME, db_target="learning", ddl=_M038.DDL,
228
+ dependencies=(_M003.NAME,)),
252
229
  # M006 + M011 are deliberately NOT here — see DEFERRED_MIGRATIONS below.
253
230
  ]
254
231
 
@@ -309,336 +286,12 @@ DEFERRED_MIGRATIONS: list[Migration] = [
309
286
  Migration(name=_M029.NAME, db_target="memory", ddl=_M029.DDL),
310
287
  # M030 bounds Entity Explorer pagination and profile-summary ranking.
311
288
  Migration(name=_M030.NAME, db_target="memory", ddl=_M030.DDL),
312
- # M034 normalizes memory_scenes.fact_ids_json after engine initialization
313
- # has created memory_scenes and atomic_facts.
314
- Migration(name=_M034.NAME, db_target="memory", ddl=_M034.DDL),
289
+ # Main-line M034 is renumbered in V4. It must remain deferred because its
290
+ # backfill joins engine-bootstrapped memory_scenes and atomic_facts.
291
+ Migration(name=_M039.NAME, db_target="memory", ddl=_M039.DDL),
315
292
  ]
316
293
 
317
294
 
318
- def _now_iso() -> str:
319
- return datetime.now(timezone.utc).isoformat()
320
-
321
-
322
- def _ddl_hash(ddl: str) -> str:
323
- return hashlib.sha256(ddl.encode("utf-8")).hexdigest()
324
-
325
-
326
- def _connect(db_path: Path) -> sqlite3.Connection:
327
- # isolation_level=None → we manage transactions explicitly via DDL.
328
- conn = sqlite3.connect(db_path, isolation_level=None)
329
- conn.execute("PRAGMA foreign_keys = OFF;")
330
- return conn
331
-
332
-
333
- def _migration_log_exists(conn: sqlite3.Connection) -> bool:
334
- row = conn.execute(
335
- "SELECT name FROM sqlite_master "
336
- "WHERE type='table' AND name='migration_log'"
337
- ).fetchone()
338
- return row is not None
339
-
340
-
341
- def _ensure_migration_log(conn: sqlite3.Connection) -> None:
342
- """Bootstrap the migration_log table on a DB if absent.
343
-
344
- Uses the M003 DDL verbatim so the runner treats migration_log identically
345
- on both learning.db and memory.db.
346
- """
347
- conn.executescript(_M003.DDL)
348
-
349
-
350
- def _get_log_row(conn: sqlite3.Connection, name: str) -> tuple | None:
351
- return conn.execute(
352
- "SELECT name, applied_at, ddl_sha256, rows_affected, status "
353
- "FROM migration_log WHERE name = ?",
354
- (name,),
355
- ).fetchone()
356
-
357
-
358
- def _upsert_log(
359
- conn: sqlite3.Connection,
360
- name: str,
361
- ddl_hash: str,
362
- status: str,
363
- rows_affected: int = 0,
364
- ) -> None:
365
- conn.execute(
366
- "INSERT INTO migration_log "
367
- "(name, applied_at, ddl_sha256, rows_affected, status) "
368
- "VALUES (?, ?, ?, ?, ?) "
369
- "ON CONFLICT(name) DO UPDATE SET "
370
- " applied_at = excluded.applied_at, "
371
- " ddl_sha256 = excluded.ddl_sha256, "
372
- " rows_affected = excluded.rows_affected, "
373
- " status = excluded.status",
374
- (name, _now_iso(), ddl_hash, rows_affected, status),
375
- )
376
-
377
-
378
- def _delete_log(conn: sqlite3.Connection, name: str) -> None:
379
- conn.execute("DELETE FROM migration_log WHERE name = ?", (name,))
380
-
381
-
382
- def _apply_single(
383
- conn: sqlite3.Connection,
384
- migration: Migration,
385
- *,
386
- dry_run: bool,
387
- ) -> tuple[str, str]:
388
- """Apply one migration against ``conn``.
389
-
390
- Returns (outcome, detail) where outcome is one of:
391
- - "applied"
392
- - "skipped"
393
- - "failed"
394
- """
395
- ddl_hash = _ddl_hash(migration.ddl)
396
-
397
- # Bootstrap: if migration_log doesn't exist yet, this MUST be M003.
398
- if not _migration_log_exists(conn):
399
- if migration.name != _M003.NAME:
400
- # Other migrations can't check state → treat as unrecoverable here.
401
- return ("failed",
402
- f"migration_log missing when attempting {migration.name}")
403
- if dry_run:
404
- return ("skipped", "dry-run: would create migration_log")
405
- try:
406
- _ensure_migration_log(conn)
407
- _upsert_log(conn, migration.name, ddl_hash, "complete")
408
- return ("applied", "bootstrapped migration_log")
409
- except sqlite3.Error as exc: # pragma: no cover — defensive
410
- logger.warning("M003 bootstrap failed: %s", exc)
411
- return ("failed", f"bootstrap error: {exc}")
412
-
413
- # M003 specifically — if log already exists, ensure M003's own row is there
414
- # (records the fact that the table was bootstrapped previously).
415
- existing = _get_log_row(conn, migration.name)
416
-
417
- if existing is not None:
418
- _, _, logged_hash, _, status = existing
419
- if status == "complete":
420
- if logged_hash != ddl_hash:
421
- # v3.7.6 (#70): a complete migration whose logged DDL hash no
422
- # longer matches the current text is only a real failure if the
423
- # schema it guarantees is actually absent. Historically-benign
424
- # DDL edits (e.g. M002's V3.4.21 <-> S9-W1 variants that build the
425
- # identical end-state) would otherwise brick readiness forever on
426
- # upgrade. Consult the migration's own verify(); if the schema is
427
- # in place, reconcile the log to the current hash and treat as
428
- # already-applied instead of failing the daemon into permanent
429
- # not_ready. Absent/failing verify keeps the hard failure.
430
- allowed_hashes = _KNOWN_EQUIVALENT_DDL_HASHES.get(
431
- migration.name, frozenset(),
432
- )
433
- mod = _MODULES.get(migration.name)
434
- verify_fn = (
435
- getattr(mod, "verify", None) if mod is not None else None
436
- )
437
- if logged_hash in allowed_hashes and verify_fn is not None:
438
- try:
439
- if verify_fn(conn):
440
- if not dry_run:
441
- try:
442
- _upsert_log(
443
- conn, migration.name, ddl_hash, "complete"
444
- )
445
- except sqlite3.Error: # pragma: no cover
446
- pass
447
- return (
448
- "skipped",
449
- "allowlisted historical DDL reconciled after "
450
- "full schema verification",
451
- )
452
- except sqlite3.Error: # pragma: no cover
453
- pass
454
- if dry_run:
455
- return (
456
- "skipped",
457
- "dry-run: would repair allowlisted historical schema",
458
- )
459
- repair_fn = getattr(mod, "repair", None) if mod is not None else None
460
- if callable(repair_fn):
461
- try:
462
- repair_fn(conn)
463
- if not bool(verify_fn(conn)):
464
- return (
465
- "failed",
466
- f"safe repair did not restore {migration.name}",
467
- )
468
- _upsert_log(conn, migration.name, ddl_hash, "complete")
469
- return (
470
- "applied",
471
- "allowlisted historical schema repaired safely",
472
- )
473
- except sqlite3.Error as exc:
474
- return (
475
- "failed",
476
- f"safe repair failed for {migration.name}: {exc}",
477
- )
478
- detail = (
479
- f"DDL drift detected for {migration.name}: "
480
- f"logged={logged_hash[:8]}... current={ddl_hash[:8]}..."
481
- )
482
- logger.warning(detail)
483
- return ("failed", detail)
484
- # A matching migration-log row is not proof that the promised
485
- # schema still exists. Existing installs can retain a migration log
486
- # while a partial restore drops an additive table or index.
487
- #
488
- # Never replay a historical migration merely because verify()
489
- # fails. Some migrations rebuild tables and transform data; replay
490
- # would be destructive (M002 is the canonical example). Only a
491
- # module-supplied repair(conn) hook is allowed to reconcile a
492
- # completed migration's end-state.
493
- mod = _MODULES.get(migration.name)
494
- verify_fn = (
495
- getattr(mod, "verify", None) if mod is not None else None
496
- )
497
- if verify_fn is None:
498
- return ("skipped", "already complete")
499
- try:
500
- schema_complete = bool(verify_fn(conn))
501
- except sqlite3.Error as exc:
502
- return (
503
- "failed",
504
- f"schema verification failed for {migration.name}: {exc}",
505
- )
506
- if schema_complete:
507
- return ("skipped", "already complete (schema verified)")
508
- if dry_run:
509
- return (
510
- "skipped",
511
- "dry-run: would repair missing migration end-state",
512
- )
513
- repair_fn = (
514
- getattr(mod, "repair", None) if mod is not None else None
515
- )
516
- if not callable(repair_fn):
517
- detail = (
518
- f"schema incomplete for completed migration "
519
- f"{migration.name}; automatic replay is disabled"
520
- )
521
- logger.warning(detail)
522
- return ("failed", detail)
523
- try:
524
- repair_fn(conn)
525
- except sqlite3.Error as exc:
526
- return (
527
- "failed",
528
- f"safe repair failed for {migration.name}: {exc}",
529
- )
530
- try:
531
- if not bool(verify_fn(conn)):
532
- return (
533
- "failed",
534
- f"safe repair did not restore {migration.name}",
535
- )
536
- except sqlite3.Error as exc:
537
- return (
538
- "failed",
539
- f"post-repair verification failed for "
540
- f"{migration.name}: {exc}",
541
- )
542
- return ("applied", "missing end-state repaired safely")
543
- # status is 'failed' or 'in_progress' → retry from scratch.
544
- if dry_run:
545
- return ("skipped", f"dry-run: would retry (status={status})")
546
- try:
547
- _delete_log(conn, migration.name)
548
- except sqlite3.Error as exc: # pragma: no cover — log table exists
549
- return ("failed", f"cannot clear prior log: {exc}")
550
-
551
- if dry_run:
552
- return ("skipped", "dry-run: would apply")
553
-
554
- # Mark in_progress, execute, update status. If DDL fails we roll our log
555
- # entry to 'failed' so next attempt will retry cleanly.
556
- try:
557
- _upsert_log(conn, migration.name, ddl_hash, "in_progress")
558
- except sqlite3.Error as exc: # pragma: no cover
559
- return ("failed", f"cannot record in_progress: {exc}")
560
-
561
- try:
562
- # A migration module may ship a custom apply(conn) for conditional logic
563
- # that static DDL can't express (e.g. SQLite has no ADD COLUMN IF NOT
564
- # EXISTS, and ALTER on a missing/already-altered table can't be guarded
565
- # in one executescript). If present, it runs instead of the DDL string;
566
- # otherwise the DDL is applied as before. Pure-DDL migrations are
567
- # unaffected.
568
- _mod = _MODULES.get(migration.name)
569
- _apply_fn = getattr(_mod, "apply", None) if _mod is not None else None
570
- if callable(_apply_fn):
571
- _apply_fn(conn)
572
- else:
573
- conn.executescript(migration.ddl)
574
- except sqlite3.Error as exc:
575
- # Best-effort rollback.
576
- try:
577
- conn.execute("ROLLBACK")
578
- except sqlite3.Error: # pragma: no cover — best-effort
579
- pass
580
- # Before marking failed, check if the migration's end-state is
581
- # already in place (e.g. crash-recovery retry against a DB where the
582
- # columns were added in a previous partial apply). If so, this is
583
- # effectively a successful idempotent re-run.
584
- mod = _MODULES.get(migration.name)
585
- verify_fn = getattr(mod, "verify", None) if mod is not None else None
586
- if verify_fn is not None:
587
- try:
588
- if verify_fn(conn):
589
- try:
590
- _upsert_log(conn, migration.name, ddl_hash, "complete")
591
- except sqlite3.Error: # pragma: no cover
592
- pass
593
- return ("applied",
594
- "already applied (verified via schema inspection)")
595
- except sqlite3.Error: # pragma: no cover
596
- pass
597
-
598
- logger.warning("Migration %s failed: %s", migration.name, exc)
599
- try:
600
- _upsert_log(conn, migration.name, ddl_hash, "failed")
601
- except sqlite3.Error: # pragma: no cover
602
- pass
603
- return ("failed", f"{type(exc).__name__}: {exc}")
604
-
605
- # S9-W1 H-DATA-01: optional post-DDL Python hook. Runs inside the same
606
- # connection (same DB file) after the DDL commits. Used by M002 to
607
- # backfill ``bytes_sha256`` on rows copied forward by the new-table
608
- # rename. If the hook raises, the migration is marked failed; the DDL
609
- # is NOT rolled back (already committed) but the runner reports the
610
- # problem so operators can intervene. Non-existent hooks are a no-op.
611
- mod = _MODULES.get(migration.name)
612
- post_hook = getattr(mod, "post_ddl_hook", None) if mod is not None else None
613
- if post_hook is not None:
614
- try:
615
- post_hook(conn)
616
- except Exception as exc: # noqa: BLE001 — report + mark failed
617
- logger.warning(
618
- "Migration %s DDL applied but post_ddl_hook failed: %s",
619
- migration.name, exc,
620
- )
621
- try:
622
- _upsert_log(conn, migration.name, ddl_hash, "failed")
623
- except sqlite3.Error: # pragma: no cover
624
- pass
625
- return ("failed", f"post_ddl_hook: {type(exc).__name__}: {exc}")
626
-
627
- try:
628
- _upsert_log(conn, migration.name, ddl_hash, "complete")
629
- except sqlite3.Error as exc: # pragma: no cover
630
- return ("failed", f"cannot record complete: {exc}")
631
- return ("applied", "ok")
632
-
633
-
634
- def _db_for(target: str, learning_db: Path, memory_db: Path) -> Path:
635
- if target == "learning":
636
- return learning_db
637
- if target == "memory":
638
- return memory_db
639
- raise ValueError(f"unknown db_target: {target}") # pragma: no cover
640
-
641
-
642
295
  def _bootstrap_both_migration_logs(
643
296
  learning_db: Path, memory_db: Path, *, dry_run: bool,
644
297
  ) -> tuple[list[str], dict[str, str]]:
@@ -712,7 +365,17 @@ def apply_all(
712
365
 
713
366
  Idempotent: already-applied migrations are skipped. Non-fatal: any
714
367
  migration that fails is recorded in ``failed`` and the runner moves on.
368
+
369
+ Raises SchemaVersionError before touching any data when the learning DB
370
+ reports a schema_version that exceeds SUPPORTED_SCHEMA_VERSION. This
371
+ prevents silent data corruption when downgrading to an older build.
715
372
  """
373
+ # Non-mutating version check: must run before any write. Both managed
374
+ # databases are validated so a downgrade is detectable regardless of which
375
+ # store carries the newer stamp.
376
+ _check_version_or_raise(learning_db)
377
+ _check_version_or_raise(memory_db)
378
+
716
379
  applied: list[str] = []
717
380
  skipped: list[str] = []
718
381
  failed: list[str] = []
@@ -736,7 +399,17 @@ def apply_all(
736
399
  failed.extend(bs_failed)
737
400
  details.update(bs_details)
738
401
 
402
+ blocked: set[str] = set()
739
403
  for migration in MIGRATIONS:
404
+ # A migration whose declared dependency did not complete must not run
405
+ # against a base schema that is missing that dependency's changes.
406
+ unmet = [d for d in migration.dependencies if d in failed or d in blocked]
407
+ if unmet:
408
+ skipped.append(migration.name)
409
+ blocked.add(migration.name)
410
+ details[migration.name] = "dependency not satisfied: " + ", ".join(unmet)
411
+ continue
412
+
740
413
  db_path = _db_for(migration.db_target, learning_db, memory_db)
741
414
  try:
742
415
  conn = _connect(db_path)
@@ -785,13 +458,32 @@ def apply_deferred(
785
458
  table is still missing, the underlying DDL raises ``no such table`` and
786
459
  the migration is recorded as ``failed`` — safe, the trainer already
787
460
  falls back to the position proxy when M006 hasn't completed.
461
+
462
+ Raises SchemaVersionError before touching any data when either managed
463
+ database reports a schema_version newer than SUPPORTED_SCHEMA_VERSION. This
464
+ path runs after engine init, so it must fail closed on a downgrade exactly
465
+ as ``apply_all`` does rather than write DDL an older build cannot interpret.
788
466
  """
467
+ # Non-mutating downgrade guard: must run before any write, mirroring
468
+ # apply_all. Both managed databases are validated so a newer stamp on
469
+ # either store halts the deferred pass.
470
+ _check_version_or_raise(learning_db)
471
+ _check_version_or_raise(memory_db)
472
+
789
473
  applied: list[str] = []
790
474
  skipped: list[str] = []
791
475
  failed: list[str] = []
792
476
  details: dict[str, str] = {}
793
477
 
478
+ blocked: set[str] = set()
794
479
  for migration in DEFERRED_MIGRATIONS:
480
+ unmet = [d for d in migration.dependencies if d in failed or d in blocked]
481
+ if unmet:
482
+ skipped.append(migration.name)
483
+ blocked.add(migration.name)
484
+ details[migration.name] = "dependency not satisfied: " + ", ".join(unmet)
485
+ continue
486
+
795
487
  db_path = _db_for(migration.db_target, learning_db, memory_db)
796
488
  try:
797
489
  conn = _connect(db_path)
@@ -831,6 +523,46 @@ def apply_deferred(
831
523
  except sqlite3.Error: # pragma: no cover
832
524
  pass
833
525
 
526
+ # The version ceiling is a completion certificate, not an intent marker.
527
+ # M039 is deferred until engine-owned tables exist, so apply_all must not
528
+ # stamp version 39. Stamp both stores only after every eager and deferred
529
+ # migration is recorded complete on its declared target.
530
+ if not failed and not dry_run:
531
+ logs = {
532
+ "learning": _read_log(learning_db),
533
+ "memory": _read_log(memory_db),
534
+ }
535
+ incomplete = [
536
+ migration.name
537
+ for migration in (*MIGRATIONS, *DEFERRED_MIGRATIONS)
538
+ if logs[migration.db_target].get(migration.name) != "complete"
539
+ ]
540
+ if incomplete:
541
+ failed.append("schema_version_stamp")
542
+ details["schema_version_stamp"] = (
543
+ "not stamped; incomplete migrations: " + ", ".join(incomplete)
544
+ )
545
+ else:
546
+ for _stamp_db in (learning_db, memory_db):
547
+ try:
548
+ _stamp_conn = _connect(_stamp_db)
549
+ try:
550
+ _ensure_schema_version_table(_stamp_conn)
551
+ _write_schema_version(
552
+ _stamp_conn, SUPPORTED_SCHEMA_VERSION,
553
+ )
554
+ finally:
555
+ try:
556
+ _stamp_conn.close()
557
+ except sqlite3.Error: # pragma: no cover
558
+ pass
559
+ except sqlite3.Error as exc: # pragma: no cover
560
+ failed.append("schema_version_stamp")
561
+ details["schema_version_stamp"] = (
562
+ f"cannot stamp {_stamp_db}: {exc}"
563
+ )
564
+ break
565
+
834
566
  return {
835
567
  "applied": applied,
836
568
  "skipped": skipped,
@@ -858,28 +590,12 @@ def status(learning_db: Path, memory_db: Path) -> dict[str, str]:
858
590
  return out
859
591
 
860
592
 
861
- def _read_log(db_path: Path) -> dict[str, str]:
862
- try:
863
- conn = sqlite3.connect(db_path)
864
- except sqlite3.Error: # pragma: no cover
865
- return {}
866
- try:
867
- if not _migration_log_exists(conn):
868
- return {}
869
- rows = conn.execute(
870
- "SELECT name, status FROM migration_log"
871
- ).fetchall()
872
- return {name: status for (name, status) in rows}
873
- except sqlite3.Error: # pragma: no cover
874
- return {}
875
- finally:
876
- conn.close()
877
-
878
-
879
593
  __all__ = (
880
594
  "Migration",
881
595
  "MIGRATIONS",
882
596
  "DEFERRED_MIGRATIONS",
597
+ "SUPPORTED_SCHEMA_VERSION",
598
+ "SchemaVersionError",
883
599
  "apply_all",
884
600
  "apply_deferred",
885
601
  "status",