superlocalmemory 3.8.13 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/ATTRIBUTION.md +4 -4
  2. package/CHANGELOG.md +113 -121
  3. package/README.md +65 -63
  4. package/docs/pi-dev-integration.md +1 -1
  5. package/package.json +6 -1
  6. package/plugin/.claude-plugin/plugin.json +1 -1
  7. package/plugin/CLAUDE.md +3 -3
  8. package/plugin/agents/slm-governance-advisor.md +1 -1
  9. package/plugin/agents/slm-loop-runner.md +1 -1
  10. package/plugin/agents/slm-memory-advisor.md +1 -1
  11. package/plugin/agents/slm-optimize-advisor.md +1 -1
  12. package/plugin/requirements.txt +1 -1
  13. package/plugin/skills/slm-cache/SKILL.md +1 -1
  14. package/plugin/skills/slm-compress/SKILL.md +1 -1
  15. package/plugin/skills/slm-governance/SKILL.md +1 -1
  16. package/plugin/skills/slm-graph/SKILL.md +1 -1
  17. package/plugin/skills/slm-loop/SKILL.md +1 -1
  18. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  19. package/plugin/skills/slm-profile/SKILL.md +1 -1
  20. package/plugin/skills/slm-recall/SKILL.md +1 -1
  21. package/plugin/skills/slm-remember/SKILL.md +1 -1
  22. package/plugin/skills/slm-scope/SKILL.md +1 -1
  23. package/plugin/skills/slm-session/SKILL.md +1 -1
  24. package/plugin/skills/slm-status/SKILL.md +1 -1
  25. package/plugin-src/rules/AGENTS.md +1 -1
  26. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-governance/SKILL.md +248 -0
  29. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  30. package/plugin-src/skills/slm-loop/SKILL.md +99 -0
  31. package/plugin-src/skills/slm-mesh/SKILL.md +282 -0
  32. package/plugin-src/skills/slm-profile/SKILL.md +148 -0
  33. package/plugin-src/skills/slm-recall/SKILL.md +1 -1
  34. package/plugin-src/skills/slm-remember/SKILL.md +1 -1
  35. package/plugin-src/skills/slm-scope/SKILL.md +176 -0
  36. package/plugin-src/skills/slm-session/SKILL.md +1 -1
  37. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  38. package/pyproject.toml +11 -4
  39. package/src/superlocalmemory/__init__.py +1 -1
  40. package/src/superlocalmemory/cli/commands.py +125 -11
  41. package/src/superlocalmemory/cli/daemon.py +5 -1
  42. package/src/superlocalmemory/cli/main.py +35 -2
  43. package/src/superlocalmemory/cli/ops_cmd.py +281 -0
  44. package/src/superlocalmemory/cli/setup_wizard.py +1 -1
  45. package/src/superlocalmemory/compliance/audit.py +65 -0
  46. package/src/superlocalmemory/compliance/eu_ai_act.py +27 -57
  47. package/src/superlocalmemory/compliance/gdpr.py +416 -20
  48. package/src/superlocalmemory/compliance/retention.py +74 -22
  49. package/src/superlocalmemory/compliance/scheduler.py +78 -9
  50. package/src/superlocalmemory/core/actor_context.py +166 -0
  51. package/src/superlocalmemory/core/admission.py +549 -0
  52. package/src/superlocalmemory/core/backend_orchestrator.py +23 -10
  53. package/src/superlocalmemory/core/config.py +202 -24
  54. package/src/superlocalmemory/core/consolidation_engine.py +13 -13
  55. package/src/superlocalmemory/core/context_cache.py +28 -0
  56. package/src/superlocalmemory/core/embeddings.py +64 -2
  57. package/src/superlocalmemory/core/engine.py +7 -2
  58. package/src/superlocalmemory/core/engine_ingestion.py +65 -3
  59. package/src/superlocalmemory/core/engine_wiring.py +36 -9
  60. package/src/superlocalmemory/core/ingest_policy.py +38 -0
  61. package/src/superlocalmemory/core/maintenance.py +255 -0
  62. package/src/superlocalmemory/core/modes.py +40 -13
  63. package/src/superlocalmemory/core/mutations.py +437 -44
  64. package/src/superlocalmemory/core/operation_policy.py +92 -0
  65. package/src/superlocalmemory/core/operation_policy_registry.py +542 -0
  66. package/src/superlocalmemory/core/operation_request.py +127 -0
  67. package/src/superlocalmemory/core/ops_remediation.py +542 -0
  68. package/src/superlocalmemory/core/recall_pipeline.py +7 -0
  69. package/src/superlocalmemory/core/remember_runtime.py +202 -4
  70. package/src/superlocalmemory/core/remote_mode.py +20 -5
  71. package/src/superlocalmemory/core/store_pipeline.py +150 -0
  72. package/src/superlocalmemory/core/topic_signature.py +19 -4
  73. package/src/superlocalmemory/core/transactions/__init__.py +78 -0
  74. package/src/superlocalmemory/core/transactions/concrete_owners.py +597 -0
  75. package/src/superlocalmemory/core/transactions/erasure.py +825 -0
  76. package/src/superlocalmemory/core/transactions/manifest.py +255 -0
  77. package/src/superlocalmemory/core/transactions/manifest_key.py +155 -0
  78. package/src/superlocalmemory/core/transactions/obligations.py +272 -0
  79. package/src/superlocalmemory/core/transactions/owners.py +114 -0
  80. package/src/superlocalmemory/core/transactions/reconciler.py +285 -0
  81. package/src/superlocalmemory/core/transactions/service.py +330 -0
  82. package/src/superlocalmemory/core/worker_pool.py +33 -5
  83. package/src/superlocalmemory/encoding/cognitive_consolidator.py +70 -28
  84. package/src/superlocalmemory/encoding/emotional.py +75 -14
  85. package/src/superlocalmemory/encoding/scene_builder.py +115 -13
  86. package/src/superlocalmemory/encoding/temporal_parser.py +4 -0
  87. package/src/superlocalmemory/evolution/blind_verifier.py +11 -4
  88. package/src/superlocalmemory/evolution/evolution_store.py +244 -4
  89. package/src/superlocalmemory/evolution/llm_dispatch.py +40 -0
  90. package/src/superlocalmemory/evolution/model_selection.py +18 -3
  91. package/src/superlocalmemory/evolution/mutation_generator.py +3 -0
  92. package/src/superlocalmemory/evolution/skill_activator.py +270 -0
  93. package/src/superlocalmemory/evolution/skill_evolver.py +281 -59
  94. package/src/superlocalmemory/evolution/types.py +30 -8
  95. package/src/superlocalmemory/graph/cozo_backend.py +17 -9
  96. package/src/superlocalmemory/hooks/auto_invoker.py +2 -1
  97. package/src/superlocalmemory/hooks/auto_recall.py +64 -30
  98. package/src/superlocalmemory/hooks/codex_assets.py +14 -1
  99. package/src/superlocalmemory/infra/backup.py +434 -7
  100. package/src/superlocalmemory/infra/process_reaper.py +18 -0
  101. package/src/superlocalmemory/infra/self_heal.py +401 -0
  102. package/src/superlocalmemory/learning/feedback.py +52 -9
  103. package/src/superlocalmemory/loops/engine.py +10 -0
  104. package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
  105. package/src/superlocalmemory/mcp/http_transport.py +30 -331
  106. package/src/superlocalmemory/mcp/profiles.py +5 -0
  107. package/src/superlocalmemory/mcp/resources.py +8 -0
  108. package/src/superlocalmemory/mcp/server.py +51 -4
  109. package/src/superlocalmemory/mcp/shared.py +19 -0
  110. package/src/superlocalmemory/mcp/tools_active.py +25 -4
  111. package/src/superlocalmemory/mcp/tools_code_graph.py +26 -18
  112. package/src/superlocalmemory/mcp/tools_context.py +50 -8
  113. package/src/superlocalmemory/mcp/tools_core.py +69 -21
  114. package/src/superlocalmemory/mcp/tools_evolution.py +9 -2
  115. package/src/superlocalmemory/mcp/tools_learning.py +21 -10
  116. package/src/superlocalmemory/mcp/tools_loops.py +29 -18
  117. package/src/superlocalmemory/mcp/tools_mesh.py +8 -0
  118. package/src/superlocalmemory/mcp/tools_ops.py +115 -0
  119. package/src/superlocalmemory/mcp/tools_optimize.py +4 -0
  120. package/src/superlocalmemory/mcp/tools_v28.py +10 -3
  121. package/src/superlocalmemory/mcp/tools_v3.py +34 -14
  122. package/src/superlocalmemory/mcp/tools_v33.py +18 -33
  123. package/src/superlocalmemory/mesh/broker.py +124 -46
  124. package/src/superlocalmemory/mesh/broker_security.py +470 -0
  125. package/src/superlocalmemory/mesh/discovery.py +365 -0
  126. package/src/superlocalmemory/mesh/lock_protocol.py +313 -0
  127. package/src/superlocalmemory/mesh/node_identity.py +97 -0
  128. package/src/superlocalmemory/mesh/outbox_remote.py +429 -0
  129. package/src/superlocalmemory/mesh/remote_sync.py +511 -28
  130. package/src/superlocalmemory/mesh/state_sync.py +286 -0
  131. package/src/superlocalmemory/optimize/config/store.py +45 -0
  132. package/src/superlocalmemory/parameterization/cross_project.py +12 -0
  133. package/src/superlocalmemory/parameterization/prompt_injector.py +13 -11
  134. package/src/superlocalmemory/parameterization/prompt_lifecycle.py +8 -2
  135. package/src/superlocalmemory/parameterization/workflow_miner.py +17 -0
  136. package/src/superlocalmemory/retrieval/ann_index.py +5 -0
  137. package/src/superlocalmemory/retrieval/bm25_channel.py +49 -2
  138. package/src/superlocalmemory/retrieval/engine.py +19 -4
  139. package/src/superlocalmemory/retrieval/fusion.py +4 -1
  140. package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -3
  141. package/src/superlocalmemory/retrieval/remote_reranker.py +47 -22
  142. package/src/superlocalmemory/retrieval/reranker.py +32 -1
  143. package/src/superlocalmemory/retrieval/temporal_channel.py +16 -3
  144. package/src/superlocalmemory/retrieval/temporal_utils.py +107 -0
  145. package/src/superlocalmemory/retrieval/temporal_validity_filter.py +155 -42
  146. package/src/superlocalmemory/retrieval/vector_store.py +214 -8
  147. package/src/superlocalmemory/server/api.py +5 -5
  148. package/src/superlocalmemory/server/egress_policy.py +258 -0
  149. package/src/superlocalmemory/server/rbac_enforce.py +32 -0
  150. package/src/superlocalmemory/server/route_mutations.py +20 -0
  151. package/src/superlocalmemory/server/routes/compliance.py +153 -7
  152. package/src/superlocalmemory/server/routes/data_io.py +43 -2
  153. package/src/superlocalmemory/server/routes/events.py +15 -0
  154. package/src/superlocalmemory/server/routes/memories.py +56 -3
  155. package/src/superlocalmemory/server/routes/mesh.py +82 -1
  156. package/src/superlocalmemory/server/routes/mesh_lock.py +54 -0
  157. package/src/superlocalmemory/server/routes/mesh_state.py +63 -0
  158. package/src/superlocalmemory/server/routes/v3_api.py +50 -27
  159. package/src/superlocalmemory/server/routes/ws.py +86 -0
  160. package/src/superlocalmemory/server/ui.py +6 -6
  161. package/src/superlocalmemory/server/unified_daemon.py +942 -119
  162. package/src/superlocalmemory/storage/_migration_internals.py +568 -0
  163. package/src/superlocalmemory/storage/_schema_version.py +110 -0
  164. package/src/superlocalmemory/storage/database.py +329 -24
  165. package/src/superlocalmemory/storage/embedding_migrator.py +246 -51
  166. package/src/superlocalmemory/storage/erasure_fence.py +45 -0
  167. package/src/superlocalmemory/storage/generation_fence.py +63 -0
  168. package/src/superlocalmemory/storage/migration_runner.py +140 -417
  169. package/src/superlocalmemory/storage/migrations/M009_model_lineage.py +40 -0
  170. package/src/superlocalmemory/storage/migrations/M033_projection_transactions.py +148 -0
  171. package/src/superlocalmemory/storage/migrations/M034_obligation_integrity.py +58 -0
  172. package/src/superlocalmemory/storage/migrations/M035_erasure_receipts.py +113 -0
  173. package/src/superlocalmemory/storage/migrations/M036_vector_row_map.py +107 -0
  174. package/src/superlocalmemory/storage/migrations/M037_manifest_hmac_version.py +162 -0
  175. package/src/superlocalmemory/storage/migrations/{M033_learning_feedback_channel.py → M038_learning_feedback_channel.py} +3 -3
  176. package/src/superlocalmemory/storage/migrations/M039_scene_fact_members.py +137 -0
  177. package/src/superlocalmemory/storage/migrations/__init__.py +4 -2
  178. package/src/superlocalmemory/storage/schema.py +67 -0
  179. package/src/superlocalmemory/storage/write_coordinator.py +125 -0
  180. package/src/superlocalmemory/trust/scorer.py +28 -4
  181. package/src/superlocalmemory/ui/index.html +14 -3
  182. package/src/superlocalmemory/ui/js/auto-settings.js +12 -1
  183. package/src/superlocalmemory/ui/js/brain.js +6 -4
  184. package/src/superlocalmemory/ui/js/compliance.js +66 -12
  185. package/src/superlocalmemory/ui/js/dashboard.js +13 -3
  186. package/src/superlocalmemory/ui/js/feedback.js +8 -2
  187. package/src/superlocalmemory/ui/js/lifecycle.js +7 -1
  188. package/src/superlocalmemory/ui/js/modal.js +272 -5
  189. package/src/superlocalmemory/ui/js/od-backup.js +9 -2
  190. package/src/superlocalmemory/ui/js/od-compliance-ext.js +301 -0
  191. package/src/superlocalmemory/ui/js/od-operations.js +154 -23
  192. package/src/superlocalmemory/ui/js/od-ops-health.js +417 -0
  193. package/src/superlocalmemory/ui/js/od-optimize.js +35 -21
  194. package/src/superlocalmemory/ui/js/od-team.js +9 -2
  195. package/src/superlocalmemory/ui/js/optimize.js +13 -16
  196. package/src/superlocalmemory/ui/js/profiles.js +7 -3
  197. package/src/superlocalmemory/ui/js/settings.js +7 -1
  198. package/src/superlocalmemory/vector/lancedb_backend.py +19 -9
  199. package/src/superlocalmemory/attribution/mathematical_dna.py +0 -235
  200. package/src/superlocalmemory/cli/post_install.py +0 -114
  201. package/src/superlocalmemory/core/clock_monitor.py +0 -45
  202. package/src/superlocalmemory/core/db_pool.py +0 -80
  203. package/src/superlocalmemory/core/error_catalog.py +0 -113
  204. package/src/superlocalmemory/core/loop_watchdog.py +0 -56
  205. package/src/superlocalmemory/core/priority_queue.py +0 -61
  206. package/src/superlocalmemory/core/pruning_engine.py +0 -216
  207. package/src/superlocalmemory/core/queue_dispatcher.py +0 -73
  208. package/src/superlocalmemory/core/slmignore.py +0 -125
  209. package/src/superlocalmemory/infra/heartbeat_monitor.py +0 -140
  210. package/src/superlocalmemory/infra/webhook_dispatcher.py +0 -247
  211. package/src/superlocalmemory/learning/quantization_scheduler.py +0 -320
  212. package/src/superlocalmemory/storage/access_control.py +0 -182
@@ -24,15 +24,17 @@ Hard rules enforced (LLD-07 §7):
24
24
  - MIG3: a failing migration does NOT prevent the runner from attempting
25
25
  the rest, and does NOT raise to the caller — result comes through the
26
26
  returned stats dict.
27
+
28
+ The private apply-engine (``Migration``, the ``migration_log`` primitives,
29
+ ``_apply_single``, and the name→module registry) lives in
30
+ ``superlocalmemory.storage._migration_internals``; this module owns the ordered
31
+ catalogue and the public orchestration functions.
27
32
  """
28
33
 
29
34
  from __future__ import annotations
30
35
 
31
- import hashlib
32
36
  import logging
33
37
  import sqlite3
34
- from dataclasses import dataclass, field
35
- from datetime import datetime, timezone
36
38
  from pathlib import Path
37
39
 
38
40
  from superlocalmemory.storage.migrations import (
@@ -129,76 +131,46 @@ from superlocalmemory.storage.migrations import (
129
131
  M032_write_coordinator_admission as _M032,
130
132
  )
131
133
  from superlocalmemory.storage.migrations import (
132
- M033_learning_feedback_channel as _M033,
133
- )
134
-
135
- # Map migration name → module (used for the optional ``verify(conn)`` hook
136
- # that lets the runner detect "already applied" state when an idempotent
137
- # retry would otherwise trigger duplicate-column / duplicate-table errors).
138
- _MODULES = {
139
- _M001.NAME: _M001,
140
- _M002.NAME: _M002,
141
- _M003.NAME: _M003,
142
- _M004.NAME: _M004,
143
- _M005.NAME: _M005,
144
- _M006.NAME: _M006,
145
- _M007.NAME: _M007,
146
- _M009.NAME: _M009,
147
- _M010.NAME: _M010,
148
- _M011.NAME: _M011,
149
- _M012.NAME: _M012,
150
- _M013.NAME: _M013,
151
- _M014.NAME: _M014,
152
- _M015.NAME: _M015,
153
- _M016.NAME: _M016,
154
- _M017.NAME: _M017,
155
- _M018.NAME: _M018,
156
- _M019.NAME: _M019,
157
- _M020.NAME: _M020,
158
- _M021.NAME: _M021,
159
- _M022.NAME: _M022,
160
- _M023.NAME: _M023,
161
- _M024.NAME: _M024,
162
- _M025.NAME: _M025,
163
- _M026.NAME: _M026,
164
- _M027.NAME: _M027,
165
- _M028.NAME: _M028,
166
- _M029.NAME: _M029,
167
- _M030.NAME: _M030,
168
- _M031.NAME: _M031,
169
- _M032.NAME: _M032,
170
- _M033.NAME: _M033,
171
- }
134
+ M033_projection_transactions as _M033,
135
+ )
136
+ from superlocalmemory.storage.migrations import (
137
+ M034_obligation_integrity as _M034,
138
+ )
139
+ from superlocalmemory.storage.migrations import (
140
+ M035_erasure_receipts as _M035,
141
+ )
142
+ from superlocalmemory.storage.migrations import (
143
+ M036_vector_row_map as _M036,
144
+ )
145
+ from superlocalmemory.storage.migrations import (
146
+ M037_manifest_hmac_version as _M037,
147
+ )
148
+ from superlocalmemory.storage.migrations import (
149
+ M038_learning_feedback_channel as _M038,
150
+ )
151
+ from superlocalmemory.storage.migrations import (
152
+ M039_scene_fact_members as _M039,
153
+ )
154
+ from superlocalmemory.storage._schema_version import (
155
+ SUPPORTED_SCHEMA_VERSION,
156
+ SchemaVersionError,
157
+ check_version_or_raise as _check_version_or_raise,
158
+ ensure_schema_version_table as _ensure_schema_version_table,
159
+ write_schema_version as _write_schema_version,
160
+ )
161
+ from superlocalmemory.storage._migration_internals import (
162
+ Migration,
163
+ _MODULES, # noqa: F401 — re-exported for test/introspection compatibility
164
+ _apply_single,
165
+ _connect,
166
+ _db_for,
167
+ _ensure_migration_log,
168
+ _migration_log_exists,
169
+ _read_log,
170
+ )
172
171
 
173
172
  logger = logging.getLogger(__name__)
174
173
 
175
- # Exact historical DDL fingerprints whose resulting schema is intentionally
176
- # accepted by the current migration. Unknown hashes are never reconciled.
177
- _KNOWN_EQUIVALENT_DDL_HASHES: dict[str, frozenset[str]] = {
178
- _M002.NAME: frozenset({
179
- # v3.4.21 hardened copy-forward variant.
180
- "347eeb2ec8aac89f7cbf373da49ac9446be9ed150e6105c382c656cd22426d4b",
181
- # v3.4.22 model_version-default variant shipped through 3.6.x.
182
- "d28666fa1dfa66e6514efd288e6748363513da2255a4cee95d80f233e6728ae7",
183
- }),
184
- _M032.NAME: frozenset({
185
- # Provisional 3.8.6 development ledger: global idempotency_key and
186
- # operation_id uniqueness. Its standalone table is safely rebuilt by
187
- # M032.repair() into the profile-scoped receipt contract.
188
- "e45df41becba3d0c3342eca5ec3bd83aa899eef76943c819d2da73b4ca1625a7",
189
- }),
190
- }
191
-
192
-
193
- @dataclass(frozen=True, slots=True)
194
- class Migration:
195
- """Single migration definition."""
196
-
197
- name: str
198
- db_target: str # 'learning' or 'memory'
199
- ddl: str
200
- dependencies: tuple[str, ...] = field(default_factory=tuple)
201
-
202
174
 
203
175
  # Order matters: M003 creates the log table. The runner handles M003's own
204
176
  # bootstrap (it can't record itself before it exists).
@@ -224,11 +196,6 @@ MIGRATIONS: list[Migration] = [
224
196
  # observations for ShadowTest persistence across daemon restart.
225
197
  Migration(name=_M012.NAME, db_target="learning", ddl=_M012.DDL,
226
198
  dependencies=(_M003.NAME,)),
227
- # M033 adds learning_feedback.channel, which pattern_miner has always
228
- # queried but which no schema ever defined. Its DDL creates the table
229
- # when absent, so it needs no dependency beyond the migration log.
230
- Migration(name=_M033.NAME, db_target="learning", ddl=_M033.DDL,
231
- dependencies=(_M003.NAME,)),
232
199
  Migration(name=_M004.NAME, db_target="memory", ddl=_M004.DDL),
233
200
  # M007 creates pending_outcomes (memory.db, LLD-00 §1.2).
234
201
  Migration(name=_M007.NAME, db_target="memory", ddl=_M007.DDL),
@@ -245,6 +212,20 @@ MIGRATIONS: list[Migration] = [
245
212
  # M032 is standalone and must precede daemon readiness: typed writes use
246
213
  # this append-only receipt ledger for durable idempotency.
247
214
  Migration(name=_M032.NAME, db_target="memory", ddl=_M032.DDL),
215
+ Migration(name=_M033.NAME, db_target="memory", ddl=_M033.DDL),
216
+ Migration(name=_M034.NAME, db_target="memory", ddl=_M034.DDL,
217
+ dependencies=(_M033.NAME,)),
218
+ Migration(name=_M035.NAME, db_target="memory", ddl=_M035.DDL,
219
+ dependencies=(_M033.NAME,)),
220
+ Migration(name=_M036.NAME, db_target="memory", ddl=_M036.DDL,
221
+ dependencies=(_M033.NAME,)),
222
+ Migration(name=_M037.NAME, db_target="memory", ddl=_M037.DDL,
223
+ dependencies=(_M033.NAME, _M035.NAME)),
224
+ # Main-line M033 is renumbered in V4 because V4 already owns M033-M037.
225
+ # It repairs the legacy learning_feedback schema before any reader mines
226
+ # channel patterns.
227
+ Migration(name=_M038.NAME, db_target="learning", ddl=_M038.DDL,
228
+ dependencies=(_M003.NAME,)),
248
229
  # M006 + M011 are deliberately NOT here — see DEFERRED_MIGRATIONS below.
249
230
  ]
250
231
 
@@ -305,333 +286,12 @@ DEFERRED_MIGRATIONS: list[Migration] = [
305
286
  Migration(name=_M029.NAME, db_target="memory", ddl=_M029.DDL),
306
287
  # M030 bounds Entity Explorer pagination and profile-summary ranking.
307
288
  Migration(name=_M030.NAME, db_target="memory", ddl=_M030.DDL),
289
+ # Main-line M034 is renumbered in V4. It must remain deferred because its
290
+ # backfill joins engine-bootstrapped memory_scenes and atomic_facts.
291
+ Migration(name=_M039.NAME, db_target="memory", ddl=_M039.DDL),
308
292
  ]
309
293
 
310
294
 
311
- def _now_iso() -> str:
312
- return datetime.now(timezone.utc).isoformat()
313
-
314
-
315
- def _ddl_hash(ddl: str) -> str:
316
- return hashlib.sha256(ddl.encode("utf-8")).hexdigest()
317
-
318
-
319
- def _connect(db_path: Path) -> sqlite3.Connection:
320
- # isolation_level=None → we manage transactions explicitly via DDL.
321
- conn = sqlite3.connect(db_path, isolation_level=None)
322
- conn.execute("PRAGMA foreign_keys = OFF;")
323
- return conn
324
-
325
-
326
- def _migration_log_exists(conn: sqlite3.Connection) -> bool:
327
- row = conn.execute(
328
- "SELECT name FROM sqlite_master "
329
- "WHERE type='table' AND name='migration_log'"
330
- ).fetchone()
331
- return row is not None
332
-
333
-
334
- def _ensure_migration_log(conn: sqlite3.Connection) -> None:
335
- """Bootstrap the migration_log table on a DB if absent.
336
-
337
- Uses the M003 DDL verbatim so the runner treats migration_log identically
338
- on both learning.db and memory.db.
339
- """
340
- conn.executescript(_M003.DDL)
341
-
342
-
343
- def _get_log_row(conn: sqlite3.Connection, name: str) -> tuple | None:
344
- return conn.execute(
345
- "SELECT name, applied_at, ddl_sha256, rows_affected, status "
346
- "FROM migration_log WHERE name = ?",
347
- (name,),
348
- ).fetchone()
349
-
350
-
351
- def _upsert_log(
352
- conn: sqlite3.Connection,
353
- name: str,
354
- ddl_hash: str,
355
- status: str,
356
- rows_affected: int = 0,
357
- ) -> None:
358
- conn.execute(
359
- "INSERT INTO migration_log "
360
- "(name, applied_at, ddl_sha256, rows_affected, status) "
361
- "VALUES (?, ?, ?, ?, ?) "
362
- "ON CONFLICT(name) DO UPDATE SET "
363
- " applied_at = excluded.applied_at, "
364
- " ddl_sha256 = excluded.ddl_sha256, "
365
- " rows_affected = excluded.rows_affected, "
366
- " status = excluded.status",
367
- (name, _now_iso(), ddl_hash, rows_affected, status),
368
- )
369
-
370
-
371
- def _delete_log(conn: sqlite3.Connection, name: str) -> None:
372
- conn.execute("DELETE FROM migration_log WHERE name = ?", (name,))
373
-
374
-
375
- def _apply_single(
376
- conn: sqlite3.Connection,
377
- migration: Migration,
378
- *,
379
- dry_run: bool,
380
- ) -> tuple[str, str]:
381
- """Apply one migration against ``conn``.
382
-
383
- Returns (outcome, detail) where outcome is one of:
384
- - "applied"
385
- - "skipped"
386
- - "failed"
387
- """
388
- ddl_hash = _ddl_hash(migration.ddl)
389
-
390
- # Bootstrap: if migration_log doesn't exist yet, this MUST be M003.
391
- if not _migration_log_exists(conn):
392
- if migration.name != _M003.NAME:
393
- # Other migrations can't check state → treat as unrecoverable here.
394
- return ("failed",
395
- f"migration_log missing when attempting {migration.name}")
396
- if dry_run:
397
- return ("skipped", "dry-run: would create migration_log")
398
- try:
399
- _ensure_migration_log(conn)
400
- _upsert_log(conn, migration.name, ddl_hash, "complete")
401
- return ("applied", "bootstrapped migration_log")
402
- except sqlite3.Error as exc: # pragma: no cover — defensive
403
- logger.warning("M003 bootstrap failed: %s", exc)
404
- return ("failed", f"bootstrap error: {exc}")
405
-
406
- # M003 specifically — if log already exists, ensure M003's own row is there
407
- # (records the fact that the table was bootstrapped previously).
408
- existing = _get_log_row(conn, migration.name)
409
-
410
- if existing is not None:
411
- _, _, logged_hash, _, status = existing
412
- if status == "complete":
413
- if logged_hash != ddl_hash:
414
- # v3.7.6 (#70): a complete migration whose logged DDL hash no
415
- # longer matches the current text is only a real failure if the
416
- # schema it guarantees is actually absent. Historically-benign
417
- # DDL edits (e.g. M002's V3.4.21 <-> S9-W1 variants that build the
418
- # identical end-state) would otherwise brick readiness forever on
419
- # upgrade. Consult the migration's own verify(); if the schema is
420
- # in place, reconcile the log to the current hash and treat as
421
- # already-applied instead of failing the daemon into permanent
422
- # not_ready. Absent/failing verify keeps the hard failure.
423
- allowed_hashes = _KNOWN_EQUIVALENT_DDL_HASHES.get(
424
- migration.name, frozenset(),
425
- )
426
- mod = _MODULES.get(migration.name)
427
- verify_fn = (
428
- getattr(mod, "verify", None) if mod is not None else None
429
- )
430
- if logged_hash in allowed_hashes and verify_fn is not None:
431
- try:
432
- if verify_fn(conn):
433
- if not dry_run:
434
- try:
435
- _upsert_log(
436
- conn, migration.name, ddl_hash, "complete"
437
- )
438
- except sqlite3.Error: # pragma: no cover
439
- pass
440
- return (
441
- "skipped",
442
- "allowlisted historical DDL reconciled after "
443
- "full schema verification",
444
- )
445
- except sqlite3.Error: # pragma: no cover
446
- pass
447
- if dry_run:
448
- return (
449
- "skipped",
450
- "dry-run: would repair allowlisted historical schema",
451
- )
452
- repair_fn = getattr(mod, "repair", None) if mod is not None else None
453
- if callable(repair_fn):
454
- try:
455
- repair_fn(conn)
456
- if not bool(verify_fn(conn)):
457
- return (
458
- "failed",
459
- f"safe repair did not restore {migration.name}",
460
- )
461
- _upsert_log(conn, migration.name, ddl_hash, "complete")
462
- return (
463
- "applied",
464
- "allowlisted historical schema repaired safely",
465
- )
466
- except sqlite3.Error as exc:
467
- return (
468
- "failed",
469
- f"safe repair failed for {migration.name}: {exc}",
470
- )
471
- detail = (
472
- f"DDL drift detected for {migration.name}: "
473
- f"logged={logged_hash[:8]}... current={ddl_hash[:8]}..."
474
- )
475
- logger.warning(detail)
476
- return ("failed", detail)
477
- # A matching migration-log row is not proof that the promised
478
- # schema still exists. Existing installs can retain a migration log
479
- # while a partial restore drops an additive table or index.
480
- #
481
- # Never replay a historical migration merely because verify()
482
- # fails. Some migrations rebuild tables and transform data; replay
483
- # would be destructive (M002 is the canonical example). Only a
484
- # module-supplied repair(conn) hook is allowed to reconcile a
485
- # completed migration's end-state.
486
- mod = _MODULES.get(migration.name)
487
- verify_fn = (
488
- getattr(mod, "verify", None) if mod is not None else None
489
- )
490
- if verify_fn is None:
491
- return ("skipped", "already complete")
492
- try:
493
- schema_complete = bool(verify_fn(conn))
494
- except sqlite3.Error as exc:
495
- return (
496
- "failed",
497
- f"schema verification failed for {migration.name}: {exc}",
498
- )
499
- if schema_complete:
500
- return ("skipped", "already complete (schema verified)")
501
- if dry_run:
502
- return (
503
- "skipped",
504
- "dry-run: would repair missing migration end-state",
505
- )
506
- repair_fn = (
507
- getattr(mod, "repair", None) if mod is not None else None
508
- )
509
- if not callable(repair_fn):
510
- detail = (
511
- f"schema incomplete for completed migration "
512
- f"{migration.name}; automatic replay is disabled"
513
- )
514
- logger.warning(detail)
515
- return ("failed", detail)
516
- try:
517
- repair_fn(conn)
518
- except sqlite3.Error as exc:
519
- return (
520
- "failed",
521
- f"safe repair failed for {migration.name}: {exc}",
522
- )
523
- try:
524
- if not bool(verify_fn(conn)):
525
- return (
526
- "failed",
527
- f"safe repair did not restore {migration.name}",
528
- )
529
- except sqlite3.Error as exc:
530
- return (
531
- "failed",
532
- f"post-repair verification failed for "
533
- f"{migration.name}: {exc}",
534
- )
535
- return ("applied", "missing end-state repaired safely")
536
- # status is 'failed' or 'in_progress' → retry from scratch.
537
- if dry_run:
538
- return ("skipped", f"dry-run: would retry (status={status})")
539
- try:
540
- _delete_log(conn, migration.name)
541
- except sqlite3.Error as exc: # pragma: no cover — log table exists
542
- return ("failed", f"cannot clear prior log: {exc}")
543
-
544
- if dry_run:
545
- return ("skipped", "dry-run: would apply")
546
-
547
- # Mark in_progress, execute, update status. If DDL fails we roll our log
548
- # entry to 'failed' so next attempt will retry cleanly.
549
- try:
550
- _upsert_log(conn, migration.name, ddl_hash, "in_progress")
551
- except sqlite3.Error as exc: # pragma: no cover
552
- return ("failed", f"cannot record in_progress: {exc}")
553
-
554
- try:
555
- # A migration module may ship a custom apply(conn) for conditional logic
556
- # that static DDL can't express (e.g. SQLite has no ADD COLUMN IF NOT
557
- # EXISTS, and ALTER on a missing/already-altered table can't be guarded
558
- # in one executescript). If present, it runs instead of the DDL string;
559
- # otherwise the DDL is applied as before. Pure-DDL migrations are
560
- # unaffected.
561
- _mod = _MODULES.get(migration.name)
562
- _apply_fn = getattr(_mod, "apply", None) if _mod is not None else None
563
- if callable(_apply_fn):
564
- _apply_fn(conn)
565
- else:
566
- conn.executescript(migration.ddl)
567
- except sqlite3.Error as exc:
568
- # Best-effort rollback.
569
- try:
570
- conn.execute("ROLLBACK")
571
- except sqlite3.Error: # pragma: no cover — best-effort
572
- pass
573
- # Before marking failed, check if the migration's end-state is
574
- # already in place (e.g. crash-recovery retry against a DB where the
575
- # columns were added in a previous partial apply). If so, this is
576
- # effectively a successful idempotent re-run.
577
- mod = _MODULES.get(migration.name)
578
- verify_fn = getattr(mod, "verify", None) if mod is not None else None
579
- if verify_fn is not None:
580
- try:
581
- if verify_fn(conn):
582
- try:
583
- _upsert_log(conn, migration.name, ddl_hash, "complete")
584
- except sqlite3.Error: # pragma: no cover
585
- pass
586
- return ("applied",
587
- "already applied (verified via schema inspection)")
588
- except sqlite3.Error: # pragma: no cover
589
- pass
590
-
591
- logger.warning("Migration %s failed: %s", migration.name, exc)
592
- try:
593
- _upsert_log(conn, migration.name, ddl_hash, "failed")
594
- except sqlite3.Error: # pragma: no cover
595
- pass
596
- return ("failed", f"{type(exc).__name__}: {exc}")
597
-
598
- # S9-W1 H-DATA-01: optional post-DDL Python hook. Runs inside the same
599
- # connection (same DB file) after the DDL commits. Used by M002 to
600
- # backfill ``bytes_sha256`` on rows copied forward by the new-table
601
- # rename. If the hook raises, the migration is marked failed; the DDL
602
- # is NOT rolled back (already committed) but the runner reports the
603
- # problem so operators can intervene. Non-existent hooks are a no-op.
604
- mod = _MODULES.get(migration.name)
605
- post_hook = getattr(mod, "post_ddl_hook", None) if mod is not None else None
606
- if post_hook is not None:
607
- try:
608
- post_hook(conn)
609
- except Exception as exc: # noqa: BLE001 — report + mark failed
610
- logger.warning(
611
- "Migration %s DDL applied but post_ddl_hook failed: %s",
612
- migration.name, exc,
613
- )
614
- try:
615
- _upsert_log(conn, migration.name, ddl_hash, "failed")
616
- except sqlite3.Error: # pragma: no cover
617
- pass
618
- return ("failed", f"post_ddl_hook: {type(exc).__name__}: {exc}")
619
-
620
- try:
621
- _upsert_log(conn, migration.name, ddl_hash, "complete")
622
- except sqlite3.Error as exc: # pragma: no cover
623
- return ("failed", f"cannot record complete: {exc}")
624
- return ("applied", "ok")
625
-
626
-
627
- def _db_for(target: str, learning_db: Path, memory_db: Path) -> Path:
628
- if target == "learning":
629
- return learning_db
630
- if target == "memory":
631
- return memory_db
632
- raise ValueError(f"unknown db_target: {target}") # pragma: no cover
633
-
634
-
635
295
  def _bootstrap_both_migration_logs(
636
296
  learning_db: Path, memory_db: Path, *, dry_run: bool,
637
297
  ) -> tuple[list[str], dict[str, str]]:
@@ -705,7 +365,17 @@ def apply_all(
705
365
 
706
366
  Idempotent: already-applied migrations are skipped. Non-fatal: any
707
367
  migration that fails is recorded in ``failed`` and the runner moves on.
368
+
369
+ Raises SchemaVersionError before touching any data when the learning DB
370
+ reports a schema_version that exceeds SUPPORTED_SCHEMA_VERSION. This
371
+ prevents silent data corruption when downgrading to an older build.
708
372
  """
373
+ # Non-mutating version check: must run before any write. Both managed
374
+ # databases are validated so a downgrade is detectable regardless of which
375
+ # store carries the newer stamp.
376
+ _check_version_or_raise(learning_db)
377
+ _check_version_or_raise(memory_db)
378
+
709
379
  applied: list[str] = []
710
380
  skipped: list[str] = []
711
381
  failed: list[str] = []
@@ -729,7 +399,17 @@ def apply_all(
729
399
  failed.extend(bs_failed)
730
400
  details.update(bs_details)
731
401
 
402
+ blocked: set[str] = set()
732
403
  for migration in MIGRATIONS:
404
+ # A migration whose declared dependency did not complete must not run
405
+ # against a base schema that is missing that dependency's changes.
406
+ unmet = [d for d in migration.dependencies if d in failed or d in blocked]
407
+ if unmet:
408
+ skipped.append(migration.name)
409
+ blocked.add(migration.name)
410
+ details[migration.name] = "dependency not satisfied: " + ", ".join(unmet)
411
+ continue
412
+
733
413
  db_path = _db_for(migration.db_target, learning_db, memory_db)
734
414
  try:
735
415
  conn = _connect(db_path)
@@ -778,13 +458,32 @@ def apply_deferred(
778
458
  table is still missing, the underlying DDL raises ``no such table`` and
779
459
  the migration is recorded as ``failed`` — safe, the trainer already
780
460
  falls back to the position proxy when M006 hasn't completed.
461
+
462
+ Raises SchemaVersionError before touching any data when either managed
463
+ database reports a schema_version newer than SUPPORTED_SCHEMA_VERSION. This
464
+ path runs after engine init, so it must fail closed on a downgrade exactly
465
+ as ``apply_all`` does rather than write DDL an older build cannot interpret.
781
466
  """
467
+ # Non-mutating downgrade guard: must run before any write, mirroring
468
+ # apply_all. Both managed databases are validated so a newer stamp on
469
+ # either store halts the deferred pass.
470
+ _check_version_or_raise(learning_db)
471
+ _check_version_or_raise(memory_db)
472
+
782
473
  applied: list[str] = []
783
474
  skipped: list[str] = []
784
475
  failed: list[str] = []
785
476
  details: dict[str, str] = {}
786
477
 
478
+ blocked: set[str] = set()
787
479
  for migration in DEFERRED_MIGRATIONS:
480
+ unmet = [d for d in migration.dependencies if d in failed or d in blocked]
481
+ if unmet:
482
+ skipped.append(migration.name)
483
+ blocked.add(migration.name)
484
+ details[migration.name] = "dependency not satisfied: " + ", ".join(unmet)
485
+ continue
486
+
788
487
  db_path = _db_for(migration.db_target, learning_db, memory_db)
789
488
  try:
790
489
  conn = _connect(db_path)
@@ -824,6 +523,46 @@ def apply_deferred(
824
523
  except sqlite3.Error: # pragma: no cover
825
524
  pass
826
525
 
526
+ # The version ceiling is a completion certificate, not an intent marker.
527
+ # M039 is deferred until engine-owned tables exist, so apply_all must not
528
+ # stamp version 39. Stamp both stores only after every eager and deferred
529
+ # migration is recorded complete on its declared target.
530
+ if not failed and not dry_run:
531
+ logs = {
532
+ "learning": _read_log(learning_db),
533
+ "memory": _read_log(memory_db),
534
+ }
535
+ incomplete = [
536
+ migration.name
537
+ for migration in (*MIGRATIONS, *DEFERRED_MIGRATIONS)
538
+ if logs[migration.db_target].get(migration.name) != "complete"
539
+ ]
540
+ if incomplete:
541
+ failed.append("schema_version_stamp")
542
+ details["schema_version_stamp"] = (
543
+ "not stamped; incomplete migrations: " + ", ".join(incomplete)
544
+ )
545
+ else:
546
+ for _stamp_db in (learning_db, memory_db):
547
+ try:
548
+ _stamp_conn = _connect(_stamp_db)
549
+ try:
550
+ _ensure_schema_version_table(_stamp_conn)
551
+ _write_schema_version(
552
+ _stamp_conn, SUPPORTED_SCHEMA_VERSION,
553
+ )
554
+ finally:
555
+ try:
556
+ _stamp_conn.close()
557
+ except sqlite3.Error: # pragma: no cover
558
+ pass
559
+ except sqlite3.Error as exc: # pragma: no cover
560
+ failed.append("schema_version_stamp")
561
+ details["schema_version_stamp"] = (
562
+ f"cannot stamp {_stamp_db}: {exc}"
563
+ )
564
+ break
565
+
827
566
  return {
828
567
  "applied": applied,
829
568
  "skipped": skipped,
@@ -851,28 +590,12 @@ def status(learning_db: Path, memory_db: Path) -> dict[str, str]:
851
590
  return out
852
591
 
853
592
 
854
- def _read_log(db_path: Path) -> dict[str, str]:
855
- try:
856
- conn = sqlite3.connect(db_path)
857
- except sqlite3.Error: # pragma: no cover
858
- return {}
859
- try:
860
- if not _migration_log_exists(conn):
861
- return {}
862
- rows = conn.execute(
863
- "SELECT name, status FROM migration_log"
864
- ).fetchall()
865
- return {name: status for (name, status) in rows}
866
- except sqlite3.Error: # pragma: no cover
867
- return {}
868
- finally:
869
- conn.close()
870
-
871
-
872
593
  __all__ = (
873
594
  "Migration",
874
595
  "MIGRATIONS",
875
596
  "DEFERRED_MIGRATIONS",
597
+ "SUPPORTED_SCHEMA_VERSION",
598
+ "SchemaVersionError",
876
599
  "apply_all",
877
600
  "apply_deferred",
878
601
  "status",