superlocalmemory 3.8.13 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/ATTRIBUTION.md +4 -4
  2. package/CHANGELOG.md +113 -121
  3. package/README.md +65 -63
  4. package/docs/pi-dev-integration.md +1 -1
  5. package/package.json +6 -1
  6. package/plugin/.claude-plugin/plugin.json +1 -1
  7. package/plugin/CLAUDE.md +3 -3
  8. package/plugin/agents/slm-governance-advisor.md +1 -1
  9. package/plugin/agents/slm-loop-runner.md +1 -1
  10. package/plugin/agents/slm-memory-advisor.md +1 -1
  11. package/plugin/agents/slm-optimize-advisor.md +1 -1
  12. package/plugin/requirements.txt +1 -1
  13. package/plugin/skills/slm-cache/SKILL.md +1 -1
  14. package/plugin/skills/slm-compress/SKILL.md +1 -1
  15. package/plugin/skills/slm-governance/SKILL.md +1 -1
  16. package/plugin/skills/slm-graph/SKILL.md +1 -1
  17. package/plugin/skills/slm-loop/SKILL.md +1 -1
  18. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  19. package/plugin/skills/slm-profile/SKILL.md +1 -1
  20. package/plugin/skills/slm-recall/SKILL.md +1 -1
  21. package/plugin/skills/slm-remember/SKILL.md +1 -1
  22. package/plugin/skills/slm-scope/SKILL.md +1 -1
  23. package/plugin/skills/slm-session/SKILL.md +1 -1
  24. package/plugin/skills/slm-status/SKILL.md +1 -1
  25. package/plugin-src/rules/AGENTS.md +1 -1
  26. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-governance/SKILL.md +248 -0
  29. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  30. package/plugin-src/skills/slm-loop/SKILL.md +99 -0
  31. package/plugin-src/skills/slm-mesh/SKILL.md +282 -0
  32. package/plugin-src/skills/slm-profile/SKILL.md +148 -0
  33. package/plugin-src/skills/slm-recall/SKILL.md +1 -1
  34. package/plugin-src/skills/slm-remember/SKILL.md +1 -1
  35. package/plugin-src/skills/slm-scope/SKILL.md +176 -0
  36. package/plugin-src/skills/slm-session/SKILL.md +1 -1
  37. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  38. package/pyproject.toml +11 -4
  39. package/src/superlocalmemory/__init__.py +1 -1
  40. package/src/superlocalmemory/cli/commands.py +125 -11
  41. package/src/superlocalmemory/cli/daemon.py +5 -1
  42. package/src/superlocalmemory/cli/main.py +35 -2
  43. package/src/superlocalmemory/cli/ops_cmd.py +281 -0
  44. package/src/superlocalmemory/cli/setup_wizard.py +1 -1
  45. package/src/superlocalmemory/compliance/audit.py +65 -0
  46. package/src/superlocalmemory/compliance/eu_ai_act.py +27 -57
  47. package/src/superlocalmemory/compliance/gdpr.py +416 -20
  48. package/src/superlocalmemory/compliance/retention.py +74 -22
  49. package/src/superlocalmemory/compliance/scheduler.py +78 -9
  50. package/src/superlocalmemory/core/actor_context.py +166 -0
  51. package/src/superlocalmemory/core/admission.py +549 -0
  52. package/src/superlocalmemory/core/backend_orchestrator.py +23 -10
  53. package/src/superlocalmemory/core/config.py +202 -24
  54. package/src/superlocalmemory/core/consolidation_engine.py +13 -13
  55. package/src/superlocalmemory/core/context_cache.py +28 -0
  56. package/src/superlocalmemory/core/embeddings.py +64 -2
  57. package/src/superlocalmemory/core/engine.py +7 -2
  58. package/src/superlocalmemory/core/engine_ingestion.py +65 -3
  59. package/src/superlocalmemory/core/engine_wiring.py +36 -9
  60. package/src/superlocalmemory/core/ingest_policy.py +38 -0
  61. package/src/superlocalmemory/core/maintenance.py +255 -0
  62. package/src/superlocalmemory/core/modes.py +40 -13
  63. package/src/superlocalmemory/core/mutations.py +437 -44
  64. package/src/superlocalmemory/core/operation_policy.py +92 -0
  65. package/src/superlocalmemory/core/operation_policy_registry.py +542 -0
  66. package/src/superlocalmemory/core/operation_request.py +127 -0
  67. package/src/superlocalmemory/core/ops_remediation.py +542 -0
  68. package/src/superlocalmemory/core/recall_pipeline.py +7 -0
  69. package/src/superlocalmemory/core/remember_runtime.py +202 -4
  70. package/src/superlocalmemory/core/remote_mode.py +20 -5
  71. package/src/superlocalmemory/core/store_pipeline.py +150 -0
  72. package/src/superlocalmemory/core/topic_signature.py +19 -4
  73. package/src/superlocalmemory/core/transactions/__init__.py +78 -0
  74. package/src/superlocalmemory/core/transactions/concrete_owners.py +597 -0
  75. package/src/superlocalmemory/core/transactions/erasure.py +825 -0
  76. package/src/superlocalmemory/core/transactions/manifest.py +255 -0
  77. package/src/superlocalmemory/core/transactions/manifest_key.py +155 -0
  78. package/src/superlocalmemory/core/transactions/obligations.py +272 -0
  79. package/src/superlocalmemory/core/transactions/owners.py +114 -0
  80. package/src/superlocalmemory/core/transactions/reconciler.py +285 -0
  81. package/src/superlocalmemory/core/transactions/service.py +330 -0
  82. package/src/superlocalmemory/core/worker_pool.py +33 -5
  83. package/src/superlocalmemory/encoding/cognitive_consolidator.py +70 -28
  84. package/src/superlocalmemory/encoding/emotional.py +75 -14
  85. package/src/superlocalmemory/encoding/scene_builder.py +115 -13
  86. package/src/superlocalmemory/encoding/temporal_parser.py +4 -0
  87. package/src/superlocalmemory/evolution/blind_verifier.py +11 -4
  88. package/src/superlocalmemory/evolution/evolution_store.py +244 -4
  89. package/src/superlocalmemory/evolution/llm_dispatch.py +40 -0
  90. package/src/superlocalmemory/evolution/model_selection.py +18 -3
  91. package/src/superlocalmemory/evolution/mutation_generator.py +3 -0
  92. package/src/superlocalmemory/evolution/skill_activator.py +270 -0
  93. package/src/superlocalmemory/evolution/skill_evolver.py +281 -59
  94. package/src/superlocalmemory/evolution/types.py +30 -8
  95. package/src/superlocalmemory/graph/cozo_backend.py +17 -9
  96. package/src/superlocalmemory/hooks/auto_invoker.py +2 -1
  97. package/src/superlocalmemory/hooks/auto_recall.py +64 -30
  98. package/src/superlocalmemory/hooks/codex_assets.py +14 -1
  99. package/src/superlocalmemory/infra/backup.py +434 -7
  100. package/src/superlocalmemory/infra/process_reaper.py +18 -0
  101. package/src/superlocalmemory/infra/self_heal.py +401 -0
  102. package/src/superlocalmemory/learning/feedback.py +52 -9
  103. package/src/superlocalmemory/loops/engine.py +10 -0
  104. package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
  105. package/src/superlocalmemory/mcp/http_transport.py +30 -331
  106. package/src/superlocalmemory/mcp/profiles.py +5 -0
  107. package/src/superlocalmemory/mcp/resources.py +8 -0
  108. package/src/superlocalmemory/mcp/server.py +51 -4
  109. package/src/superlocalmemory/mcp/shared.py +19 -0
  110. package/src/superlocalmemory/mcp/tools_active.py +25 -4
  111. package/src/superlocalmemory/mcp/tools_code_graph.py +26 -18
  112. package/src/superlocalmemory/mcp/tools_context.py +50 -8
  113. package/src/superlocalmemory/mcp/tools_core.py +69 -21
  114. package/src/superlocalmemory/mcp/tools_evolution.py +9 -2
  115. package/src/superlocalmemory/mcp/tools_learning.py +21 -10
  116. package/src/superlocalmemory/mcp/tools_loops.py +29 -18
  117. package/src/superlocalmemory/mcp/tools_mesh.py +8 -0
  118. package/src/superlocalmemory/mcp/tools_ops.py +115 -0
  119. package/src/superlocalmemory/mcp/tools_optimize.py +4 -0
  120. package/src/superlocalmemory/mcp/tools_v28.py +10 -3
  121. package/src/superlocalmemory/mcp/tools_v3.py +34 -14
  122. package/src/superlocalmemory/mcp/tools_v33.py +18 -33
  123. package/src/superlocalmemory/mesh/broker.py +124 -46
  124. package/src/superlocalmemory/mesh/broker_security.py +470 -0
  125. package/src/superlocalmemory/mesh/discovery.py +365 -0
  126. package/src/superlocalmemory/mesh/lock_protocol.py +313 -0
  127. package/src/superlocalmemory/mesh/node_identity.py +97 -0
  128. package/src/superlocalmemory/mesh/outbox_remote.py +429 -0
  129. package/src/superlocalmemory/mesh/remote_sync.py +511 -28
  130. package/src/superlocalmemory/mesh/state_sync.py +286 -0
  131. package/src/superlocalmemory/optimize/config/store.py +45 -0
  132. package/src/superlocalmemory/parameterization/cross_project.py +12 -0
  133. package/src/superlocalmemory/parameterization/prompt_injector.py +13 -11
  134. package/src/superlocalmemory/parameterization/prompt_lifecycle.py +8 -2
  135. package/src/superlocalmemory/parameterization/workflow_miner.py +17 -0
  136. package/src/superlocalmemory/retrieval/ann_index.py +5 -0
  137. package/src/superlocalmemory/retrieval/bm25_channel.py +49 -2
  138. package/src/superlocalmemory/retrieval/engine.py +19 -4
  139. package/src/superlocalmemory/retrieval/fusion.py +4 -1
  140. package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -3
  141. package/src/superlocalmemory/retrieval/remote_reranker.py +47 -22
  142. package/src/superlocalmemory/retrieval/reranker.py +32 -1
  143. package/src/superlocalmemory/retrieval/temporal_channel.py +16 -3
  144. package/src/superlocalmemory/retrieval/temporal_utils.py +107 -0
  145. package/src/superlocalmemory/retrieval/temporal_validity_filter.py +155 -42
  146. package/src/superlocalmemory/retrieval/vector_store.py +214 -8
  147. package/src/superlocalmemory/server/api.py +5 -5
  148. package/src/superlocalmemory/server/egress_policy.py +258 -0
  149. package/src/superlocalmemory/server/rbac_enforce.py +32 -0
  150. package/src/superlocalmemory/server/route_mutations.py +20 -0
  151. package/src/superlocalmemory/server/routes/compliance.py +153 -7
  152. package/src/superlocalmemory/server/routes/data_io.py +43 -2
  153. package/src/superlocalmemory/server/routes/events.py +15 -0
  154. package/src/superlocalmemory/server/routes/memories.py +56 -3
  155. package/src/superlocalmemory/server/routes/mesh.py +82 -1
  156. package/src/superlocalmemory/server/routes/mesh_lock.py +54 -0
  157. package/src/superlocalmemory/server/routes/mesh_state.py +63 -0
  158. package/src/superlocalmemory/server/routes/v3_api.py +50 -27
  159. package/src/superlocalmemory/server/routes/ws.py +86 -0
  160. package/src/superlocalmemory/server/ui.py +6 -6
  161. package/src/superlocalmemory/server/unified_daemon.py +942 -119
  162. package/src/superlocalmemory/storage/_migration_internals.py +568 -0
  163. package/src/superlocalmemory/storage/_schema_version.py +110 -0
  164. package/src/superlocalmemory/storage/database.py +329 -24
  165. package/src/superlocalmemory/storage/embedding_migrator.py +246 -51
  166. package/src/superlocalmemory/storage/erasure_fence.py +45 -0
  167. package/src/superlocalmemory/storage/generation_fence.py +63 -0
  168. package/src/superlocalmemory/storage/migration_runner.py +140 -417
  169. package/src/superlocalmemory/storage/migrations/M009_model_lineage.py +40 -0
  170. package/src/superlocalmemory/storage/migrations/M033_projection_transactions.py +148 -0
  171. package/src/superlocalmemory/storage/migrations/M034_obligation_integrity.py +58 -0
  172. package/src/superlocalmemory/storage/migrations/M035_erasure_receipts.py +113 -0
  173. package/src/superlocalmemory/storage/migrations/M036_vector_row_map.py +107 -0
  174. package/src/superlocalmemory/storage/migrations/M037_manifest_hmac_version.py +162 -0
  175. package/src/superlocalmemory/storage/migrations/{M033_learning_feedback_channel.py → M038_learning_feedback_channel.py} +3 -3
  176. package/src/superlocalmemory/storage/migrations/M039_scene_fact_members.py +137 -0
  177. package/src/superlocalmemory/storage/migrations/__init__.py +4 -2
  178. package/src/superlocalmemory/storage/schema.py +67 -0
  179. package/src/superlocalmemory/storage/write_coordinator.py +125 -0
  180. package/src/superlocalmemory/trust/scorer.py +28 -4
  181. package/src/superlocalmemory/ui/index.html +14 -3
  182. package/src/superlocalmemory/ui/js/auto-settings.js +12 -1
  183. package/src/superlocalmemory/ui/js/brain.js +6 -4
  184. package/src/superlocalmemory/ui/js/compliance.js +66 -12
  185. package/src/superlocalmemory/ui/js/dashboard.js +13 -3
  186. package/src/superlocalmemory/ui/js/feedback.js +8 -2
  187. package/src/superlocalmemory/ui/js/lifecycle.js +7 -1
  188. package/src/superlocalmemory/ui/js/modal.js +272 -5
  189. package/src/superlocalmemory/ui/js/od-backup.js +9 -2
  190. package/src/superlocalmemory/ui/js/od-compliance-ext.js +301 -0
  191. package/src/superlocalmemory/ui/js/od-operations.js +154 -23
  192. package/src/superlocalmemory/ui/js/od-ops-health.js +417 -0
  193. package/src/superlocalmemory/ui/js/od-optimize.js +35 -21
  194. package/src/superlocalmemory/ui/js/od-team.js +9 -2
  195. package/src/superlocalmemory/ui/js/optimize.js +13 -16
  196. package/src/superlocalmemory/ui/js/profiles.js +7 -3
  197. package/src/superlocalmemory/ui/js/settings.js +7 -1
  198. package/src/superlocalmemory/vector/lancedb_backend.py +19 -9
  199. package/src/superlocalmemory/attribution/mathematical_dna.py +0 -235
  200. package/src/superlocalmemory/cli/post_install.py +0 -114
  201. package/src/superlocalmemory/core/clock_monitor.py +0 -45
  202. package/src/superlocalmemory/core/db_pool.py +0 -80
  203. package/src/superlocalmemory/core/error_catalog.py +0 -113
  204. package/src/superlocalmemory/core/loop_watchdog.py +0 -56
  205. package/src/superlocalmemory/core/priority_queue.py +0 -61
  206. package/src/superlocalmemory/core/pruning_engine.py +0 -216
  207. package/src/superlocalmemory/core/queue_dispatcher.py +0 -73
  208. package/src/superlocalmemory/core/slmignore.py +0 -125
  209. package/src/superlocalmemory/infra/heartbeat_monitor.py +0 -140
  210. package/src/superlocalmemory/infra/webhook_dispatcher.py +0 -247
  211. package/src/superlocalmemory/learning/quantization_scheduler.py +0 -320
  212. package/src/superlocalmemory/storage/access_control.py +0 -182
@@ -0,0 +1,568 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later - see LICENSE file
3
+ # Part of SuperLocalMemory v3.4.22 — LLD-07 §4
4
+
5
+ """Internal apply-engine for the forward-only migration runner.
6
+
7
+ This module holds the private machinery that ``migration_runner`` builds its
8
+ public ``apply_all`` / ``apply_deferred`` / ``status`` API on top of:
9
+
10
+ - ``Migration`` — the single-migration record type.
11
+ - ``_MODULES`` / ``_KNOWN_EQUIVALENT_DDL_HASHES`` — the name→module registry
12
+ and the allowlist of historically-benign DDL fingerprints.
13
+ - The ``sqlite3`` connection / ``migration_log`` primitives.
14
+ - ``_apply_single`` — the transactional apply-one-migration engine shared by
15
+ both the eager (``apply_all``) and deferred (``apply_deferred``) passes.
16
+
17
+ Nothing here is part of the public surface; ``migration_runner`` re-imports the
18
+ symbols it needs. It carries no dependency on ``migration_runner`` itself, so
19
+ importing it never risks a cycle. The catalogue (the ordered ``MIGRATIONS`` /
20
+ ``DEFERRED_MIGRATIONS`` lists) and the public orchestration functions remain in
21
+ ``migration_runner``.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import hashlib
27
+ import logging
28
+ import sqlite3
29
+ from dataclasses import dataclass, field
30
+ from datetime import datetime, timezone
31
+ from pathlib import Path
32
+
33
+ from superlocalmemory.storage.migrations import (
34
+ M001_add_signal_features_columns as _M001,
35
+ )
36
+ from superlocalmemory.storage.migrations import (
37
+ M002_model_state_history as _M002,
38
+ )
39
+ from superlocalmemory.storage.migrations import (
40
+ M003_migration_log as _M003,
41
+ )
42
+ from superlocalmemory.storage.migrations import (
43
+ M004_cross_platform_sync_log as _M004,
44
+ )
45
+ from superlocalmemory.storage.migrations import (
46
+ M005_bandit_tables as _M005,
47
+ )
48
+ from superlocalmemory.storage.migrations import (
49
+ M006_action_outcomes_reward as _M006,
50
+ )
51
+ from superlocalmemory.storage.migrations import (
52
+ M007_pending_outcomes as _M007,
53
+ )
54
+ from superlocalmemory.storage.migrations import (
55
+ M009_model_lineage as _M009,
56
+ )
57
+ from superlocalmemory.storage.migrations import (
58
+ M010_evolution_config as _M010,
59
+ )
60
+ from superlocalmemory.storage.migrations import (
61
+ M011_archive_and_merge as _M011,
62
+ )
63
+ from superlocalmemory.storage.migrations import (
64
+ M012_shadow_observations as _M012,
65
+ )
66
+ from superlocalmemory.storage.migrations import (
67
+ M013_bi_temporal_columns as _M013,
68
+ )
69
+ from superlocalmemory.storage.migrations import (
70
+ M014_v345_scale_ready as _M014,
71
+ )
72
+ from superlocalmemory.storage.migrations import (
73
+ M015_add_pinned_column as _M015,
74
+ )
75
+ from superlocalmemory.storage.migrations import (
76
+ M016_add_scope_support as _M016,
77
+ )
78
+ from superlocalmemory.storage.migrations import (
79
+ M017_ccq_scope_column as _M017,
80
+ )
81
+ from superlocalmemory.storage.migrations import (
82
+ M018_ingestion_operations as _M018,
83
+ )
84
+ from superlocalmemory.storage.migrations import (
85
+ M019_derivation_lineage as _M019,
86
+ )
87
+ from superlocalmemory.storage.migrations import (
88
+ M020_model_state_integrity as _M020,
89
+ )
90
+ from superlocalmemory.storage.migrations import (
91
+ M021_ingestion_log_profile as _M021,
92
+ )
93
+ from superlocalmemory.storage.migrations import (
94
+ M022_entity_aliases_profile as _M022,
95
+ )
96
+ from superlocalmemory.storage.migrations import (
97
+ M023_mesh_profile_isolation as _M023,
98
+ )
99
+ from superlocalmemory.storage.migrations import (
100
+ M024_rbac_users_roles as _M024,
101
+ )
102
+ from superlocalmemory.storage.migrations import (
103
+ M025_perf_indexes as _M025,
104
+ )
105
+ from superlocalmemory.storage.migrations import (
106
+ M026_rbac_memberships_fk as _M026,
107
+ )
108
+ from superlocalmemory.storage.migrations import (
109
+ M027_transferable_patterns_profile as _M027,
110
+ )
111
+ from superlocalmemory.storage.migrations import (
112
+ M028_fact_entity_associations as _M028,
113
+ )
114
+ from superlocalmemory.storage.migrations import (
115
+ M029_behavioral_history_indexes as _M029,
116
+ )
117
+ from superlocalmemory.storage.migrations import (
118
+ M030_entity_explorer_indexes as _M030,
119
+ )
120
+ from superlocalmemory.storage.migrations import (
121
+ M031_dead_letter_operations as _M031,
122
+ )
123
+ from superlocalmemory.storage.migrations import (
124
+ M032_write_coordinator_admission as _M032,
125
+ )
126
+ from superlocalmemory.storage.migrations import (
127
+ M033_projection_transactions as _M033,
128
+ )
129
+ from superlocalmemory.storage.migrations import (
130
+ M034_obligation_integrity as _M034,
131
+ )
132
+ from superlocalmemory.storage.migrations import (
133
+ M035_erasure_receipts as _M035,
134
+ )
135
+ from superlocalmemory.storage.migrations import (
136
+ M036_vector_row_map as _M036,
137
+ )
138
+ from superlocalmemory.storage.migrations import (
139
+ M037_manifest_hmac_version as _M037,
140
+ )
141
+ from superlocalmemory.storage.migrations import (
142
+ M038_learning_feedback_channel as _M038,
143
+ )
144
+ from superlocalmemory.storage.migrations import (
145
+ M039_scene_fact_members as _M039,
146
+ )
147
+
148
+ # Emit under the runner's logger name so operational log filters that key on
149
+ # "superlocalmemory.storage.migration_runner" keep matching after this split.
150
+ logger = logging.getLogger("superlocalmemory.storage.migration_runner")
151
+
152
+ # Map migration name → module (used for the optional ``verify(conn)`` hook
153
+ # that lets the runner detect "already applied" state when an idempotent
154
+ # retry would otherwise trigger duplicate-column / duplicate-table errors).
155
+ _MODULES = {
156
+ _M001.NAME: _M001,
157
+ _M002.NAME: _M002,
158
+ _M003.NAME: _M003,
159
+ _M004.NAME: _M004,
160
+ _M005.NAME: _M005,
161
+ _M006.NAME: _M006,
162
+ _M007.NAME: _M007,
163
+ _M009.NAME: _M009,
164
+ _M010.NAME: _M010,
165
+ _M011.NAME: _M011,
166
+ _M012.NAME: _M012,
167
+ _M013.NAME: _M013,
168
+ _M014.NAME: _M014,
169
+ _M015.NAME: _M015,
170
+ _M016.NAME: _M016,
171
+ _M017.NAME: _M017,
172
+ _M018.NAME: _M018,
173
+ _M019.NAME: _M019,
174
+ _M020.NAME: _M020,
175
+ _M021.NAME: _M021,
176
+ _M022.NAME: _M022,
177
+ _M023.NAME: _M023,
178
+ _M024.NAME: _M024,
179
+ _M025.NAME: _M025,
180
+ _M026.NAME: _M026,
181
+ _M027.NAME: _M027,
182
+ _M028.NAME: _M028,
183
+ _M029.NAME: _M029,
184
+ _M030.NAME: _M030,
185
+ _M031.NAME: _M031,
186
+ _M032.NAME: _M032,
187
+ _M033.NAME: _M033,
188
+ _M034.NAME: _M034,
189
+ _M035.NAME: _M035,
190
+ _M036.NAME: _M036,
191
+ _M037.NAME: _M037,
192
+ _M038.NAME: _M038,
193
+ _M039.NAME: _M039,
194
+ }
195
+
196
+ # Exact historical DDL fingerprints whose resulting schema is intentionally
197
+ # accepted by the current migration. Unknown hashes are never reconciled.
198
+ _KNOWN_EQUIVALENT_DDL_HASHES: dict[str, frozenset[str]] = {
199
+ _M002.NAME: frozenset({
200
+ # v3.4.21 hardened copy-forward variant.
201
+ "347eeb2ec8aac89f7cbf373da49ac9446be9ed150e6105c382c656cd22426d4b",
202
+ # v3.4.22 model_version-default variant shipped through 3.6.x.
203
+ "d28666fa1dfa66e6514efd288e6748363513da2255a4cee95d80f233e6728ae7",
204
+ }),
205
+ _M032.NAME: frozenset({
206
+ # Provisional 3.8.6 development ledger: global idempotency_key and
207
+ # operation_id uniqueness. Its standalone table is safely rebuilt by
208
+ # M032.repair() into the profile-scoped receipt contract.
209
+ "e45df41becba3d0c3342eca5ec3bd83aa899eef76943c819d2da73b4ca1625a7",
210
+ }),
211
+ }
212
+
213
+
214
+ @dataclass(frozen=True, slots=True)
215
+ class Migration:
216
+ """Single migration definition."""
217
+
218
+ name: str
219
+ db_target: str # 'learning' or 'memory'
220
+ ddl: str
221
+ dependencies: tuple[str, ...] = field(default_factory=tuple)
222
+
223
+
224
+ def _now_iso() -> str:
225
+ return datetime.now(timezone.utc).isoformat()
226
+
227
+
228
+ def _ddl_hash(ddl: str) -> str:
229
+ return hashlib.sha256(ddl.encode("utf-8")).hexdigest()
230
+
231
+
232
+ def _connect(db_path: Path) -> sqlite3.Connection:
233
+ # isolation_level=None → we manage transactions explicitly via DDL.
234
+ conn = sqlite3.connect(db_path, isolation_level=None)
235
+ conn.execute("PRAGMA foreign_keys = OFF;")
236
+ return conn
237
+
238
+
239
+ def _migration_log_exists(conn: sqlite3.Connection) -> bool:
240
+ row = conn.execute(
241
+ "SELECT name FROM sqlite_master "
242
+ "WHERE type='table' AND name='migration_log'"
243
+ ).fetchone()
244
+ return row is not None
245
+
246
+
247
+ def _ensure_migration_log(conn: sqlite3.Connection) -> None:
248
+ """Bootstrap the migration_log table on a DB if absent.
249
+
250
+ Uses the M003 DDL verbatim so the runner treats migration_log identically
251
+ on both learning.db and memory.db.
252
+ """
253
+ conn.executescript(_M003.DDL)
254
+
255
+
256
+ def _get_log_row(conn: sqlite3.Connection, name: str) -> tuple | None:
257
+ return conn.execute(
258
+ "SELECT name, applied_at, ddl_sha256, rows_affected, status "
259
+ "FROM migration_log WHERE name = ?",
260
+ (name,),
261
+ ).fetchone()
262
+
263
+
264
+ def _upsert_log(
265
+ conn: sqlite3.Connection,
266
+ name: str,
267
+ ddl_hash: str,
268
+ status: str,
269
+ rows_affected: int = 0,
270
+ ) -> None:
271
+ conn.execute(
272
+ "INSERT INTO migration_log "
273
+ "(name, applied_at, ddl_sha256, rows_affected, status) "
274
+ "VALUES (?, ?, ?, ?, ?) "
275
+ "ON CONFLICT(name) DO UPDATE SET "
276
+ " applied_at = excluded.applied_at, "
277
+ " ddl_sha256 = excluded.ddl_sha256, "
278
+ " rows_affected = excluded.rows_affected, "
279
+ " status = excluded.status",
280
+ (name, _now_iso(), ddl_hash, rows_affected, status),
281
+ )
282
+
283
+
284
+ def _delete_log(conn: sqlite3.Connection, name: str) -> None:
285
+ conn.execute("DELETE FROM migration_log WHERE name = ?", (name,))
286
+
287
+
288
+ def _apply_single(
289
+ conn: sqlite3.Connection,
290
+ migration: Migration,
291
+ *,
292
+ dry_run: bool,
293
+ ) -> tuple[str, str]:
294
+ """Apply one migration against ``conn``.
295
+
296
+ Returns (outcome, detail) where outcome is one of:
297
+ - "applied"
298
+ - "skipped"
299
+ - "failed"
300
+ """
301
+ ddl_hash = _ddl_hash(migration.ddl)
302
+
303
+ # Bootstrap: if migration_log doesn't exist yet, this MUST be M003.
304
+ if not _migration_log_exists(conn):
305
+ if migration.name != _M003.NAME:
306
+ # Other migrations can't check state → treat as unrecoverable here.
307
+ return ("failed",
308
+ f"migration_log missing when attempting {migration.name}")
309
+ if dry_run:
310
+ return ("skipped", "dry-run: would create migration_log")
311
+ try:
312
+ _ensure_migration_log(conn)
313
+ _upsert_log(conn, migration.name, ddl_hash, "complete")
314
+ return ("applied", "bootstrapped migration_log")
315
+ except sqlite3.Error as exc: # pragma: no cover — defensive
316
+ logger.warning("M003 bootstrap failed: %s", exc)
317
+ return ("failed", f"bootstrap error: {exc}")
318
+
319
+ # M003 specifically — if log already exists, ensure M003's own row is there
320
+ # (records the fact that the table was bootstrapped previously).
321
+ existing = _get_log_row(conn, migration.name)
322
+
323
+ if existing is not None:
324
+ _, _, logged_hash, _, status = existing
325
+ if status == "complete":
326
+ if logged_hash != ddl_hash:
327
+ # v3.7.6 (#70): a complete migration whose logged DDL hash no
328
+ # longer matches the current text is only a real failure if the
329
+ # schema it guarantees is actually absent. Historically-benign
330
+ # DDL edits (e.g. M002's V3.4.21 <-> S9-W1 variants that build the
331
+ # identical end-state) would otherwise brick readiness forever on
332
+ # upgrade. Consult the migration's own verify(); if the schema is
333
+ # in place, reconcile the log to the current hash and treat as
334
+ # already-applied instead of failing the daemon into permanent
335
+ # not_ready. Absent/failing verify keeps the hard failure.
336
+ allowed_hashes = _KNOWN_EQUIVALENT_DDL_HASHES.get(
337
+ migration.name, frozenset(),
338
+ )
339
+ mod = _MODULES.get(migration.name)
340
+ verify_fn = (
341
+ getattr(mod, "verify", None) if mod is not None else None
342
+ )
343
+ if logged_hash in allowed_hashes and verify_fn is not None:
344
+ try:
345
+ if verify_fn(conn):
346
+ if not dry_run:
347
+ try:
348
+ _upsert_log(
349
+ conn, migration.name, ddl_hash, "complete"
350
+ )
351
+ except sqlite3.Error: # pragma: no cover
352
+ pass
353
+ return (
354
+ "skipped",
355
+ "allowlisted historical DDL reconciled after "
356
+ "full schema verification",
357
+ )
358
+ except sqlite3.Error: # pragma: no cover
359
+ pass
360
+ if dry_run:
361
+ return (
362
+ "skipped",
363
+ "dry-run: would repair allowlisted historical schema",
364
+ )
365
+ repair_fn = getattr(mod, "repair", None) if mod is not None else None
366
+ if callable(repair_fn):
367
+ try:
368
+ repair_fn(conn)
369
+ if not bool(verify_fn(conn)):
370
+ return (
371
+ "failed",
372
+ f"safe repair did not restore {migration.name}",
373
+ )
374
+ _upsert_log(conn, migration.name, ddl_hash, "complete")
375
+ return (
376
+ "applied",
377
+ "allowlisted historical schema repaired safely",
378
+ )
379
+ except sqlite3.Error as exc:
380
+ return (
381
+ "failed",
382
+ f"safe repair failed for {migration.name}: {exc}",
383
+ )
384
+ detail = (
385
+ f"DDL drift detected for {migration.name}: "
386
+ f"logged={logged_hash[:8]}... current={ddl_hash[:8]}..."
387
+ )
388
+ logger.warning(detail)
389
+ return ("failed", detail)
390
+ # A matching migration-log row is not proof that the promised
391
+ # schema still exists. Existing installs can retain a migration log
392
+ # while a partial restore drops an additive table or index.
393
+ #
394
+ # Never replay a historical migration merely because verify()
395
+ # fails. Some migrations rebuild tables and transform data; replay
396
+ # would be destructive (M002 is the canonical example). Only a
397
+ # module-supplied repair(conn) hook is allowed to reconcile a
398
+ # completed migration's end-state.
399
+ mod = _MODULES.get(migration.name)
400
+ verify_fn = (
401
+ getattr(mod, "verify", None) if mod is not None else None
402
+ )
403
+ if verify_fn is None:
404
+ return ("skipped", "already complete")
405
+ try:
406
+ schema_complete = bool(verify_fn(conn))
407
+ except sqlite3.Error as exc:
408
+ return (
409
+ "failed",
410
+ f"schema verification failed for {migration.name}: {exc}",
411
+ )
412
+ if schema_complete:
413
+ return ("skipped", "already complete (schema verified)")
414
+ if dry_run:
415
+ return (
416
+ "skipped",
417
+ "dry-run: would repair missing migration end-state",
418
+ )
419
+ repair_fn = (
420
+ getattr(mod, "repair", None) if mod is not None else None
421
+ )
422
+ if not callable(repair_fn):
423
+ detail = (
424
+ f"schema incomplete for completed migration "
425
+ f"{migration.name}; automatic replay is disabled"
426
+ )
427
+ logger.warning(detail)
428
+ return ("failed", detail)
429
+ try:
430
+ repair_fn(conn)
431
+ except sqlite3.Error as exc:
432
+ return (
433
+ "failed",
434
+ f"safe repair failed for {migration.name}: {exc}",
435
+ )
436
+ try:
437
+ if not bool(verify_fn(conn)):
438
+ return (
439
+ "failed",
440
+ f"safe repair did not restore {migration.name}",
441
+ )
442
+ except sqlite3.Error as exc:
443
+ return (
444
+ "failed",
445
+ f"post-repair verification failed for "
446
+ f"{migration.name}: {exc}",
447
+ )
448
+ return ("applied", "missing end-state repaired safely")
449
+ # status is 'failed' or 'in_progress' → retry from scratch.
450
+ if dry_run:
451
+ return ("skipped", f"dry-run: would retry (status={status})")
452
+ try:
453
+ _delete_log(conn, migration.name)
454
+ except sqlite3.Error as exc: # pragma: no cover — log table exists
455
+ return ("failed", f"cannot clear prior log: {exc}")
456
+
457
+ if dry_run:
458
+ return ("skipped", "dry-run: would apply")
459
+
460
+ # Mark in_progress, execute, update status. If DDL fails we roll our log
461
+ # entry to 'failed' so next attempt will retry cleanly.
462
+ try:
463
+ _upsert_log(conn, migration.name, ddl_hash, "in_progress")
464
+ except sqlite3.Error as exc: # pragma: no cover
465
+ return ("failed", f"cannot record in_progress: {exc}")
466
+
467
+ try:
468
+ # A migration module may ship a custom apply(conn) for conditional logic
469
+ # that static DDL can't express (e.g. SQLite has no ADD COLUMN IF NOT
470
+ # EXISTS, and ALTER on a missing/already-altered table can't be guarded
471
+ # in one executescript). If present, it runs instead of the DDL string;
472
+ # otherwise the DDL is applied as before. Pure-DDL migrations are
473
+ # unaffected.
474
+ _mod = _MODULES.get(migration.name)
475
+ _apply_fn = getattr(_mod, "apply", None) if _mod is not None else None
476
+ if callable(_apply_fn):
477
+ _apply_fn(conn)
478
+ else:
479
+ # Atomicity is opt-in per migration: a migration that must be all-or-
480
+ # nothing wraps its own BEGIN/COMMIT (or ships a custom apply()); a
481
+ # bare DDL script is deliberately best-effort so a non-essential
482
+ # trailing statement (e.g. a perf index over a table that may not
483
+ # exist yet) can fail without discarding the essential leading DDL.
484
+ conn.executescript(migration.ddl)
485
+ except sqlite3.Error as exc:
486
+ # Best-effort rollback.
487
+ try:
488
+ conn.execute("ROLLBACK")
489
+ except sqlite3.Error: # pragma: no cover — best-effort
490
+ pass
491
+ # Before marking failed, check if the migration's end-state is
492
+ # already in place (e.g. crash-recovery retry against a DB where the
493
+ # columns were added in a previous partial apply). If so, this is
494
+ # effectively a successful idempotent re-run.
495
+ mod = _MODULES.get(migration.name)
496
+ verify_fn = getattr(mod, "verify", None) if mod is not None else None
497
+ if verify_fn is not None:
498
+ try:
499
+ if verify_fn(conn):
500
+ try:
501
+ _upsert_log(conn, migration.name, ddl_hash, "complete")
502
+ except sqlite3.Error: # pragma: no cover
503
+ pass
504
+ return ("applied",
505
+ "already applied (verified via schema inspection)")
506
+ except sqlite3.Error: # pragma: no cover
507
+ pass
508
+
509
+ logger.warning("Migration %s failed: %s", migration.name, exc)
510
+ try:
511
+ _upsert_log(conn, migration.name, ddl_hash, "failed")
512
+ except sqlite3.Error: # pragma: no cover
513
+ pass
514
+ return ("failed", f"{type(exc).__name__}: {exc}")
515
+
516
+ # S9-W1 H-DATA-01: optional post-DDL Python hook. Runs inside the same
517
+ # connection (same DB file) after the DDL commits. Used by M002 to
518
+ # backfill ``bytes_sha256`` on rows copied forward by the new-table
519
+ # rename. If the hook raises, the migration is marked failed; the DDL
520
+ # is NOT rolled back (already committed) but the runner reports the
521
+ # problem so operators can intervene. Non-existent hooks are a no-op.
522
+ mod = _MODULES.get(migration.name)
523
+ post_hook = getattr(mod, "post_ddl_hook", None) if mod is not None else None
524
+ if post_hook is not None:
525
+ try:
526
+ post_hook(conn)
527
+ except Exception as exc: # noqa: BLE001 — report + mark failed
528
+ logger.warning(
529
+ "Migration %s DDL applied but post_ddl_hook failed: %s",
530
+ migration.name, exc,
531
+ )
532
+ try:
533
+ _upsert_log(conn, migration.name, ddl_hash, "failed")
534
+ except sqlite3.Error: # pragma: no cover
535
+ pass
536
+ return ("failed", f"post_ddl_hook: {type(exc).__name__}: {exc}")
537
+
538
+ try:
539
+ _upsert_log(conn, migration.name, ddl_hash, "complete")
540
+ except sqlite3.Error as exc: # pragma: no cover
541
+ return ("failed", f"cannot record complete: {exc}")
542
+ return ("applied", "ok")
543
+
544
+
545
+ def _db_for(target: str, learning_db: Path, memory_db: Path) -> Path:
546
+ if target == "learning":
547
+ return learning_db
548
+ if target == "memory":
549
+ return memory_db
550
+ raise ValueError(f"unknown db_target: {target}") # pragma: no cover
551
+
552
+
553
+ def _read_log(db_path: Path) -> dict[str, str]:
554
+ try:
555
+ conn = sqlite3.connect(db_path)
556
+ except sqlite3.Error: # pragma: no cover
557
+ return {}
558
+ try:
559
+ if not _migration_log_exists(conn):
560
+ return {}
561
+ rows = conn.execute(
562
+ "SELECT name, status FROM migration_log"
563
+ ).fetchall()
564
+ return {name: status for (name, status) in rows}
565
+ except sqlite3.Error: # pragma: no cover
566
+ return {}
567
+ finally:
568
+ conn.close()
@@ -0,0 +1,110 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later - see LICENSE file
3
+
4
+ """Schema-version singleton: read, write, and version-conflict guard.
5
+
6
+ Encapsulates the one-row ``schema_version`` table written to the learning DB
7
+ after each zero-failure migration run. A stored version that exceeds the
8
+ runner's supported ceiling means a newer build wrote the database and this
9
+ installation must not operate on it.
10
+
11
+ Design:
12
+ - ``_read_schema_version_from_db`` is strictly read-only (no writes).
13
+ - ``_ensure_schema_version_table`` + ``_write_schema_version`` are only
14
+ called by the runner after a confirmed zero-failure apply_all.
15
+ - Missing table or unreadable DB → version 0 (legacy; safe to upgrade).
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import sqlite3
21
+ from pathlib import Path
22
+
23
+ #: Highest schema_version this runner can write. Matches the trailing serial
24
+ #: of the latest migration (M039). Increment when adding new migrations or
25
+ #: table-level breaking changes.
26
+ SUPPORTED_SCHEMA_VERSION: int = 39
27
+
28
+
29
+ class SchemaVersionError(RuntimeError):
30
+ """Raised when the DB's recorded schema_version exceeds the supported max.
31
+
32
+ The caller must not attempt any write after catching this — the DB was
33
+ produced by a newer installation and this build is too old for it.
34
+ """
35
+
36
+
37
+ # NOTE: this singleton lives in its OWN table ``slm_schema_version`` rather than
38
+ # ``schema_version``. The legacy multi-row migration-history table
39
+ # ``schema_version`` (columns version/applied_at/description, seeded by
40
+ # schema.py + the schema_v34x migrations) already exists on every real database
41
+ # — fresh installs and upgrades alike — so a ``CREATE TABLE IF NOT EXISTS
42
+ # schema_version`` here would no-op against that legacy shape and every
43
+ # ``id``-keyed read/write would fail ("no column named id"). A distinct table
44
+ # name decouples the version-ceiling guard from that legacy history table.
45
+ _SCHEMA_VERSION_DDL = """\
46
+ CREATE TABLE IF NOT EXISTS slm_schema_version (
47
+ id INTEGER PRIMARY KEY CHECK (id = 1),
48
+ version INTEGER NOT NULL DEFAULT 0,
49
+ updated_at TEXT NOT NULL DEFAULT (datetime('now'))
50
+ );
51
+ INSERT OR IGNORE INTO slm_schema_version (id, version, updated_at)
52
+ VALUES (1, 0, datetime('now'));
53
+ """
54
+
55
+
56
+ def read_schema_version(db_path: Path) -> int:
57
+ """Read the stored schema_version without any write. Returns 0 if absent."""
58
+ try:
59
+ conn = sqlite3.connect(str(db_path))
60
+ try:
61
+ row = conn.execute(
62
+ "SELECT version FROM slm_schema_version WHERE id = 1"
63
+ ).fetchone()
64
+ return int(row[0]) if row is not None else 0
65
+ except sqlite3.OperationalError:
66
+ return 0 # slm_schema_version table does not exist (legacy DB)
67
+ finally:
68
+ conn.close()
69
+ except sqlite3.Error:
70
+ return 0 # cannot open DB; let the runner surface the open error
71
+
72
+
73
+ def ensure_schema_version_table(conn: sqlite3.Connection) -> None:
74
+ """Create the schema_version table if absent and seed the singleton row."""
75
+ conn.executescript(_SCHEMA_VERSION_DDL)
76
+
77
+
78
+ def write_schema_version(conn: sqlite3.Connection, version: int) -> None:
79
+ """Overwrite the singleton version record."""
80
+ conn.execute(
81
+ "UPDATE slm_schema_version "
82
+ "SET version = ?, updated_at = datetime('now') WHERE id = 1",
83
+ (version,),
84
+ )
85
+
86
+
87
+ def check_version_or_raise(db_path: Path) -> None:
88
+ """Raise SchemaVersionError if the DB's stored version exceeds supported max.
89
+
90
+ Non-mutating. Must be called before any write at startup.
91
+ """
92
+ stored = read_schema_version(db_path)
93
+ if stored > SUPPORTED_SCHEMA_VERSION:
94
+ raise SchemaVersionError(
95
+ f"Database schema_version={stored} exceeds the maximum "
96
+ f"supported version={SUPPORTED_SCHEMA_VERSION}. "
97
+ "This installation is too old for this database. "
98
+ "Upgrade SLM to a version that supports schema_version "
99
+ f"{stored} or higher."
100
+ )
101
+
102
+
103
+ __all__ = (
104
+ "SUPPORTED_SCHEMA_VERSION",
105
+ "SchemaVersionError",
106
+ "read_schema_version",
107
+ "ensure_schema_version_table",
108
+ "write_schema_version",
109
+ "check_version_or_raise",
110
+ )