superlocalmemory 3.8.0 → 3.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/CHANGELOG.md +112 -0
  2. package/README.md +32 -120
  3. package/package.json +9 -2
  4. package/plugin/.claude-plugin/plugin.json +1 -2
  5. package/plugin/CLAUDE.md +3 -3
  6. package/plugin/agents/slm-governance-advisor.md +1 -1
  7. package/plugin/agents/slm-loop-runner.md +1 -1
  8. package/plugin/agents/slm-memory-advisor.md +2 -2
  9. package/plugin/agents/slm-optimize-advisor.md +1 -1
  10. package/plugin/requirements.txt +1 -1
  11. package/plugin/skills/slm-cache/SKILL.md +1 -1
  12. package/plugin/skills/slm-compress/SKILL.md +1 -1
  13. package/plugin/skills/slm-governance/SKILL.md +1 -1
  14. package/plugin/skills/slm-graph/SKILL.md +1 -1
  15. package/plugin/skills/slm-loop/SKILL.md +1 -1
  16. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  17. package/plugin/skills/slm-profile/SKILL.md +1 -1
  18. package/plugin/skills/slm-recall/SKILL.md +3 -5
  19. package/plugin/skills/slm-remember/SKILL.md +1 -1
  20. package/plugin/skills/slm-scope/SKILL.md +1 -1
  21. package/plugin/skills/slm-session/SKILL.md +1 -1
  22. package/plugin/skills/slm-status/SKILL.md +1 -1
  23. package/plugin-src/rules/AGENTS.md +2 -1
  24. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  25. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  26. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-recall/SKILL.md +3 -5
  28. package/plugin-src/skills/slm-remember/SKILL.md +1 -1
  29. package/plugin-src/skills/slm-session/SKILL.md +1 -1
  30. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  31. package/pyproject.toml +2 -1
  32. package/scripts/postinstall.js +7 -1
  33. package/src/superlocalmemory/__init__.py +1 -1
  34. package/src/superlocalmemory/cli/commands.py +494 -9
  35. package/src/superlocalmemory/cli/daemon.py +7 -0
  36. package/src/superlocalmemory/cli/loop_cmd.py +2 -7
  37. package/src/superlocalmemory/cli/main.py +72 -7
  38. package/src/superlocalmemory/cli/setup_wizard.py +142 -16
  39. package/src/superlocalmemory/cli/version_banner.py +17 -3
  40. package/src/superlocalmemory/core/backend_orchestrator.py +18 -16
  41. package/src/superlocalmemory/core/component_healer.py +144 -0
  42. package/src/superlocalmemory/core/component_registry.py +487 -0
  43. package/src/superlocalmemory/core/config.py +21 -0
  44. package/src/superlocalmemory/core/embedding_worker.py +4 -5
  45. package/src/superlocalmemory/core/embeddings.py +132 -45
  46. package/src/superlocalmemory/core/engine.py +29 -22
  47. package/src/superlocalmemory/core/engine_ingestion.py +332 -45
  48. package/src/superlocalmemory/core/ingestion_command.py +154 -25
  49. package/src/superlocalmemory/core/injection.py +12 -7
  50. package/src/superlocalmemory/core/maintenance.py +43 -0
  51. package/src/superlocalmemory/core/maintenance_scheduler.py +44 -6
  52. package/src/superlocalmemory/core/recall_pipeline.py +42 -4
  53. package/src/superlocalmemory/core/store_pipeline.py +195 -20
  54. package/src/superlocalmemory/hooks/hook_handlers.py +6 -1
  55. package/src/superlocalmemory/hooks/portable_kit.py +34 -2
  56. package/src/superlocalmemory/learning/model_rollback.py +3 -0
  57. package/src/superlocalmemory/learning/ranker_retrain_online.py +2 -0
  58. package/src/superlocalmemory/learning/reward.py +50 -0
  59. package/src/superlocalmemory/learning/source_quality.py +523 -1
  60. package/src/superlocalmemory/loops/ledger.py +25 -5
  61. package/src/superlocalmemory/mcp/_daemon_proxy.py +6 -2
  62. package/src/superlocalmemory/mcp/_pool_adapter.py +4 -1
  63. package/src/superlocalmemory/mcp/server.py +11 -30
  64. package/src/superlocalmemory/mcp/tools_active.py +1 -1
  65. package/src/superlocalmemory/mcp/tools_core.py +21 -5
  66. package/src/superlocalmemory/mcp/tools_learning.py +2 -2
  67. package/src/superlocalmemory/retrieval/bridge_discovery.py +14 -0
  68. package/src/superlocalmemory/retrieval/engine.py +53 -21
  69. package/src/superlocalmemory/retrieval/reranker.py +3 -4
  70. package/src/superlocalmemory/retrieval/spreading_activation.py +68 -38
  71. package/src/superlocalmemory/server/config_file.py +90 -0
  72. package/src/superlocalmemory/server/origin.py +50 -0
  73. package/src/superlocalmemory/server/routes/backup.py +293 -70
  74. package/src/superlocalmemory/server/routes/behavioral.py +342 -61
  75. package/src/superlocalmemory/server/routes/brain.py +57 -16
  76. package/src/superlocalmemory/server/routes/config_api.py +84 -82
  77. package/src/superlocalmemory/server/routes/entity.py +100 -23
  78. package/src/superlocalmemory/server/routes/evolution.py +103 -100
  79. package/src/superlocalmemory/server/routes/learning.py +286 -105
  80. package/src/superlocalmemory/server/routes/learning_telemetry.py +153 -0
  81. package/src/superlocalmemory/server/routes/memories.py +8 -3
  82. package/src/superlocalmemory/server/routes/mesh.py +121 -32
  83. package/src/superlocalmemory/server/routes/ratelimit.py +33 -25
  84. package/src/superlocalmemory/server/routes/stats.py +93 -155
  85. package/src/superlocalmemory/server/routes/token.py +3 -13
  86. package/src/superlocalmemory/server/routes/v3_api.py +184 -20
  87. package/src/superlocalmemory/server/unified_daemon.py +732 -41
  88. package/src/superlocalmemory/storage/embedding_migrator.py +235 -0
  89. package/src/superlocalmemory/storage/migration_runner.py +79 -1
  90. package/src/superlocalmemory/storage/migrations/M010_evolution_config.py +5 -0
  91. package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +270 -0
  92. package/src/superlocalmemory/storage/migrations/M029_behavioral_history_indexes.py +137 -0
  93. package/src/superlocalmemory/storage/migrations/M030_entity_explorer_indexes.py +93 -0
  94. package/src/superlocalmemory/storage/migrations/__init__.py +4 -0
  95. package/src/superlocalmemory/storage/schema.py +49 -1
  96. package/src/superlocalmemory/storage/schema_v32.py +2 -0
  97. package/src/superlocalmemory/storage/schema_v347.py +4 -0
  98. package/src/superlocalmemory/ui/index.html +6 -8
  99. package/src/superlocalmemory/ui/js/core.js +52 -9
  100. package/src/superlocalmemory/ui/js/dashboard.js +169 -82
  101. package/src/superlocalmemory/ui/js/od-backup.js +156 -65
  102. package/src/superlocalmemory/ui/js/od-brain.js +88 -51
  103. package/src/superlocalmemory/ui/js/od-components.js +147 -0
  104. package/src/superlocalmemory/ui/js/od-entities.js +65 -22
  105. package/src/superlocalmemory/ui/js/od-graph.js +46 -4
  106. package/src/superlocalmemory/ui/js/od-health.js +18 -0
  107. package/src/superlocalmemory/ui/js/od-memories.js +84 -5
  108. package/src/superlocalmemory/ui/js/od-mesh.js +23 -9
  109. package/src/superlocalmemory/ui/js/od-operations.js +36 -0
  110. package/src/superlocalmemory/ui/js/od-settings.js +186 -63
  111. package/src/superlocalmemory/ui/js/od-shell.js +249 -33
  112. package/src/superlocalmemory/ui/js/od-skills.js +44 -17
  113. package/src/superlocalmemory/ui/js/settings.js +15 -1
  114. package/plugin-src/.mcp.json +0 -12
  115. package/plugin-src/agents/slm-governance-advisor.md +0 -80
  116. package/plugin-src/agents/slm-loop-runner.md +0 -71
  117. package/plugin-src/agents/slm-memory-advisor.md +0 -49
  118. package/plugin-src/agents/slm-optimize-advisor.md +0 -44
  119. package/plugin-src/commands/slm-loop.md +0 -31
  120. package/plugin-src/hooks/.gitkeep +0 -0
  121. package/plugin-src/hooks/hooks.json +0 -102
  122. package/plugin-src/manifest.json +0 -30
  123. package/plugin-src/requirements.txt +0 -1
  124. package/plugin-src/rules/CLAUDE.md.fragment +0 -44
  125. package/plugin-src/scripts/ensure-venv.bat +0 -122
  126. package/plugin-src/scripts/ensure-venv.sh +0 -105
  127. package/plugin-src/scripts/slm-launch +0 -62
  128. package/plugin-src/scripts/slm-launch.bat +0 -23
  129. package/plugin-src/settings.json +0 -25
  130. package/plugin-src/skills/slm-governance/SKILL.md +0 -248
  131. package/plugin-src/skills/slm-loop/SKILL.md +0 -99
  132. package/plugin-src/skills/slm-mesh/SKILL.md +0 -282
  133. package/plugin-src/skills/slm-profile/SKILL.md +0 -148
  134. package/plugin-src/skills/slm-scope/SKILL.md +0 -176
@@ -26,6 +26,23 @@ if TYPE_CHECKING:
26
26
 
27
27
  logger = logging.getLogger(__name__)
28
28
 
29
+ # ---------------------------------------------------------------------------
30
+ # Backfill constants
31
+ # ---------------------------------------------------------------------------
32
+
33
+ #: Default batch size for backfill_missing_embeddings.
34
+ _BACKFILL_BATCH_SIZE = 50
35
+
36
+ #: Max characters embedded per fact during backfill. The embedding model
37
+ #: (nomic-embed-text-v1.5) truncates at ~8192 tokens anyway, but a raw
38
+ #: oversized document (observed up to 107 KB on a real DB) makes the shared
39
+ #: single-worker embedder busy for 15-20s on ONE fact — starving foreground
40
+ #: recall during a self-heal pass. Bounding the input keeps every fact's embed
41
+ #: fast and the worker responsive; the leading slice captures the fact's gist
42
+ #: for semantic recall. Facts this large are documents that were almost
43
+ #: certainly NULL because they failed to embed at ingestion for the same reason.
44
+ _MAX_EMBED_CHARS = 8000
45
+
29
46
  # Sentinel stored in config.json when no model has been set yet.
30
47
  _NO_MODEL = ""
31
48
 
@@ -44,6 +61,24 @@ def _model_signature(config: SLMConfig) -> str:
44
61
  return f"{emb.model_name}::{emb.dimension}"
45
62
 
46
63
 
64
+ def _normalize_signature(signature: str) -> str:
65
+ """Normalize a signature for equivalence comparison.
66
+
67
+ v3.8.2 self-healing: the SAME embedding model has been recorded under
68
+ different name strings across releases — notably the HuggingFace org
69
+ prefix drifted (``nomic-ai/nomic-embed-text-v1.5`` vs the bare
70
+ ``nomic-embed-text-v1.5``). A prefix-only difference does NOT change the
71
+ embedding vector space, so it must not trigger a full multi-hour re-embed
72
+ when a non-technical user upgrades. This collapses the model name to its
73
+ basename (segment after the last ``/``) while keeping the ``::dimension``
74
+ suffix — a genuine model change (different basename OR dimension) still
75
+ differs and still triggers migration.
76
+ """
77
+ model, sep, dim = signature.partition("::")
78
+ model = model.rsplit("/", 1)[-1].strip()
79
+ return f"{model}{sep}{dim}" if sep else model
80
+
81
+
47
82
  def _read_stored_signature(config_dir: Path) -> str:
48
83
  """Read the last-used embedding model signature from config.json."""
49
84
  config_path = config_dir / "config.json"
@@ -88,6 +123,19 @@ def check_embedding_migration(config: SLMConfig) -> bool:
88
123
  if stored_sig == current_sig:
89
124
  return False
90
125
 
126
+ # v3.8.2 self-healing: a prefix-only model-name drift (e.g. the nomic-ai/
127
+ # org prefix appearing/disappearing between releases) is the SAME vector
128
+ # space — absorb the transition by refreshing the stored signature to the
129
+ # current form, with NO re-embed. This spares non-technical users a
130
+ # multi-hour full re-index on a cosmetic upgrade.
131
+ if _normalize_signature(stored_sig) == _normalize_signature(current_sig):
132
+ _write_stored_signature(config.base_dir, current_sig)
133
+ logger.info(
134
+ "Embedding signature normalized (no re-embed): %s ~= %s",
135
+ stored_sig, current_sig,
136
+ )
137
+ return False
138
+
91
139
  logger.warning(
92
140
  "Embedding model changed: %s -> %s. Re-indexing required.",
93
141
  stored_sig, current_sig,
@@ -177,3 +225,190 @@ def run_embedding_migration(
177
225
  reindexed, total,
178
226
  )
179
227
  return reindexed
228
+
229
+
230
+ # ---------------------------------------------------------------------------
231
+ # Backfill: embed facts that were NEVER embedded (embedding IS NULL)
232
+ # ---------------------------------------------------------------------------
233
+
234
+ def _count_null_embeddings(
235
+ db: Any,
236
+ profile_id: str,
237
+ all_profiles: bool,
238
+ ) -> int:
239
+ """Return count of atomic_facts rows with NULL embedding."""
240
+ if all_profiles:
241
+ rows = db.execute(
242
+ "SELECT count(*) AS c FROM atomic_facts WHERE embedding IS NULL",
243
+ )
244
+ else:
245
+ rows = db.execute(
246
+ "SELECT count(*) AS c FROM atomic_facts "
247
+ "WHERE embedding IS NULL AND profile_id = ?",
248
+ (profile_id,),
249
+ )
250
+ return int(rows[0]["c"]) if rows else 0
251
+
252
+
253
+ def backfill_missing_embeddings(
254
+ config: "SLMConfig",
255
+ db: Any,
256
+ embedder: Any,
257
+ batch_size: int = _BACKFILL_BATCH_SIZE,
258
+ limit: int | None = None,
259
+ all_profiles: bool = False,
260
+ ) -> dict[str, int]:
261
+ """Embed atomic_facts rows whose ``embedding`` column is NULL.
262
+
263
+ Unlike :func:`run_embedding_migration` (which re-embeds on model-signature
264
+ change), this function handles facts that were *never* embedded — for
265
+ example facts stored while the embedder was unavailable.
266
+
267
+ Resumable and idempotent: re-running after a partial run only processes
268
+ the remaining NULLs. Fail-open per-fact: a single bad fact logs a warning
269
+ and is skipped; the batch continues.
270
+
271
+ Writes mirror :func:`run_embedding_migration` exactly:
272
+ * ``atomic_facts.embedding`` ← ``json.dumps(vector)``
273
+ * ``embedding_metadata`` ← upserted row with current model name + dimension
274
+
275
+ Args:
276
+ config: Active SLMConfig (provides profile_id, model name, dimension).
277
+ db: DatabaseManager (or duck-compatible object with ``.execute()``).
278
+ embedder: Object implementing ``embed_batch(texts) -> list[vec|None]``
279
+ and (optionally) ``embed(text) -> vec|None``. Pass ``None`` to
280
+ make this a no-op (returns zero counts).
281
+ batch_size: Facts per embed_batch() call. Defaults to 50.
282
+ limit: Maximum facts to embed in this call. ``None`` means no cap —
283
+ all NULL-embedding facts are processed. Use a bounded limit for
284
+ the maintenance self-healing path so each pass is quick.
285
+ all_profiles: When ``True``, processes facts from every profile in the
286
+ database. When ``False`` (default), scopes to
287
+ ``config.active_profile``.
288
+
289
+ Returns:
290
+ ``{"scanned": int, "embedded": int, "remaining_null": int}``
291
+
292
+ *scanned*: total NULL-embedding facts found before applying *limit*.
293
+ *embedded*: facts successfully written in this call.
294
+ *remaining_null*: NULL count after the call (includes facts not yet
295
+ reached because of *limit*).
296
+ """
297
+ profile_id = config.active_profile
298
+
299
+ if embedder is None:
300
+ logger.warning(
301
+ "backfill_missing_embeddings: no embedder available — skipping."
302
+ )
303
+ return {"scanned": 0, "embedded": 0, "remaining_null": 0}
304
+
305
+ # ------------------------------------------------------------------
306
+ # 1. Fetch all NULL-embedding facts (cheap query; only reads IDs + content)
307
+ # ------------------------------------------------------------------
308
+ if all_profiles:
309
+ rows = db.execute(
310
+ "SELECT fact_id, content, profile_id FROM atomic_facts "
311
+ "WHERE embedding IS NULL ORDER BY created_at",
312
+ )
313
+ else:
314
+ rows = db.execute(
315
+ "SELECT fact_id, content, profile_id FROM atomic_facts "
316
+ "WHERE embedding IS NULL AND profile_id = ? ORDER BY created_at",
317
+ (profile_id,),
318
+ )
319
+
320
+ facts: list[tuple[str, str, str]] = [
321
+ (dict(r)["fact_id"], dict(r)["content"], dict(r)["profile_id"])
322
+ for r in rows
323
+ ]
324
+ scanned = len(facts)
325
+
326
+ if scanned == 0:
327
+ return {"scanned": 0, "embedded": 0, "remaining_null": 0}
328
+
329
+ # Apply call-level limit (resumability: next call picks up where this left off)
330
+ if limit is not None:
331
+ facts = facts[:limit]
332
+
333
+ current_model = config.embedding.model_name
334
+ current_dim = config.embedding.dimension
335
+ embedded = 0
336
+
337
+ # ------------------------------------------------------------------
338
+ # 2. Batch embed and write back
339
+ # ------------------------------------------------------------------
340
+ for batch_start in range(0, len(facts), batch_size):
341
+ batch = facts[batch_start : batch_start + batch_size]
342
+ # Bound per-fact input so an oversized document doesn't monopolize the
343
+ # shared embedding worker (starving foreground recall during self-heal).
344
+ texts = [(content or "")[:_MAX_EMBED_CHARS] for _, content, _ in batch]
345
+ fact_ids = [fid for fid, _, _ in batch]
346
+ prof_ids = [pid for _, _, pid in batch]
347
+
348
+ # Attempt batch embed; fall back to per-fact on batch failure.
349
+ try:
350
+ vectors: list[Any] = embedder.embed_batch(texts)
351
+ except Exception as exc:
352
+ logger.warning(
353
+ "backfill: batch embed failed for facts %d-%d: %s — "
354
+ "retrying per-fact.",
355
+ batch_start,
356
+ batch_start + len(batch),
357
+ exc,
358
+ )
359
+ vectors = []
360
+ for text in texts:
361
+ try:
362
+ vec = embedder.embed(text)
363
+ vectors.append(vec)
364
+ except Exception as per_fact_exc:
365
+ logger.warning(
366
+ "backfill: per-fact embed failed for '%s...': %s",
367
+ text[:40],
368
+ per_fact_exc,
369
+ )
370
+ vectors.append(None)
371
+
372
+ # Write each successfully-embedded fact back to the DB.
373
+ for fid, vec, pid in zip(fact_ids, vectors, prof_ids):
374
+ if vec is None:
375
+ logger.warning(
376
+ "backfill: null vector for fact %s — skipping.", fid[:16]
377
+ )
378
+ continue
379
+ try:
380
+ embedding_json = json.dumps(vec)
381
+ # Mirror run_embedding_migration's write path exactly.
382
+ db.execute(
383
+ "UPDATE atomic_facts SET embedding = ? WHERE fact_id = ?",
384
+ (embedding_json, fid),
385
+ )
386
+ # Upsert embedding_metadata. NULL-embedding facts have no row
387
+ # here yet, so we INSERT; if a row somehow exists, update it.
388
+ db.execute(
389
+ "INSERT INTO embedding_metadata"
390
+ " (fact_id, profile_id, model_name, dimension)"
391
+ " VALUES (?, ?, ?, ?)"
392
+ " ON CONFLICT(fact_id) DO UPDATE SET"
393
+ " model_name = excluded.model_name",
394
+ (fid, pid, current_model, current_dim),
395
+ )
396
+ embedded += 1
397
+ except Exception as exc:
398
+ logger.warning(
399
+ "backfill: failed to write fact %s: %s", fid[:16], exc
400
+ )
401
+
402
+ # ------------------------------------------------------------------
403
+ # 3. Count remaining NULLs (accounts for the limit; tells caller how
404
+ # many passes remain before full convergence).
405
+ # ------------------------------------------------------------------
406
+ remaining = _count_null_embeddings(db, profile_id, all_profiles)
407
+
408
+ logger.info(
409
+ "Embedding backfill: %d/%d facts embedded, %d remaining NULL.",
410
+ embedded,
411
+ scanned,
412
+ remaining,
413
+ )
414
+ return {"scanned": scanned, "embedded": embedded, "remaining_null": remaining}
@@ -113,6 +113,15 @@ from superlocalmemory.storage.migrations import (
113
113
  from superlocalmemory.storage.migrations import (
114
114
  M027_transferable_patterns_profile as _M027,
115
115
  )
116
+ from superlocalmemory.storage.migrations import (
117
+ M028_fact_entity_associations as _M028,
118
+ )
119
+ from superlocalmemory.storage.migrations import (
120
+ M029_behavioral_history_indexes as _M029,
121
+ )
122
+ from superlocalmemory.storage.migrations import (
123
+ M030_entity_explorer_indexes as _M030,
124
+ )
116
125
 
117
126
  # Map migration name → module (used for the optional ``verify(conn)`` hook
118
127
  # that lets the runner detect "already applied" state when an idempotent
@@ -144,6 +153,9 @@ _MODULES = {
144
153
  _M025.NAME: _M025,
145
154
  _M026.NAME: _M026,
146
155
  _M027.NAME: _M027,
156
+ _M028.NAME: _M028,
157
+ _M029.NAME: _M029,
158
+ _M030.NAME: _M030,
147
159
  }
148
160
 
149
161
  logger = logging.getLogger(__name__)
@@ -218,6 +230,10 @@ MIGRATIONS: list[Migration] = [
218
230
  # position proxy when the column is absent, so a failed deferred apply never
219
231
  # crashes the trainer — it just keeps the old label path.
220
232
  DEFERRED_MIGRATIONS: list[Migration] = [
233
+ # M028 captures an atomic_facts rowid high-water mark before readiness.
234
+ # atomic_facts/canonical_entities are bootstrapped by MemoryEngine.
235
+ Migration(name=_M028.NAME, db_target="memory", ddl=_M028.DDL,
236
+ dependencies=(_M018.NAME,)),
221
237
  Migration(name=_M006.NAME, db_target="memory", ddl=_M006.DDL),
222
238
  # M011 extends atomic_facts + creates memory_archive / memory_merge_log.
223
239
  # atomic_facts is bootstrapped at engine init, so M011 defers alongside M006.
@@ -256,6 +272,10 @@ DEFERRED_MIGRATIONS: list[Migration] = [
256
272
  # consolidation run, not during engine init or apply_all. apply() is a no-op
257
273
  # when the table is absent (first install after the schema change).
258
274
  Migration(name=_M027.NAME, db_target="learning", ddl=_M027.DDL),
275
+ # M029 indexes behavioral tables bootstrapped during engine initialization.
276
+ Migration(name=_M029.NAME, db_target="memory", ddl=_M029.DDL),
277
+ # M030 bounds Entity Explorer pagination and profile-summary ranking.
278
+ Migration(name=_M030.NAME, db_target="memory", ddl=_M030.DDL),
259
279
  ]
260
280
 
261
281
 
@@ -401,7 +421,65 @@ def _apply_single(
401
421
  )
402
422
  logger.warning(detail)
403
423
  return ("failed", detail)
404
- return ("skipped", "already complete")
424
+ # A matching migration-log row is not proof that the promised
425
+ # schema still exists. Existing installs can retain a migration log
426
+ # while a partial restore drops an additive table or index.
427
+ #
428
+ # Never replay a historical migration merely because verify()
429
+ # fails. Some migrations rebuild tables and transform data; replay
430
+ # would be destructive (M002 is the canonical example). Only a
431
+ # module-supplied repair(conn) hook is allowed to reconcile a
432
+ # completed migration's end-state.
433
+ mod = _MODULES.get(migration.name)
434
+ verify_fn = (
435
+ getattr(mod, "verify", None) if mod is not None else None
436
+ )
437
+ if verify_fn is None:
438
+ return ("skipped", "already complete")
439
+ try:
440
+ schema_complete = bool(verify_fn(conn))
441
+ except sqlite3.Error as exc:
442
+ return (
443
+ "failed",
444
+ f"schema verification failed for {migration.name}: {exc}",
445
+ )
446
+ if schema_complete:
447
+ return ("skipped", "already complete (schema verified)")
448
+ if dry_run:
449
+ return (
450
+ "skipped",
451
+ "dry-run: would repair missing migration end-state",
452
+ )
453
+ repair_fn = (
454
+ getattr(mod, "repair", None) if mod is not None else None
455
+ )
456
+ if not callable(repair_fn):
457
+ detail = (
458
+ f"schema incomplete for completed migration "
459
+ f"{migration.name}; automatic replay is disabled"
460
+ )
461
+ logger.warning(detail)
462
+ return ("failed", detail)
463
+ try:
464
+ repair_fn(conn)
465
+ except sqlite3.Error as exc:
466
+ return (
467
+ "failed",
468
+ f"safe repair failed for {migration.name}: {exc}",
469
+ )
470
+ try:
471
+ if not bool(verify_fn(conn)):
472
+ return (
473
+ "failed",
474
+ f"safe repair did not restore {migration.name}",
475
+ )
476
+ except sqlite3.Error as exc:
477
+ return (
478
+ "failed",
479
+ f"post-repair verification failed for "
480
+ f"{migration.name}: {exc}",
481
+ )
482
+ return ("applied", "missing end-state repaired safely")
405
483
  # status is 'failed' or 'in_progress' → retry from scratch.
406
484
  if dry_run:
407
485
  return ("skipped", f"dry-run: would retry (status={status})")
@@ -50,6 +50,11 @@ def verify(conn: sqlite3.Connection) -> bool:
50
50
  return _REQUIRED_TABLES.issubset(names)
51
51
 
52
52
 
53
+ def repair(conn: sqlite3.Connection) -> None:
54
+ """Restore additive Skill Evolution tables without replaying other migrations."""
55
+ conn.executescript(DDL)
56
+
57
+
53
58
  DDL = """
54
59
  CREATE TABLE IF NOT EXISTS evolution_config (
55
60
  profile_id TEXT PRIMARY KEY,
@@ -0,0 +1,270 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later
3
+
4
+ """M028 — normalized fact/entity associations for O(1) ingest effects.
5
+
6
+ The table is both an association index and an idempotency ledger. Its
7
+ composite primary key means a fact can contribute to an entity's ``fact_count``
8
+ once, regardless of ingestion retries or how many operations converge on the
9
+ same consolidated fact.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import sqlite3
16
+ from datetime import UTC, datetime
17
+ from pathlib import Path
18
+
19
+ NAME = "M028_fact_entity_associations"
20
+ DB_TARGET = "memory"
21
+
22
+ DDL = """
23
+ CREATE TABLE IF NOT EXISTS fact_entity_associations (
24
+ profile_id TEXT NOT NULL,
25
+ fact_id TEXT NOT NULL,
26
+ entity_id TEXT NOT NULL,
27
+ first_operation_id TEXT NOT NULL DEFAULT '',
28
+ count_applied INTEGER NOT NULL DEFAULT 0
29
+ CHECK (count_applied IN (0, 1)),
30
+ created_at TEXT NOT NULL DEFAULT (
31
+ strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
32
+ ),
33
+ PRIMARY KEY (profile_id, fact_id, entity_id),
34
+ FOREIGN KEY (fact_id) REFERENCES atomic_facts(fact_id) ON DELETE CASCADE,
35
+ FOREIGN KEY (entity_id)
36
+ REFERENCES canonical_entities(entity_id) ON DELETE CASCADE
37
+ );
38
+ CREATE INDEX IF NOT EXISTS idx_fact_entity_associations_entity
39
+ ON fact_entity_associations(profile_id, entity_id, fact_id);
40
+ CREATE TABLE IF NOT EXISTS fact_entity_association_repair_state (
41
+ repair_key TEXT PRIMARY KEY,
42
+ state TEXT NOT NULL DEFAULT 'pending'
43
+ CHECK (state IN ('pending', 'running', 'retrying', 'complete')),
44
+ target_fact_rowid INTEGER NOT NULL DEFAULT -1,
45
+ last_fact_rowid INTEGER NOT NULL DEFAULT 0,
46
+ scanned INTEGER NOT NULL DEFAULT 0,
47
+ inserted INTEGER NOT NULL DEFAULT 0,
48
+ last_error TEXT NOT NULL DEFAULT '',
49
+ updated_at TEXT NOT NULL
50
+ );
51
+ """
52
+
53
+
54
+ def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
55
+ return conn.execute(
56
+ "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?",
57
+ (table,),
58
+ ).fetchone() is not None
59
+
60
+
61
+ def apply(conn: sqlite3.Connection) -> None:
62
+ """Install schema and capture a constant-time historical rowid boundary."""
63
+ conn.executescript(DDL)
64
+ association_columns = {
65
+ row[1]
66
+ for row in conn.execute(
67
+ "PRAGMA table_info(fact_entity_associations)"
68
+ ).fetchall()
69
+ }
70
+ if "count_applied" not in association_columns:
71
+ conn.execute(
72
+ "ALTER TABLE fact_entity_associations "
73
+ "ADD COLUMN count_applied INTEGER NOT NULL DEFAULT 0 "
74
+ "CHECK (count_applied IN (0, 1))"
75
+ )
76
+ repair_columns = {
77
+ row[1]
78
+ for row in conn.execute(
79
+ "PRAGMA table_info(fact_entity_association_repair_state)"
80
+ ).fetchall()
81
+ }
82
+ if "target_fact_rowid" not in repair_columns:
83
+ conn.execute(
84
+ "ALTER TABLE fact_entity_association_repair_state "
85
+ "ADD COLUMN target_fact_rowid INTEGER NOT NULL DEFAULT -1"
86
+ )
87
+ target = int(conn.execute(
88
+ "SELECT COALESCE(MAX(rowid), 0) FROM atomic_facts"
89
+ ).fetchone()[0])
90
+ conn.executescript(
91
+ "INSERT OR IGNORE INTO fact_entity_association_repair_state "
92
+ "(repair_key,state,target_fact_rowid,updated_at) "
93
+ f"VALUES ('historical-backfill','pending',{target},'');"
94
+ "UPDATE fact_entity_association_repair_state "
95
+ f"SET target_fact_rowid={target} "
96
+ "WHERE repair_key='historical-backfill' AND target_fact_rowid < 0;"
97
+ )
98
+
99
+
100
+ def _now() -> str:
101
+ return datetime.now(UTC).isoformat()
102
+
103
+
104
+ def _connect(db_path: Path) -> sqlite3.Connection:
105
+ conn = sqlite3.connect(str(db_path), timeout=5)
106
+ conn.row_factory = sqlite3.Row
107
+ conn.execute("PRAGMA busy_timeout=5000")
108
+ conn.execute("PRAGMA foreign_keys=ON")
109
+ return conn
110
+
111
+
112
+ def get_repair_status(db_path: Path) -> dict[str, int | str]:
113
+ """Read durable backfill progress without inferring it from schema."""
114
+ conn = _connect(Path(db_path))
115
+ try:
116
+ row = conn.execute(
117
+ "SELECT state,target_fact_rowid,last_fact_rowid,scanned,inserted,"
118
+ "last_error,updated_at "
119
+ "FROM fact_entity_association_repair_state "
120
+ "WHERE repair_key='historical-backfill'"
121
+ ).fetchone()
122
+ if row is None:
123
+ return {
124
+ "state": "pending", "target_fact_rowid": 0,
125
+ "last_fact_rowid": 0,
126
+ "scanned": 0, "inserted": 0,
127
+ "last_error": "", "updated_at": "",
128
+ }
129
+ return dict(row)
130
+ finally:
131
+ conn.close()
132
+
133
+
134
+ def _entity_ids(raw: object) -> tuple[str, ...]:
135
+ try:
136
+ values = json.loads(str(raw or "[]"))
137
+ except (TypeError, ValueError):
138
+ return ()
139
+ if not isinstance(values, list):
140
+ return ()
141
+ return tuple(dict.fromkeys(str(value) for value in values if value))
142
+
143
+
144
+ def _repair_batch(conn: sqlite3.Connection, batch_size: int) -> dict[str, int | bool]:
145
+ conn.execute("BEGIN IMMEDIATE")
146
+ try:
147
+ status = conn.execute(
148
+ "SELECT last_fact_rowid,target_fact_rowid "
149
+ "FROM fact_entity_association_repair_state "
150
+ "WHERE repair_key='historical-backfill'"
151
+ ).fetchone()
152
+ cursor = int(status["last_fact_rowid"] or 0)
153
+ target = int(status["target_fact_rowid"])
154
+ rows = conn.execute(
155
+ "SELECT rowid,fact_id,profile_id,canonical_entities_json "
156
+ "FROM atomic_facts WHERE rowid>? AND rowid<=? "
157
+ "ORDER BY rowid LIMIT ?",
158
+ (cursor, target, batch_size),
159
+ ).fetchall()
160
+ if not rows:
161
+ conn.execute(
162
+ "UPDATE fact_entity_association_repair_state "
163
+ "SET state='complete',last_error='',updated_at=? "
164
+ "WHERE repair_key='historical-backfill'",
165
+ (_now(),),
166
+ )
167
+ conn.commit()
168
+ return {"scanned": 0, "inserted": 0, "complete": True}
169
+ inserted = 0
170
+ for row in rows:
171
+ for entity_id in _entity_ids(row["canonical_entities_json"]):
172
+ result = conn.execute(
173
+ "INSERT OR IGNORE INTO fact_entity_associations "
174
+ "(profile_id,fact_id,entity_id,first_operation_id,"
175
+ "count_applied) "
176
+ "SELECT ?,?,?,?,? FROM canonical_entities "
177
+ "WHERE profile_id=? AND entity_id=?",
178
+ (
179
+ row["profile_id"], row["fact_id"], entity_id,
180
+ "migration-backfill", 0,
181
+ row["profile_id"], entity_id,
182
+ ),
183
+ )
184
+ inserted += max(0, result.rowcount)
185
+ conn.execute(
186
+ "UPDATE fact_entity_association_repair_state SET "
187
+ "state='running',last_fact_rowid=?,scanned=scanned+?,"
188
+ "inserted=inserted+?,last_error='',updated_at=? "
189
+ "WHERE repair_key='historical-backfill'",
190
+ (int(rows[-1]["rowid"]), len(rows), inserted, _now()),
191
+ )
192
+ conn.commit()
193
+ return {"scanned": len(rows), "inserted": inserted, "complete": False}
194
+ except Exception:
195
+ conn.rollback()
196
+ raise
197
+
198
+
199
+ def repair_fact_entity_associations(
200
+ db_path: Path,
201
+ *,
202
+ batch_size: int = 250,
203
+ max_batches: int = 1,
204
+ ) -> dict[str, int | bool]:
205
+ """Run bounded, restartable short-transaction backfill batches."""
206
+ if batch_size < 1 or max_batches < 1:
207
+ raise ValueError("batch_size and max_batches must be positive")
208
+ totals = {"scanned": 0, "inserted": 0, "complete": False}
209
+ conn = _connect(Path(db_path))
210
+ try:
211
+ for _ in range(max_batches):
212
+ result = _repair_batch(conn, batch_size)
213
+ totals["scanned"] += int(result["scanned"])
214
+ totals["inserted"] += int(result["inserted"])
215
+ totals["complete"] = bool(result["complete"])
216
+ if totals["complete"]:
217
+ break
218
+ return totals
219
+ except sqlite3.Error as exc:
220
+ try:
221
+ conn.execute(
222
+ "UPDATE fact_entity_association_repair_state SET "
223
+ "state='retrying',last_error=?,updated_at=? "
224
+ "WHERE repair_key='historical-backfill'",
225
+ (type(exc).__name__, _now()),
226
+ )
227
+ conn.commit()
228
+ except sqlite3.Error:
229
+ pass
230
+ raise
231
+ finally:
232
+ conn.close()
233
+
234
+
235
+ def verify(conn: sqlite3.Connection) -> bool:
236
+ """Return true when the normalized association contract is indexed."""
237
+ if not (
238
+ _table_exists(conn, "fact_entity_associations")
239
+ and _table_exists(conn, "fact_entity_association_repair_state")
240
+ ):
241
+ return False
242
+ association_columns = {
243
+ row[1]
244
+ for row in conn.execute(
245
+ "PRAGMA table_info(fact_entity_associations)"
246
+ ).fetchall()
247
+ }
248
+ repair_columns = {
249
+ row[1]
250
+ for row in conn.execute(
251
+ "PRAGMA table_info(fact_entity_association_repair_state)"
252
+ ).fetchall()
253
+ }
254
+ if (
255
+ "count_applied" not in association_columns
256
+ or "target_fact_rowid" not in repair_columns
257
+ ):
258
+ return False
259
+ indexes = {
260
+ row[1]
261
+ for row in conn.execute(
262
+ "PRAGMA index_list(fact_entity_associations)"
263
+ ).fetchall()
264
+ }
265
+ return "idx_fact_entity_associations_entity" in indexes
266
+
267
+
268
+ def repair(conn: sqlite3.Connection) -> None:
269
+ """Restore additive M028 schema without recapturing its high-water mark."""
270
+ apply(conn)