superlocalmemory 3.8.0 → 3.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/CHANGELOG.md +60 -0
  2. package/README.md +32 -120
  3. package/package.json +9 -2
  4. package/plugin/.claude-plugin/plugin.json +1 -2
  5. package/plugin/CLAUDE.md +3 -3
  6. package/plugin/agents/slm-governance-advisor.md +1 -1
  7. package/plugin/agents/slm-loop-runner.md +1 -1
  8. package/plugin/agents/slm-memory-advisor.md +1 -1
  9. package/plugin/agents/slm-optimize-advisor.md +1 -1
  10. package/plugin/requirements.txt +1 -1
  11. package/plugin/skills/slm-cache/SKILL.md +1 -1
  12. package/plugin/skills/slm-compress/SKILL.md +1 -1
  13. package/plugin/skills/slm-governance/SKILL.md +1 -1
  14. package/plugin/skills/slm-graph/SKILL.md +1 -1
  15. package/plugin/skills/slm-loop/SKILL.md +1 -1
  16. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  17. package/plugin/skills/slm-profile/SKILL.md +1 -1
  18. package/plugin/skills/slm-recall/SKILL.md +1 -1
  19. package/plugin/skills/slm-remember/SKILL.md +1 -1
  20. package/plugin/skills/slm-scope/SKILL.md +1 -1
  21. package/plugin/skills/slm-session/SKILL.md +1 -1
  22. package/plugin/skills/slm-status/SKILL.md +1 -1
  23. package/plugin-src/rules/AGENTS.md +1 -1
  24. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  25. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  26. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-recall/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-remember/SKILL.md +1 -1
  29. package/plugin-src/skills/slm-session/SKILL.md +1 -1
  30. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  31. package/pyproject.toml +2 -1
  32. package/src/superlocalmemory/__init__.py +1 -1
  33. package/src/superlocalmemory/cli/commands.py +134 -7
  34. package/src/superlocalmemory/cli/daemon.py +7 -0
  35. package/src/superlocalmemory/cli/loop_cmd.py +2 -7
  36. package/src/superlocalmemory/cli/main.py +10 -4
  37. package/src/superlocalmemory/cli/version_banner.py +17 -3
  38. package/src/superlocalmemory/core/backend_orchestrator.py +18 -16
  39. package/src/superlocalmemory/core/embedding_worker.py +4 -5
  40. package/src/superlocalmemory/core/embeddings.py +119 -45
  41. package/src/superlocalmemory/core/engine.py +24 -21
  42. package/src/superlocalmemory/core/engine_ingestion.py +332 -45
  43. package/src/superlocalmemory/core/ingestion_command.py +134 -25
  44. package/src/superlocalmemory/core/injection.py +12 -7
  45. package/src/superlocalmemory/core/maintenance_scheduler.py +17 -7
  46. package/src/superlocalmemory/core/recall_pipeline.py +5 -3
  47. package/src/superlocalmemory/core/store_pipeline.py +153 -20
  48. package/src/superlocalmemory/hooks/hook_handlers.py +6 -1
  49. package/src/superlocalmemory/hooks/portable_kit.py +34 -2
  50. package/src/superlocalmemory/learning/model_rollback.py +3 -0
  51. package/src/superlocalmemory/learning/ranker_retrain_online.py +2 -0
  52. package/src/superlocalmemory/learning/reward.py +50 -0
  53. package/src/superlocalmemory/learning/source_quality.py +523 -1
  54. package/src/superlocalmemory/loops/ledger.py +25 -5
  55. package/src/superlocalmemory/mcp/server.py +11 -30
  56. package/src/superlocalmemory/mcp/tools_core.py +4 -3
  57. package/src/superlocalmemory/mcp/tools_learning.py +2 -2
  58. package/src/superlocalmemory/retrieval/engine.py +53 -21
  59. package/src/superlocalmemory/retrieval/reranker.py +3 -4
  60. package/src/superlocalmemory/server/config_file.py +90 -0
  61. package/src/superlocalmemory/server/origin.py +50 -0
  62. package/src/superlocalmemory/server/routes/backup.py +293 -70
  63. package/src/superlocalmemory/server/routes/behavioral.py +336 -59
  64. package/src/superlocalmemory/server/routes/brain.py +57 -16
  65. package/src/superlocalmemory/server/routes/config_api.py +84 -82
  66. package/src/superlocalmemory/server/routes/entity.py +100 -23
  67. package/src/superlocalmemory/server/routes/evolution.py +103 -100
  68. package/src/superlocalmemory/server/routes/learning.py +276 -105
  69. package/src/superlocalmemory/server/routes/learning_telemetry.py +153 -0
  70. package/src/superlocalmemory/server/routes/mesh.py +121 -32
  71. package/src/superlocalmemory/server/routes/ratelimit.py +33 -25
  72. package/src/superlocalmemory/server/routes/stats.py +93 -155
  73. package/src/superlocalmemory/server/routes/token.py +3 -13
  74. package/src/superlocalmemory/server/routes/v3_api.py +64 -20
  75. package/src/superlocalmemory/server/unified_daemon.py +467 -40
  76. package/src/superlocalmemory/storage/migration_runner.py +79 -1
  77. package/src/superlocalmemory/storage/migrations/M010_evolution_config.py +5 -0
  78. package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +270 -0
  79. package/src/superlocalmemory/storage/migrations/M029_behavioral_history_indexes.py +137 -0
  80. package/src/superlocalmemory/storage/migrations/M030_entity_explorer_indexes.py +93 -0
  81. package/src/superlocalmemory/storage/migrations/__init__.py +4 -0
  82. package/src/superlocalmemory/storage/schema.py +49 -1
  83. package/src/superlocalmemory/storage/schema_v32.py +2 -0
  84. package/src/superlocalmemory/storage/schema_v347.py +4 -0
  85. package/src/superlocalmemory/ui/index.html +3 -6
  86. package/src/superlocalmemory/ui/js/core.js +52 -9
  87. package/src/superlocalmemory/ui/js/dashboard.js +169 -82
  88. package/src/superlocalmemory/ui/js/od-backup.js +156 -65
  89. package/src/superlocalmemory/ui/js/od-brain.js +88 -51
  90. package/src/superlocalmemory/ui/js/od-entities.js +22 -22
  91. package/src/superlocalmemory/ui/js/od-graph.js +11 -4
  92. package/src/superlocalmemory/ui/js/od-memories.js +47 -5
  93. package/src/superlocalmemory/ui/js/od-mesh.js +23 -9
  94. package/src/superlocalmemory/ui/js/od-settings.js +113 -59
  95. package/src/superlocalmemory/ui/js/od-shell.js +249 -33
  96. package/src/superlocalmemory/ui/js/od-skills.js +44 -17
  97. package/src/superlocalmemory/ui/js/settings.js +15 -1
  98. package/plugin-src/.mcp.json +0 -12
  99. package/plugin-src/agents/slm-governance-advisor.md +0 -80
  100. package/plugin-src/agents/slm-loop-runner.md +0 -71
  101. package/plugin-src/agents/slm-memory-advisor.md +0 -49
  102. package/plugin-src/agents/slm-optimize-advisor.md +0 -44
  103. package/plugin-src/commands/slm-loop.md +0 -31
  104. package/plugin-src/hooks/.gitkeep +0 -0
  105. package/plugin-src/hooks/hooks.json +0 -102
  106. package/plugin-src/manifest.json +0 -30
  107. package/plugin-src/requirements.txt +0 -1
  108. package/plugin-src/rules/CLAUDE.md.fragment +0 -44
  109. package/plugin-src/scripts/ensure-venv.bat +0 -122
  110. package/plugin-src/scripts/ensure-venv.sh +0 -105
  111. package/plugin-src/scripts/slm-launch +0 -62
  112. package/plugin-src/scripts/slm-launch.bat +0 -23
  113. package/plugin-src/settings.json +0 -25
  114. package/plugin-src/skills/slm-governance/SKILL.md +0 -248
  115. package/plugin-src/skills/slm-loop/SKILL.md +0 -99
  116. package/plugin-src/skills/slm-mesh/SKILL.md +0 -282
  117. package/plugin-src/skills/slm-profile/SKILL.md +0 -148
  118. package/plugin-src/skills/slm-scope/SKILL.md +0 -176
@@ -113,6 +113,15 @@ from superlocalmemory.storage.migrations import (
113
113
  from superlocalmemory.storage.migrations import (
114
114
  M027_transferable_patterns_profile as _M027,
115
115
  )
116
+ from superlocalmemory.storage.migrations import (
117
+ M028_fact_entity_associations as _M028,
118
+ )
119
+ from superlocalmemory.storage.migrations import (
120
+ M029_behavioral_history_indexes as _M029,
121
+ )
122
+ from superlocalmemory.storage.migrations import (
123
+ M030_entity_explorer_indexes as _M030,
124
+ )
116
125
 
117
126
  # Map migration name → module (used for the optional ``verify(conn)`` hook
118
127
  # that lets the runner detect "already applied" state when an idempotent
@@ -144,6 +153,9 @@ _MODULES = {
144
153
  _M025.NAME: _M025,
145
154
  _M026.NAME: _M026,
146
155
  _M027.NAME: _M027,
156
+ _M028.NAME: _M028,
157
+ _M029.NAME: _M029,
158
+ _M030.NAME: _M030,
147
159
  }
148
160
 
149
161
  logger = logging.getLogger(__name__)
@@ -218,6 +230,10 @@ MIGRATIONS: list[Migration] = [
218
230
  # position proxy when the column is absent, so a failed deferred apply never
219
231
  # crashes the trainer — it just keeps the old label path.
220
232
  DEFERRED_MIGRATIONS: list[Migration] = [
233
+ # M028 captures an atomic_facts rowid high-water mark before readiness.
234
+ # atomic_facts/canonical_entities are bootstrapped by MemoryEngine.
235
+ Migration(name=_M028.NAME, db_target="memory", ddl=_M028.DDL,
236
+ dependencies=(_M018.NAME,)),
221
237
  Migration(name=_M006.NAME, db_target="memory", ddl=_M006.DDL),
222
238
  # M011 extends atomic_facts + creates memory_archive / memory_merge_log.
223
239
  # atomic_facts is bootstrapped at engine init, so M011 defers alongside M006.
@@ -256,6 +272,10 @@ DEFERRED_MIGRATIONS: list[Migration] = [
256
272
  # consolidation run, not during engine init or apply_all. apply() is a no-op
257
273
  # when the table is absent (first install after the schema change).
258
274
  Migration(name=_M027.NAME, db_target="learning", ddl=_M027.DDL),
275
+ # M029 indexes behavioral tables bootstrapped during engine initialization.
276
+ Migration(name=_M029.NAME, db_target="memory", ddl=_M029.DDL),
277
+ # M030 bounds Entity Explorer pagination and profile-summary ranking.
278
+ Migration(name=_M030.NAME, db_target="memory", ddl=_M030.DDL),
259
279
  ]
260
280
 
261
281
 
@@ -401,7 +421,65 @@ def _apply_single(
401
421
  )
402
422
  logger.warning(detail)
403
423
  return ("failed", detail)
404
- return ("skipped", "already complete")
424
+ # A matching migration-log row is not proof that the promised
425
+ # schema still exists. Existing installs can retain a migration log
426
+ # while a partial restore drops an additive table or index.
427
+ #
428
+ # Never replay a historical migration merely because verify()
429
+ # fails. Some migrations rebuild tables and transform data; replay
430
+ # would be destructive (M002 is the canonical example). Only a
431
+ # module-supplied repair(conn) hook is allowed to reconcile a
432
+ # completed migration's end-state.
433
+ mod = _MODULES.get(migration.name)
434
+ verify_fn = (
435
+ getattr(mod, "verify", None) if mod is not None else None
436
+ )
437
+ if verify_fn is None:
438
+ return ("skipped", "already complete")
439
+ try:
440
+ schema_complete = bool(verify_fn(conn))
441
+ except sqlite3.Error as exc:
442
+ return (
443
+ "failed",
444
+ f"schema verification failed for {migration.name}: {exc}",
445
+ )
446
+ if schema_complete:
447
+ return ("skipped", "already complete (schema verified)")
448
+ if dry_run:
449
+ return (
450
+ "skipped",
451
+ "dry-run: would repair missing migration end-state",
452
+ )
453
+ repair_fn = (
454
+ getattr(mod, "repair", None) if mod is not None else None
455
+ )
456
+ if not callable(repair_fn):
457
+ detail = (
458
+ f"schema incomplete for completed migration "
459
+ f"{migration.name}; automatic replay is disabled"
460
+ )
461
+ logger.warning(detail)
462
+ return ("failed", detail)
463
+ try:
464
+ repair_fn(conn)
465
+ except sqlite3.Error as exc:
466
+ return (
467
+ "failed",
468
+ f"safe repair failed for {migration.name}: {exc}",
469
+ )
470
+ try:
471
+ if not bool(verify_fn(conn)):
472
+ return (
473
+ "failed",
474
+ f"safe repair did not restore {migration.name}",
475
+ )
476
+ except sqlite3.Error as exc:
477
+ return (
478
+ "failed",
479
+ f"post-repair verification failed for "
480
+ f"{migration.name}: {exc}",
481
+ )
482
+ return ("applied", "missing end-state repaired safely")
405
483
  # status is 'failed' or 'in_progress' → retry from scratch.
406
484
  if dry_run:
407
485
  return ("skipped", f"dry-run: would retry (status={status})")
@@ -50,6 +50,11 @@ def verify(conn: sqlite3.Connection) -> bool:
50
50
  return _REQUIRED_TABLES.issubset(names)
51
51
 
52
52
 
53
+ def repair(conn: sqlite3.Connection) -> None:
54
+ """Restore additive Skill Evolution tables without replaying other migrations."""
55
+ conn.executescript(DDL)
56
+
57
+
53
58
  DDL = """
54
59
  CREATE TABLE IF NOT EXISTS evolution_config (
55
60
  profile_id TEXT PRIMARY KEY,
@@ -0,0 +1,270 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later
3
+
4
+ """M028 — normalized fact/entity associations for O(1) ingest effects.
5
+
6
+ The table is both an association index and an idempotency ledger. Its
7
+ composite primary key means a fact can contribute to an entity's ``fact_count``
8
+ once, regardless of ingestion retries or how many operations converge on the
9
+ same consolidated fact.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import sqlite3
16
+ from datetime import UTC, datetime
17
+ from pathlib import Path
18
+
19
+ NAME = "M028_fact_entity_associations"
20
+ DB_TARGET = "memory"
21
+
22
+ DDL = """
23
+ CREATE TABLE IF NOT EXISTS fact_entity_associations (
24
+ profile_id TEXT NOT NULL,
25
+ fact_id TEXT NOT NULL,
26
+ entity_id TEXT NOT NULL,
27
+ first_operation_id TEXT NOT NULL DEFAULT '',
28
+ count_applied INTEGER NOT NULL DEFAULT 0
29
+ CHECK (count_applied IN (0, 1)),
30
+ created_at TEXT NOT NULL DEFAULT (
31
+ strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
32
+ ),
33
+ PRIMARY KEY (profile_id, fact_id, entity_id),
34
+ FOREIGN KEY (fact_id) REFERENCES atomic_facts(fact_id) ON DELETE CASCADE,
35
+ FOREIGN KEY (entity_id)
36
+ REFERENCES canonical_entities(entity_id) ON DELETE CASCADE
37
+ );
38
+ CREATE INDEX IF NOT EXISTS idx_fact_entity_associations_entity
39
+ ON fact_entity_associations(profile_id, entity_id, fact_id);
40
+ CREATE TABLE IF NOT EXISTS fact_entity_association_repair_state (
41
+ repair_key TEXT PRIMARY KEY,
42
+ state TEXT NOT NULL DEFAULT 'pending'
43
+ CHECK (state IN ('pending', 'running', 'retrying', 'complete')),
44
+ target_fact_rowid INTEGER NOT NULL DEFAULT -1,
45
+ last_fact_rowid INTEGER NOT NULL DEFAULT 0,
46
+ scanned INTEGER NOT NULL DEFAULT 0,
47
+ inserted INTEGER NOT NULL DEFAULT 0,
48
+ last_error TEXT NOT NULL DEFAULT '',
49
+ updated_at TEXT NOT NULL
50
+ );
51
+ """
52
+
53
+
54
+ def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
55
+ return conn.execute(
56
+ "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?",
57
+ (table,),
58
+ ).fetchone() is not None
59
+
60
+
61
+ def apply(conn: sqlite3.Connection) -> None:
62
+ """Install schema and capture a constant-time historical rowid boundary."""
63
+ conn.executescript(DDL)
64
+ association_columns = {
65
+ row[1]
66
+ for row in conn.execute(
67
+ "PRAGMA table_info(fact_entity_associations)"
68
+ ).fetchall()
69
+ }
70
+ if "count_applied" not in association_columns:
71
+ conn.execute(
72
+ "ALTER TABLE fact_entity_associations "
73
+ "ADD COLUMN count_applied INTEGER NOT NULL DEFAULT 0 "
74
+ "CHECK (count_applied IN (0, 1))"
75
+ )
76
+ repair_columns = {
77
+ row[1]
78
+ for row in conn.execute(
79
+ "PRAGMA table_info(fact_entity_association_repair_state)"
80
+ ).fetchall()
81
+ }
82
+ if "target_fact_rowid" not in repair_columns:
83
+ conn.execute(
84
+ "ALTER TABLE fact_entity_association_repair_state "
85
+ "ADD COLUMN target_fact_rowid INTEGER NOT NULL DEFAULT -1"
86
+ )
87
+ target = int(conn.execute(
88
+ "SELECT COALESCE(MAX(rowid), 0) FROM atomic_facts"
89
+ ).fetchone()[0])
90
+ conn.executescript(
91
+ "INSERT OR IGNORE INTO fact_entity_association_repair_state "
92
+ "(repair_key,state,target_fact_rowid,updated_at) "
93
+ f"VALUES ('historical-backfill','pending',{target},'');"
94
+ "UPDATE fact_entity_association_repair_state "
95
+ f"SET target_fact_rowid={target} "
96
+ "WHERE repair_key='historical-backfill' AND target_fact_rowid < 0;"
97
+ )
98
+
99
+
100
+ def _now() -> str:
101
+ return datetime.now(UTC).isoformat()
102
+
103
+
104
+ def _connect(db_path: Path) -> sqlite3.Connection:
105
+ conn = sqlite3.connect(str(db_path), timeout=5)
106
+ conn.row_factory = sqlite3.Row
107
+ conn.execute("PRAGMA busy_timeout=5000")
108
+ conn.execute("PRAGMA foreign_keys=ON")
109
+ return conn
110
+
111
+
112
+ def get_repair_status(db_path: Path) -> dict[str, int | str]:
113
+ """Read durable backfill progress without inferring it from schema."""
114
+ conn = _connect(Path(db_path))
115
+ try:
116
+ row = conn.execute(
117
+ "SELECT state,target_fact_rowid,last_fact_rowid,scanned,inserted,"
118
+ "last_error,updated_at "
119
+ "FROM fact_entity_association_repair_state "
120
+ "WHERE repair_key='historical-backfill'"
121
+ ).fetchone()
122
+ if row is None:
123
+ return {
124
+ "state": "pending", "target_fact_rowid": 0,
125
+ "last_fact_rowid": 0,
126
+ "scanned": 0, "inserted": 0,
127
+ "last_error": "", "updated_at": "",
128
+ }
129
+ return dict(row)
130
+ finally:
131
+ conn.close()
132
+
133
+
134
+ def _entity_ids(raw: object) -> tuple[str, ...]:
135
+ try:
136
+ values = json.loads(str(raw or "[]"))
137
+ except (TypeError, ValueError):
138
+ return ()
139
+ if not isinstance(values, list):
140
+ return ()
141
+ return tuple(dict.fromkeys(str(value) for value in values if value))
142
+
143
+
144
+ def _repair_batch(conn: sqlite3.Connection, batch_size: int) -> dict[str, int | bool]:
145
+ conn.execute("BEGIN IMMEDIATE")
146
+ try:
147
+ status = conn.execute(
148
+ "SELECT last_fact_rowid,target_fact_rowid "
149
+ "FROM fact_entity_association_repair_state "
150
+ "WHERE repair_key='historical-backfill'"
151
+ ).fetchone()
152
+ cursor = int(status["last_fact_rowid"] or 0)
153
+ target = int(status["target_fact_rowid"])
154
+ rows = conn.execute(
155
+ "SELECT rowid,fact_id,profile_id,canonical_entities_json "
156
+ "FROM atomic_facts WHERE rowid>? AND rowid<=? "
157
+ "ORDER BY rowid LIMIT ?",
158
+ (cursor, target, batch_size),
159
+ ).fetchall()
160
+ if not rows:
161
+ conn.execute(
162
+ "UPDATE fact_entity_association_repair_state "
163
+ "SET state='complete',last_error='',updated_at=? "
164
+ "WHERE repair_key='historical-backfill'",
165
+ (_now(),),
166
+ )
167
+ conn.commit()
168
+ return {"scanned": 0, "inserted": 0, "complete": True}
169
+ inserted = 0
170
+ for row in rows:
171
+ for entity_id in _entity_ids(row["canonical_entities_json"]):
172
+ result = conn.execute(
173
+ "INSERT OR IGNORE INTO fact_entity_associations "
174
+ "(profile_id,fact_id,entity_id,first_operation_id,"
175
+ "count_applied) "
176
+ "SELECT ?,?,?,?,? FROM canonical_entities "
177
+ "WHERE profile_id=? AND entity_id=?",
178
+ (
179
+ row["profile_id"], row["fact_id"], entity_id,
180
+ "migration-backfill", 0,
181
+ row["profile_id"], entity_id,
182
+ ),
183
+ )
184
+ inserted += max(0, result.rowcount)
185
+ conn.execute(
186
+ "UPDATE fact_entity_association_repair_state SET "
187
+ "state='running',last_fact_rowid=?,scanned=scanned+?,"
188
+ "inserted=inserted+?,last_error='',updated_at=? "
189
+ "WHERE repair_key='historical-backfill'",
190
+ (int(rows[-1]["rowid"]), len(rows), inserted, _now()),
191
+ )
192
+ conn.commit()
193
+ return {"scanned": len(rows), "inserted": inserted, "complete": False}
194
+ except Exception:
195
+ conn.rollback()
196
+ raise
197
+
198
+
199
+ def repair_fact_entity_associations(
200
+ db_path: Path,
201
+ *,
202
+ batch_size: int = 250,
203
+ max_batches: int = 1,
204
+ ) -> dict[str, int | bool]:
205
+ """Run bounded, restartable short-transaction backfill batches."""
206
+ if batch_size < 1 or max_batches < 1:
207
+ raise ValueError("batch_size and max_batches must be positive")
208
+ totals = {"scanned": 0, "inserted": 0, "complete": False}
209
+ conn = _connect(Path(db_path))
210
+ try:
211
+ for _ in range(max_batches):
212
+ result = _repair_batch(conn, batch_size)
213
+ totals["scanned"] += int(result["scanned"])
214
+ totals["inserted"] += int(result["inserted"])
215
+ totals["complete"] = bool(result["complete"])
216
+ if totals["complete"]:
217
+ break
218
+ return totals
219
+ except sqlite3.Error as exc:
220
+ try:
221
+ conn.execute(
222
+ "UPDATE fact_entity_association_repair_state SET "
223
+ "state='retrying',last_error=?,updated_at=? "
224
+ "WHERE repair_key='historical-backfill'",
225
+ (type(exc).__name__, _now()),
226
+ )
227
+ conn.commit()
228
+ except sqlite3.Error:
229
+ pass
230
+ raise
231
+ finally:
232
+ conn.close()
233
+
234
+
235
+ def verify(conn: sqlite3.Connection) -> bool:
236
+ """Return true when the normalized association contract is indexed."""
237
+ if not (
238
+ _table_exists(conn, "fact_entity_associations")
239
+ and _table_exists(conn, "fact_entity_association_repair_state")
240
+ ):
241
+ return False
242
+ association_columns = {
243
+ row[1]
244
+ for row in conn.execute(
245
+ "PRAGMA table_info(fact_entity_associations)"
246
+ ).fetchall()
247
+ }
248
+ repair_columns = {
249
+ row[1]
250
+ for row in conn.execute(
251
+ "PRAGMA table_info(fact_entity_association_repair_state)"
252
+ ).fetchall()
253
+ }
254
+ if (
255
+ "count_applied" not in association_columns
256
+ or "target_fact_rowid" not in repair_columns
257
+ ):
258
+ return False
259
+ indexes = {
260
+ row[1]
261
+ for row in conn.execute(
262
+ "PRAGMA index_list(fact_entity_associations)"
263
+ ).fetchall()
264
+ }
265
+ return "idx_fact_entity_associations_entity" in indexes
266
+
267
+
268
+ def repair(conn: sqlite3.Connection) -> None:
269
+ """Restore additive M028 schema without recapturing its high-water mark."""
270
+ apply(conn)
@@ -0,0 +1,137 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later
3
+
4
+ """M029 — profile-scoped indexes for large behavioral histories.
5
+
6
+ Dashboard reads always filter behavioral and outcome history by profile before
7
+ sorting or aggregating. These composite indexes keep those operations bounded
8
+ to one tenant as the shared database grows.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import sqlite3
14
+
15
+ NAME = "M029_behavioral_history_indexes"
16
+ DB_TARGET = "memory"
17
+
18
+ DDL = """
19
+ CREATE INDEX IF NOT EXISTS idx_outcomes_profile_outcome_time
20
+ ON action_outcomes(profile_id, outcome, timestamp DESC);
21
+ CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_time
22
+ ON action_outcomes(profile_id, settled, settled_at DESC);
23
+ CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_cursor
24
+ ON action_outcomes(
25
+ profile_id, settled, COALESCE(settled_at, ''), outcome_id
26
+ );
27
+ CREATE INDEX IF NOT EXISTS idx_assertions_profile_confidence
28
+ ON behavioral_assertions(profile_id, confidence DESC);
29
+ CREATE INDEX IF NOT EXISTS idx_tool_events_profile_created
30
+ ON tool_events(profile_id, created_at DESC);
31
+ CREATE INDEX IF NOT EXISTS idx_soft_prompts_profile_active_category
32
+ ON soft_prompt_templates(profile_id, active, category, prompt_id);
33
+ """
34
+
35
+ _INDEXES: tuple[tuple[str, str, tuple[str, ...]], ...] = (
36
+ (
37
+ "idx_outcomes_profile_outcome_time",
38
+ "action_outcomes",
39
+ ("profile_id", "outcome", "timestamp"),
40
+ ),
41
+ (
42
+ "idx_outcomes_profile_settled_time",
43
+ "action_outcomes",
44
+ ("profile_id", "settled", "settled_at"),
45
+ ),
46
+ (
47
+ "idx_outcomes_profile_settled_cursor",
48
+ "action_outcomes",
49
+ ("profile_id", "settled", "settled_at", "outcome_id"),
50
+ ),
51
+ (
52
+ "idx_assertions_profile_confidence",
53
+ "behavioral_assertions",
54
+ ("profile_id", "confidence"),
55
+ ),
56
+ (
57
+ "idx_tool_events_profile_created",
58
+ "tool_events",
59
+ ("profile_id", "created_at"),
60
+ ),
61
+ (
62
+ "idx_soft_prompts_profile_active_category",
63
+ "soft_prompt_templates",
64
+ ("profile_id", "active", "category", "prompt_id"),
65
+ ),
66
+ )
67
+
68
+
69
+ def _columns(conn: sqlite3.Connection, table: str) -> set[str]:
70
+ return {
71
+ str(row[1])
72
+ for row in conn.execute(f"PRAGMA table_info({table})").fetchall()
73
+ }
74
+
75
+
76
+ def _index_sql(name: str) -> str:
77
+ statements = {
78
+ "idx_outcomes_profile_outcome_time": (
79
+ "CREATE INDEX IF NOT EXISTS idx_outcomes_profile_outcome_time "
80
+ "ON action_outcomes(profile_id, outcome, timestamp DESC)"
81
+ ),
82
+ "idx_outcomes_profile_settled_time": (
83
+ "CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_time "
84
+ "ON action_outcomes(profile_id, settled, settled_at DESC)"
85
+ ),
86
+ "idx_outcomes_profile_settled_cursor": (
87
+ "CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_cursor "
88
+ "ON action_outcomes("
89
+ "profile_id, settled, COALESCE(settled_at, ''), outcome_id)"
90
+ ),
91
+ "idx_assertions_profile_confidence": (
92
+ "CREATE INDEX IF NOT EXISTS idx_assertions_profile_confidence "
93
+ "ON behavioral_assertions(profile_id, confidence DESC)"
94
+ ),
95
+ "idx_tool_events_profile_created": (
96
+ "CREATE INDEX IF NOT EXISTS idx_tool_events_profile_created "
97
+ "ON tool_events(profile_id, created_at DESC)"
98
+ ),
99
+ "idx_soft_prompts_profile_active_category": (
100
+ "CREATE INDEX IF NOT EXISTS "
101
+ "idx_soft_prompts_profile_active_category "
102
+ "ON soft_prompt_templates(profile_id, active, category, prompt_id)"
103
+ ),
104
+ }
105
+ return statements[name]
106
+
107
+
108
+ def apply(conn: sqlite3.Connection) -> None:
109
+ """Add each index supported by the installed runtime schema.
110
+
111
+ Optional v3.2/v3.4.7 tables also declare these indexes in their owning
112
+ schema modules, so a partial bootstrap cannot permanently miss an index
113
+ when those tables are created later.
114
+ """
115
+ for name, table, required_columns in _INDEXES:
116
+ if set(required_columns) <= _columns(conn, table):
117
+ conn.execute(_index_sql(name))
118
+
119
+
120
+ def verify(conn: sqlite3.Connection) -> bool:
121
+ """Require each index whose backing table and columns are present."""
122
+ names = {
123
+ str(row[0])
124
+ for row in conn.execute(
125
+ "SELECT name FROM sqlite_master WHERE type='index'"
126
+ ).fetchall()
127
+ }
128
+ return all(
129
+ name in names
130
+ for name, table, required_columns in _INDEXES
131
+ if set(required_columns) <= _columns(conn, table)
132
+ )
133
+
134
+
135
+ def repair(conn: sqlite3.Connection) -> None:
136
+ """Create only indexes supported by the current optional schema."""
137
+ apply(conn)
@@ -0,0 +1,93 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later
3
+
4
+ """M030 — page-first Entity Explorer indexes for large installations."""
5
+
6
+ from __future__ import annotations
7
+
8
+ import sqlite3
9
+
10
+ NAME = "M030_entity_explorer_indexes"
11
+ DB_TARGET = "memory"
12
+
13
+ DDL = """
14
+ CREATE INDEX IF NOT EXISTS idx_entities_profile_fact_count_id
15
+ ON canonical_entities(profile_id, fact_count DESC, entity_id ASC);
16
+ CREATE INDEX IF NOT EXISTS idx_entities_profile_type_fact_count_id
17
+ ON canonical_entities(
18
+ profile_id, entity_type COLLATE NOCASE, fact_count DESC, entity_id ASC
19
+ );
20
+ CREATE INDEX IF NOT EXISTS idx_entity_profiles_profile_entity_rank
21
+ ON entity_profiles(
22
+ profile_id, entity_id, last_compiled_at DESC,
23
+ project_name COLLATE NOCASE
24
+ );
25
+ """
26
+
27
+ _INDEXES: tuple[tuple[str, str, tuple[str, ...], str], ...] = (
28
+ (
29
+ "idx_entities_profile_fact_count_id",
30
+ "canonical_entities",
31
+ ("profile_id", "fact_count", "entity_id"),
32
+ "CREATE INDEX IF NOT EXISTS idx_entities_profile_fact_count_id "
33
+ "ON canonical_entities(profile_id, fact_count DESC, entity_id ASC)",
34
+ ),
35
+ (
36
+ "idx_entities_profile_type_fact_count_id",
37
+ "canonical_entities",
38
+ ("profile_id", "entity_type", "fact_count", "entity_id"),
39
+ "CREATE INDEX IF NOT EXISTS idx_entities_profile_type_fact_count_id "
40
+ "ON canonical_entities("
41
+ "profile_id, entity_type COLLATE NOCASE, fact_count DESC, entity_id ASC"
42
+ ")",
43
+ ),
44
+ (
45
+ "idx_entity_profiles_profile_entity_rank",
46
+ "entity_profiles",
47
+ (
48
+ "profile_id",
49
+ "entity_id",
50
+ "last_compiled_at",
51
+ "project_name",
52
+ ),
53
+ "CREATE INDEX IF NOT EXISTS idx_entity_profiles_profile_entity_rank "
54
+ "ON entity_profiles("
55
+ "profile_id, entity_id, last_compiled_at DESC, "
56
+ "project_name COLLATE NOCASE"
57
+ ")",
58
+ ),
59
+ )
60
+
61
+
62
+ def _columns(conn: sqlite3.Connection, table: str) -> set[str]:
63
+ return {
64
+ str(row[1])
65
+ for row in conn.execute(f"PRAGMA table_info({table})").fetchall()
66
+ }
67
+
68
+
69
+ def apply(conn: sqlite3.Connection) -> None:
70
+ """Create every index supported by the installed optional schema."""
71
+ for _, table, required_columns, sql in _INDEXES:
72
+ if set(required_columns) <= _columns(conn, table):
73
+ conn.execute(sql)
74
+
75
+
76
+ def verify(conn: sqlite3.Connection) -> bool:
77
+ """Require indexes only when their complete backing schema exists."""
78
+ names = {
79
+ str(row[0])
80
+ for row in conn.execute(
81
+ "SELECT name FROM sqlite_master WHERE type='index'"
82
+ ).fetchall()
83
+ }
84
+ return all(
85
+ name in names
86
+ for name, table, required_columns, _ in _INDEXES
87
+ if set(required_columns) <= _columns(conn, table)
88
+ )
89
+
90
+
91
+ def repair(conn: sqlite3.Connection) -> None:
92
+ """Safely restore a dropped index without replaying data migrations."""
93
+ apply(conn)
@@ -26,6 +26,8 @@ from . import (
26
26
  M015_add_pinned_column,
27
27
  M019_derivation_lineage,
28
28
  M020_model_state_integrity,
29
+ M029_behavioral_history_indexes,
30
+ M030_entity_explorer_indexes,
29
31
  )
30
32
 
31
33
  # ---------------------------------------------------------------------------
@@ -77,6 +79,8 @@ __all__ = (
77
79
  "M015_add_pinned_column",
78
80
  "M019_derivation_lineage",
79
81
  "M020_model_state_integrity",
82
+ "M029_behavioral_history_indexes",
83
+ "M030_entity_explorer_indexes",
80
84
  # Legacy re-exports (backward compat):
81
85
  "CURRENT_SCHEMA_VERSION",
82
86
  "get_schema_version",