superlocalmemory 3.8.0 → 3.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/README.md +32 -120
- package/package.json +9 -2
- package/plugin/.claude-plugin/plugin.json +1 -2
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +2 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +134 -7
- package/src/superlocalmemory/cli/daemon.py +7 -0
- package/src/superlocalmemory/cli/loop_cmd.py +2 -7
- package/src/superlocalmemory/cli/main.py +10 -4
- package/src/superlocalmemory/cli/version_banner.py +17 -3
- package/src/superlocalmemory/core/backend_orchestrator.py +18 -16
- package/src/superlocalmemory/core/embedding_worker.py +4 -5
- package/src/superlocalmemory/core/embeddings.py +119 -45
- package/src/superlocalmemory/core/engine.py +24 -21
- package/src/superlocalmemory/core/engine_ingestion.py +332 -45
- package/src/superlocalmemory/core/ingestion_command.py +134 -25
- package/src/superlocalmemory/core/injection.py +12 -7
- package/src/superlocalmemory/core/maintenance_scheduler.py +17 -7
- package/src/superlocalmemory/core/recall_pipeline.py +5 -3
- package/src/superlocalmemory/core/store_pipeline.py +153 -20
- package/src/superlocalmemory/hooks/hook_handlers.py +6 -1
- package/src/superlocalmemory/hooks/portable_kit.py +34 -2
- package/src/superlocalmemory/learning/model_rollback.py +3 -0
- package/src/superlocalmemory/learning/ranker_retrain_online.py +2 -0
- package/src/superlocalmemory/learning/reward.py +50 -0
- package/src/superlocalmemory/learning/source_quality.py +523 -1
- package/src/superlocalmemory/loops/ledger.py +25 -5
- package/src/superlocalmemory/mcp/server.py +11 -30
- package/src/superlocalmemory/mcp/tools_core.py +4 -3
- package/src/superlocalmemory/mcp/tools_learning.py +2 -2
- package/src/superlocalmemory/retrieval/engine.py +53 -21
- package/src/superlocalmemory/retrieval/reranker.py +3 -4
- package/src/superlocalmemory/server/config_file.py +90 -0
- package/src/superlocalmemory/server/origin.py +50 -0
- package/src/superlocalmemory/server/routes/backup.py +293 -70
- package/src/superlocalmemory/server/routes/behavioral.py +336 -59
- package/src/superlocalmemory/server/routes/brain.py +57 -16
- package/src/superlocalmemory/server/routes/config_api.py +84 -82
- package/src/superlocalmemory/server/routes/entity.py +100 -23
- package/src/superlocalmemory/server/routes/evolution.py +103 -100
- package/src/superlocalmemory/server/routes/learning.py +276 -105
- package/src/superlocalmemory/server/routes/learning_telemetry.py +153 -0
- package/src/superlocalmemory/server/routes/mesh.py +121 -32
- package/src/superlocalmemory/server/routes/ratelimit.py +33 -25
- package/src/superlocalmemory/server/routes/stats.py +93 -155
- package/src/superlocalmemory/server/routes/token.py +3 -13
- package/src/superlocalmemory/server/routes/v3_api.py +64 -20
- package/src/superlocalmemory/server/unified_daemon.py +467 -40
- package/src/superlocalmemory/storage/migration_runner.py +79 -1
- package/src/superlocalmemory/storage/migrations/M010_evolution_config.py +5 -0
- package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +270 -0
- package/src/superlocalmemory/storage/migrations/M029_behavioral_history_indexes.py +137 -0
- package/src/superlocalmemory/storage/migrations/M030_entity_explorer_indexes.py +93 -0
- package/src/superlocalmemory/storage/migrations/__init__.py +4 -0
- package/src/superlocalmemory/storage/schema.py +49 -1
- package/src/superlocalmemory/storage/schema_v32.py +2 -0
- package/src/superlocalmemory/storage/schema_v347.py +4 -0
- package/src/superlocalmemory/ui/index.html +3 -6
- package/src/superlocalmemory/ui/js/core.js +52 -9
- package/src/superlocalmemory/ui/js/dashboard.js +169 -82
- package/src/superlocalmemory/ui/js/od-backup.js +156 -65
- package/src/superlocalmemory/ui/js/od-brain.js +88 -51
- package/src/superlocalmemory/ui/js/od-entities.js +22 -22
- package/src/superlocalmemory/ui/js/od-graph.js +11 -4
- package/src/superlocalmemory/ui/js/od-memories.js +47 -5
- package/src/superlocalmemory/ui/js/od-mesh.js +23 -9
- package/src/superlocalmemory/ui/js/od-settings.js +113 -59
- package/src/superlocalmemory/ui/js/od-shell.js +249 -33
- package/src/superlocalmemory/ui/js/od-skills.js +44 -17
- package/src/superlocalmemory/ui/js/settings.js +15 -1
- package/plugin-src/.mcp.json +0 -12
- package/plugin-src/agents/slm-governance-advisor.md +0 -80
- package/plugin-src/agents/slm-loop-runner.md +0 -71
- package/plugin-src/agents/slm-memory-advisor.md +0 -49
- package/plugin-src/agents/slm-optimize-advisor.md +0 -44
- package/plugin-src/commands/slm-loop.md +0 -31
- package/plugin-src/hooks/.gitkeep +0 -0
- package/plugin-src/hooks/hooks.json +0 -102
- package/plugin-src/manifest.json +0 -30
- package/plugin-src/requirements.txt +0 -1
- package/plugin-src/rules/CLAUDE.md.fragment +0 -44
- package/plugin-src/scripts/ensure-venv.bat +0 -122
- package/plugin-src/scripts/ensure-venv.sh +0 -105
- package/plugin-src/scripts/slm-launch +0 -62
- package/plugin-src/scripts/slm-launch.bat +0 -23
- package/plugin-src/settings.json +0 -25
- package/plugin-src/skills/slm-governance/SKILL.md +0 -248
- package/plugin-src/skills/slm-loop/SKILL.md +0 -99
- package/plugin-src/skills/slm-mesh/SKILL.md +0 -282
- package/plugin-src/skills/slm-profile/SKILL.md +0 -148
- package/plugin-src/skills/slm-scope/SKILL.md +0 -176
|
@@ -113,6 +113,15 @@ from superlocalmemory.storage.migrations import (
|
|
|
113
113
|
from superlocalmemory.storage.migrations import (
|
|
114
114
|
M027_transferable_patterns_profile as _M027,
|
|
115
115
|
)
|
|
116
|
+
from superlocalmemory.storage.migrations import (
|
|
117
|
+
M028_fact_entity_associations as _M028,
|
|
118
|
+
)
|
|
119
|
+
from superlocalmemory.storage.migrations import (
|
|
120
|
+
M029_behavioral_history_indexes as _M029,
|
|
121
|
+
)
|
|
122
|
+
from superlocalmemory.storage.migrations import (
|
|
123
|
+
M030_entity_explorer_indexes as _M030,
|
|
124
|
+
)
|
|
116
125
|
|
|
117
126
|
# Map migration name → module (used for the optional ``verify(conn)`` hook
|
|
118
127
|
# that lets the runner detect "already applied" state when an idempotent
|
|
@@ -144,6 +153,9 @@ _MODULES = {
|
|
|
144
153
|
_M025.NAME: _M025,
|
|
145
154
|
_M026.NAME: _M026,
|
|
146
155
|
_M027.NAME: _M027,
|
|
156
|
+
_M028.NAME: _M028,
|
|
157
|
+
_M029.NAME: _M029,
|
|
158
|
+
_M030.NAME: _M030,
|
|
147
159
|
}
|
|
148
160
|
|
|
149
161
|
logger = logging.getLogger(__name__)
|
|
@@ -218,6 +230,10 @@ MIGRATIONS: list[Migration] = [
|
|
|
218
230
|
# position proxy when the column is absent, so a failed deferred apply never
|
|
219
231
|
# crashes the trainer — it just keeps the old label path.
|
|
220
232
|
DEFERRED_MIGRATIONS: list[Migration] = [
|
|
233
|
+
# M028 captures an atomic_facts rowid high-water mark before readiness.
|
|
234
|
+
# atomic_facts/canonical_entities are bootstrapped by MemoryEngine.
|
|
235
|
+
Migration(name=_M028.NAME, db_target="memory", ddl=_M028.DDL,
|
|
236
|
+
dependencies=(_M018.NAME,)),
|
|
221
237
|
Migration(name=_M006.NAME, db_target="memory", ddl=_M006.DDL),
|
|
222
238
|
# M011 extends atomic_facts + creates memory_archive / memory_merge_log.
|
|
223
239
|
# atomic_facts is bootstrapped at engine init, so M011 defers alongside M006.
|
|
@@ -256,6 +272,10 @@ DEFERRED_MIGRATIONS: list[Migration] = [
|
|
|
256
272
|
# consolidation run, not during engine init or apply_all. apply() is a no-op
|
|
257
273
|
# when the table is absent (first install after the schema change).
|
|
258
274
|
Migration(name=_M027.NAME, db_target="learning", ddl=_M027.DDL),
|
|
275
|
+
# M029 indexes behavioral tables bootstrapped during engine initialization.
|
|
276
|
+
Migration(name=_M029.NAME, db_target="memory", ddl=_M029.DDL),
|
|
277
|
+
# M030 bounds Entity Explorer pagination and profile-summary ranking.
|
|
278
|
+
Migration(name=_M030.NAME, db_target="memory", ddl=_M030.DDL),
|
|
259
279
|
]
|
|
260
280
|
|
|
261
281
|
|
|
@@ -401,7 +421,65 @@ def _apply_single(
|
|
|
401
421
|
)
|
|
402
422
|
logger.warning(detail)
|
|
403
423
|
return ("failed", detail)
|
|
404
|
-
|
|
424
|
+
# A matching migration-log row is not proof that the promised
|
|
425
|
+
# schema still exists. Existing installs can retain a migration log
|
|
426
|
+
# while a partial restore drops an additive table or index.
|
|
427
|
+
#
|
|
428
|
+
# Never replay a historical migration merely because verify()
|
|
429
|
+
# fails. Some migrations rebuild tables and transform data; replay
|
|
430
|
+
# would be destructive (M002 is the canonical example). Only a
|
|
431
|
+
# module-supplied repair(conn) hook is allowed to reconcile a
|
|
432
|
+
# completed migration's end-state.
|
|
433
|
+
mod = _MODULES.get(migration.name)
|
|
434
|
+
verify_fn = (
|
|
435
|
+
getattr(mod, "verify", None) if mod is not None else None
|
|
436
|
+
)
|
|
437
|
+
if verify_fn is None:
|
|
438
|
+
return ("skipped", "already complete")
|
|
439
|
+
try:
|
|
440
|
+
schema_complete = bool(verify_fn(conn))
|
|
441
|
+
except sqlite3.Error as exc:
|
|
442
|
+
return (
|
|
443
|
+
"failed",
|
|
444
|
+
f"schema verification failed for {migration.name}: {exc}",
|
|
445
|
+
)
|
|
446
|
+
if schema_complete:
|
|
447
|
+
return ("skipped", "already complete (schema verified)")
|
|
448
|
+
if dry_run:
|
|
449
|
+
return (
|
|
450
|
+
"skipped",
|
|
451
|
+
"dry-run: would repair missing migration end-state",
|
|
452
|
+
)
|
|
453
|
+
repair_fn = (
|
|
454
|
+
getattr(mod, "repair", None) if mod is not None else None
|
|
455
|
+
)
|
|
456
|
+
if not callable(repair_fn):
|
|
457
|
+
detail = (
|
|
458
|
+
f"schema incomplete for completed migration "
|
|
459
|
+
f"{migration.name}; automatic replay is disabled"
|
|
460
|
+
)
|
|
461
|
+
logger.warning(detail)
|
|
462
|
+
return ("failed", detail)
|
|
463
|
+
try:
|
|
464
|
+
repair_fn(conn)
|
|
465
|
+
except sqlite3.Error as exc:
|
|
466
|
+
return (
|
|
467
|
+
"failed",
|
|
468
|
+
f"safe repair failed for {migration.name}: {exc}",
|
|
469
|
+
)
|
|
470
|
+
try:
|
|
471
|
+
if not bool(verify_fn(conn)):
|
|
472
|
+
return (
|
|
473
|
+
"failed",
|
|
474
|
+
f"safe repair did not restore {migration.name}",
|
|
475
|
+
)
|
|
476
|
+
except sqlite3.Error as exc:
|
|
477
|
+
return (
|
|
478
|
+
"failed",
|
|
479
|
+
f"post-repair verification failed for "
|
|
480
|
+
f"{migration.name}: {exc}",
|
|
481
|
+
)
|
|
482
|
+
return ("applied", "missing end-state repaired safely")
|
|
405
483
|
# status is 'failed' or 'in_progress' → retry from scratch.
|
|
406
484
|
if dry_run:
|
|
407
485
|
return ("skipped", f"dry-run: would retry (status={status})")
|
|
@@ -50,6 +50,11 @@ def verify(conn: sqlite3.Connection) -> bool:
|
|
|
50
50
|
return _REQUIRED_TABLES.issubset(names)
|
|
51
51
|
|
|
52
52
|
|
|
53
|
+
def repair(conn: sqlite3.Connection) -> None:
|
|
54
|
+
"""Restore additive Skill Evolution tables without replaying other migrations."""
|
|
55
|
+
conn.executescript(DDL)
|
|
56
|
+
|
|
57
|
+
|
|
53
58
|
DDL = """
|
|
54
59
|
CREATE TABLE IF NOT EXISTS evolution_config (
|
|
55
60
|
profile_id TEXT PRIMARY KEY,
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
"""M028 — normalized fact/entity associations for O(1) ingest effects.
|
|
5
|
+
|
|
6
|
+
The table is both an association index and an idempotency ledger. Its
|
|
7
|
+
composite primary key means a fact can contribute to an entity's ``fact_count``
|
|
8
|
+
once, regardless of ingestion retries or how many operations converge on the
|
|
9
|
+
same consolidated fact.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import sqlite3
|
|
16
|
+
from datetime import UTC, datetime
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
NAME = "M028_fact_entity_associations"
|
|
20
|
+
DB_TARGET = "memory"
|
|
21
|
+
|
|
22
|
+
DDL = """
|
|
23
|
+
CREATE TABLE IF NOT EXISTS fact_entity_associations (
|
|
24
|
+
profile_id TEXT NOT NULL,
|
|
25
|
+
fact_id TEXT NOT NULL,
|
|
26
|
+
entity_id TEXT NOT NULL,
|
|
27
|
+
first_operation_id TEXT NOT NULL DEFAULT '',
|
|
28
|
+
count_applied INTEGER NOT NULL DEFAULT 0
|
|
29
|
+
CHECK (count_applied IN (0, 1)),
|
|
30
|
+
created_at TEXT NOT NULL DEFAULT (
|
|
31
|
+
strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
|
|
32
|
+
),
|
|
33
|
+
PRIMARY KEY (profile_id, fact_id, entity_id),
|
|
34
|
+
FOREIGN KEY (fact_id) REFERENCES atomic_facts(fact_id) ON DELETE CASCADE,
|
|
35
|
+
FOREIGN KEY (entity_id)
|
|
36
|
+
REFERENCES canonical_entities(entity_id) ON DELETE CASCADE
|
|
37
|
+
);
|
|
38
|
+
CREATE INDEX IF NOT EXISTS idx_fact_entity_associations_entity
|
|
39
|
+
ON fact_entity_associations(profile_id, entity_id, fact_id);
|
|
40
|
+
CREATE TABLE IF NOT EXISTS fact_entity_association_repair_state (
|
|
41
|
+
repair_key TEXT PRIMARY KEY,
|
|
42
|
+
state TEXT NOT NULL DEFAULT 'pending'
|
|
43
|
+
CHECK (state IN ('pending', 'running', 'retrying', 'complete')),
|
|
44
|
+
target_fact_rowid INTEGER NOT NULL DEFAULT -1,
|
|
45
|
+
last_fact_rowid INTEGER NOT NULL DEFAULT 0,
|
|
46
|
+
scanned INTEGER NOT NULL DEFAULT 0,
|
|
47
|
+
inserted INTEGER NOT NULL DEFAULT 0,
|
|
48
|
+
last_error TEXT NOT NULL DEFAULT '',
|
|
49
|
+
updated_at TEXT NOT NULL
|
|
50
|
+
);
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
|
|
55
|
+
return conn.execute(
|
|
56
|
+
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?",
|
|
57
|
+
(table,),
|
|
58
|
+
).fetchone() is not None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def apply(conn: sqlite3.Connection) -> None:
|
|
62
|
+
"""Install schema and capture a constant-time historical rowid boundary."""
|
|
63
|
+
conn.executescript(DDL)
|
|
64
|
+
association_columns = {
|
|
65
|
+
row[1]
|
|
66
|
+
for row in conn.execute(
|
|
67
|
+
"PRAGMA table_info(fact_entity_associations)"
|
|
68
|
+
).fetchall()
|
|
69
|
+
}
|
|
70
|
+
if "count_applied" not in association_columns:
|
|
71
|
+
conn.execute(
|
|
72
|
+
"ALTER TABLE fact_entity_associations "
|
|
73
|
+
"ADD COLUMN count_applied INTEGER NOT NULL DEFAULT 0 "
|
|
74
|
+
"CHECK (count_applied IN (0, 1))"
|
|
75
|
+
)
|
|
76
|
+
repair_columns = {
|
|
77
|
+
row[1]
|
|
78
|
+
for row in conn.execute(
|
|
79
|
+
"PRAGMA table_info(fact_entity_association_repair_state)"
|
|
80
|
+
).fetchall()
|
|
81
|
+
}
|
|
82
|
+
if "target_fact_rowid" not in repair_columns:
|
|
83
|
+
conn.execute(
|
|
84
|
+
"ALTER TABLE fact_entity_association_repair_state "
|
|
85
|
+
"ADD COLUMN target_fact_rowid INTEGER NOT NULL DEFAULT -1"
|
|
86
|
+
)
|
|
87
|
+
target = int(conn.execute(
|
|
88
|
+
"SELECT COALESCE(MAX(rowid), 0) FROM atomic_facts"
|
|
89
|
+
).fetchone()[0])
|
|
90
|
+
conn.executescript(
|
|
91
|
+
"INSERT OR IGNORE INTO fact_entity_association_repair_state "
|
|
92
|
+
"(repair_key,state,target_fact_rowid,updated_at) "
|
|
93
|
+
f"VALUES ('historical-backfill','pending',{target},'');"
|
|
94
|
+
"UPDATE fact_entity_association_repair_state "
|
|
95
|
+
f"SET target_fact_rowid={target} "
|
|
96
|
+
"WHERE repair_key='historical-backfill' AND target_fact_rowid < 0;"
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _now() -> str:
|
|
101
|
+
return datetime.now(UTC).isoformat()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _connect(db_path: Path) -> sqlite3.Connection:
|
|
105
|
+
conn = sqlite3.connect(str(db_path), timeout=5)
|
|
106
|
+
conn.row_factory = sqlite3.Row
|
|
107
|
+
conn.execute("PRAGMA busy_timeout=5000")
|
|
108
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
109
|
+
return conn
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def get_repair_status(db_path: Path) -> dict[str, int | str]:
|
|
113
|
+
"""Read durable backfill progress without inferring it from schema."""
|
|
114
|
+
conn = _connect(Path(db_path))
|
|
115
|
+
try:
|
|
116
|
+
row = conn.execute(
|
|
117
|
+
"SELECT state,target_fact_rowid,last_fact_rowid,scanned,inserted,"
|
|
118
|
+
"last_error,updated_at "
|
|
119
|
+
"FROM fact_entity_association_repair_state "
|
|
120
|
+
"WHERE repair_key='historical-backfill'"
|
|
121
|
+
).fetchone()
|
|
122
|
+
if row is None:
|
|
123
|
+
return {
|
|
124
|
+
"state": "pending", "target_fact_rowid": 0,
|
|
125
|
+
"last_fact_rowid": 0,
|
|
126
|
+
"scanned": 0, "inserted": 0,
|
|
127
|
+
"last_error": "", "updated_at": "",
|
|
128
|
+
}
|
|
129
|
+
return dict(row)
|
|
130
|
+
finally:
|
|
131
|
+
conn.close()
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _entity_ids(raw: object) -> tuple[str, ...]:
|
|
135
|
+
try:
|
|
136
|
+
values = json.loads(str(raw or "[]"))
|
|
137
|
+
except (TypeError, ValueError):
|
|
138
|
+
return ()
|
|
139
|
+
if not isinstance(values, list):
|
|
140
|
+
return ()
|
|
141
|
+
return tuple(dict.fromkeys(str(value) for value in values if value))
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _repair_batch(conn: sqlite3.Connection, batch_size: int) -> dict[str, int | bool]:
|
|
145
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
146
|
+
try:
|
|
147
|
+
status = conn.execute(
|
|
148
|
+
"SELECT last_fact_rowid,target_fact_rowid "
|
|
149
|
+
"FROM fact_entity_association_repair_state "
|
|
150
|
+
"WHERE repair_key='historical-backfill'"
|
|
151
|
+
).fetchone()
|
|
152
|
+
cursor = int(status["last_fact_rowid"] or 0)
|
|
153
|
+
target = int(status["target_fact_rowid"])
|
|
154
|
+
rows = conn.execute(
|
|
155
|
+
"SELECT rowid,fact_id,profile_id,canonical_entities_json "
|
|
156
|
+
"FROM atomic_facts WHERE rowid>? AND rowid<=? "
|
|
157
|
+
"ORDER BY rowid LIMIT ?",
|
|
158
|
+
(cursor, target, batch_size),
|
|
159
|
+
).fetchall()
|
|
160
|
+
if not rows:
|
|
161
|
+
conn.execute(
|
|
162
|
+
"UPDATE fact_entity_association_repair_state "
|
|
163
|
+
"SET state='complete',last_error='',updated_at=? "
|
|
164
|
+
"WHERE repair_key='historical-backfill'",
|
|
165
|
+
(_now(),),
|
|
166
|
+
)
|
|
167
|
+
conn.commit()
|
|
168
|
+
return {"scanned": 0, "inserted": 0, "complete": True}
|
|
169
|
+
inserted = 0
|
|
170
|
+
for row in rows:
|
|
171
|
+
for entity_id in _entity_ids(row["canonical_entities_json"]):
|
|
172
|
+
result = conn.execute(
|
|
173
|
+
"INSERT OR IGNORE INTO fact_entity_associations "
|
|
174
|
+
"(profile_id,fact_id,entity_id,first_operation_id,"
|
|
175
|
+
"count_applied) "
|
|
176
|
+
"SELECT ?,?,?,?,? FROM canonical_entities "
|
|
177
|
+
"WHERE profile_id=? AND entity_id=?",
|
|
178
|
+
(
|
|
179
|
+
row["profile_id"], row["fact_id"], entity_id,
|
|
180
|
+
"migration-backfill", 0,
|
|
181
|
+
row["profile_id"], entity_id,
|
|
182
|
+
),
|
|
183
|
+
)
|
|
184
|
+
inserted += max(0, result.rowcount)
|
|
185
|
+
conn.execute(
|
|
186
|
+
"UPDATE fact_entity_association_repair_state SET "
|
|
187
|
+
"state='running',last_fact_rowid=?,scanned=scanned+?,"
|
|
188
|
+
"inserted=inserted+?,last_error='',updated_at=? "
|
|
189
|
+
"WHERE repair_key='historical-backfill'",
|
|
190
|
+
(int(rows[-1]["rowid"]), len(rows), inserted, _now()),
|
|
191
|
+
)
|
|
192
|
+
conn.commit()
|
|
193
|
+
return {"scanned": len(rows), "inserted": inserted, "complete": False}
|
|
194
|
+
except Exception:
|
|
195
|
+
conn.rollback()
|
|
196
|
+
raise
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def repair_fact_entity_associations(
|
|
200
|
+
db_path: Path,
|
|
201
|
+
*,
|
|
202
|
+
batch_size: int = 250,
|
|
203
|
+
max_batches: int = 1,
|
|
204
|
+
) -> dict[str, int | bool]:
|
|
205
|
+
"""Run bounded, restartable short-transaction backfill batches."""
|
|
206
|
+
if batch_size < 1 or max_batches < 1:
|
|
207
|
+
raise ValueError("batch_size and max_batches must be positive")
|
|
208
|
+
totals = {"scanned": 0, "inserted": 0, "complete": False}
|
|
209
|
+
conn = _connect(Path(db_path))
|
|
210
|
+
try:
|
|
211
|
+
for _ in range(max_batches):
|
|
212
|
+
result = _repair_batch(conn, batch_size)
|
|
213
|
+
totals["scanned"] += int(result["scanned"])
|
|
214
|
+
totals["inserted"] += int(result["inserted"])
|
|
215
|
+
totals["complete"] = bool(result["complete"])
|
|
216
|
+
if totals["complete"]:
|
|
217
|
+
break
|
|
218
|
+
return totals
|
|
219
|
+
except sqlite3.Error as exc:
|
|
220
|
+
try:
|
|
221
|
+
conn.execute(
|
|
222
|
+
"UPDATE fact_entity_association_repair_state SET "
|
|
223
|
+
"state='retrying',last_error=?,updated_at=? "
|
|
224
|
+
"WHERE repair_key='historical-backfill'",
|
|
225
|
+
(type(exc).__name__, _now()),
|
|
226
|
+
)
|
|
227
|
+
conn.commit()
|
|
228
|
+
except sqlite3.Error:
|
|
229
|
+
pass
|
|
230
|
+
raise
|
|
231
|
+
finally:
|
|
232
|
+
conn.close()
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def verify(conn: sqlite3.Connection) -> bool:
|
|
236
|
+
"""Return true when the normalized association contract is indexed."""
|
|
237
|
+
if not (
|
|
238
|
+
_table_exists(conn, "fact_entity_associations")
|
|
239
|
+
and _table_exists(conn, "fact_entity_association_repair_state")
|
|
240
|
+
):
|
|
241
|
+
return False
|
|
242
|
+
association_columns = {
|
|
243
|
+
row[1]
|
|
244
|
+
for row in conn.execute(
|
|
245
|
+
"PRAGMA table_info(fact_entity_associations)"
|
|
246
|
+
).fetchall()
|
|
247
|
+
}
|
|
248
|
+
repair_columns = {
|
|
249
|
+
row[1]
|
|
250
|
+
for row in conn.execute(
|
|
251
|
+
"PRAGMA table_info(fact_entity_association_repair_state)"
|
|
252
|
+
).fetchall()
|
|
253
|
+
}
|
|
254
|
+
if (
|
|
255
|
+
"count_applied" not in association_columns
|
|
256
|
+
or "target_fact_rowid" not in repair_columns
|
|
257
|
+
):
|
|
258
|
+
return False
|
|
259
|
+
indexes = {
|
|
260
|
+
row[1]
|
|
261
|
+
for row in conn.execute(
|
|
262
|
+
"PRAGMA index_list(fact_entity_associations)"
|
|
263
|
+
).fetchall()
|
|
264
|
+
}
|
|
265
|
+
return "idx_fact_entity_associations_entity" in indexes
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def repair(conn: sqlite3.Connection) -> None:
|
|
269
|
+
"""Restore additive M028 schema without recapturing its high-water mark."""
|
|
270
|
+
apply(conn)
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
"""M029 — profile-scoped indexes for large behavioral histories.
|
|
5
|
+
|
|
6
|
+
Dashboard reads always filter behavioral and outcome history by profile before
|
|
7
|
+
sorting or aggregating. These composite indexes keep those operations bounded
|
|
8
|
+
to one tenant as the shared database grows.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import sqlite3
|
|
14
|
+
|
|
15
|
+
NAME = "M029_behavioral_history_indexes"
|
|
16
|
+
DB_TARGET = "memory"
|
|
17
|
+
|
|
18
|
+
DDL = """
|
|
19
|
+
CREATE INDEX IF NOT EXISTS idx_outcomes_profile_outcome_time
|
|
20
|
+
ON action_outcomes(profile_id, outcome, timestamp DESC);
|
|
21
|
+
CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_time
|
|
22
|
+
ON action_outcomes(profile_id, settled, settled_at DESC);
|
|
23
|
+
CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_cursor
|
|
24
|
+
ON action_outcomes(
|
|
25
|
+
profile_id, settled, COALESCE(settled_at, ''), outcome_id
|
|
26
|
+
);
|
|
27
|
+
CREATE INDEX IF NOT EXISTS idx_assertions_profile_confidence
|
|
28
|
+
ON behavioral_assertions(profile_id, confidence DESC);
|
|
29
|
+
CREATE INDEX IF NOT EXISTS idx_tool_events_profile_created
|
|
30
|
+
ON tool_events(profile_id, created_at DESC);
|
|
31
|
+
CREATE INDEX IF NOT EXISTS idx_soft_prompts_profile_active_category
|
|
32
|
+
ON soft_prompt_templates(profile_id, active, category, prompt_id);
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
_INDEXES: tuple[tuple[str, str, tuple[str, ...]], ...] = (
|
|
36
|
+
(
|
|
37
|
+
"idx_outcomes_profile_outcome_time",
|
|
38
|
+
"action_outcomes",
|
|
39
|
+
("profile_id", "outcome", "timestamp"),
|
|
40
|
+
),
|
|
41
|
+
(
|
|
42
|
+
"idx_outcomes_profile_settled_time",
|
|
43
|
+
"action_outcomes",
|
|
44
|
+
("profile_id", "settled", "settled_at"),
|
|
45
|
+
),
|
|
46
|
+
(
|
|
47
|
+
"idx_outcomes_profile_settled_cursor",
|
|
48
|
+
"action_outcomes",
|
|
49
|
+
("profile_id", "settled", "settled_at", "outcome_id"),
|
|
50
|
+
),
|
|
51
|
+
(
|
|
52
|
+
"idx_assertions_profile_confidence",
|
|
53
|
+
"behavioral_assertions",
|
|
54
|
+
("profile_id", "confidence"),
|
|
55
|
+
),
|
|
56
|
+
(
|
|
57
|
+
"idx_tool_events_profile_created",
|
|
58
|
+
"tool_events",
|
|
59
|
+
("profile_id", "created_at"),
|
|
60
|
+
),
|
|
61
|
+
(
|
|
62
|
+
"idx_soft_prompts_profile_active_category",
|
|
63
|
+
"soft_prompt_templates",
|
|
64
|
+
("profile_id", "active", "category", "prompt_id"),
|
|
65
|
+
),
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _columns(conn: sqlite3.Connection, table: str) -> set[str]:
|
|
70
|
+
return {
|
|
71
|
+
str(row[1])
|
|
72
|
+
for row in conn.execute(f"PRAGMA table_info({table})").fetchall()
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _index_sql(name: str) -> str:
|
|
77
|
+
statements = {
|
|
78
|
+
"idx_outcomes_profile_outcome_time": (
|
|
79
|
+
"CREATE INDEX IF NOT EXISTS idx_outcomes_profile_outcome_time "
|
|
80
|
+
"ON action_outcomes(profile_id, outcome, timestamp DESC)"
|
|
81
|
+
),
|
|
82
|
+
"idx_outcomes_profile_settled_time": (
|
|
83
|
+
"CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_time "
|
|
84
|
+
"ON action_outcomes(profile_id, settled, settled_at DESC)"
|
|
85
|
+
),
|
|
86
|
+
"idx_outcomes_profile_settled_cursor": (
|
|
87
|
+
"CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_cursor "
|
|
88
|
+
"ON action_outcomes("
|
|
89
|
+
"profile_id, settled, COALESCE(settled_at, ''), outcome_id)"
|
|
90
|
+
),
|
|
91
|
+
"idx_assertions_profile_confidence": (
|
|
92
|
+
"CREATE INDEX IF NOT EXISTS idx_assertions_profile_confidence "
|
|
93
|
+
"ON behavioral_assertions(profile_id, confidence DESC)"
|
|
94
|
+
),
|
|
95
|
+
"idx_tool_events_profile_created": (
|
|
96
|
+
"CREATE INDEX IF NOT EXISTS idx_tool_events_profile_created "
|
|
97
|
+
"ON tool_events(profile_id, created_at DESC)"
|
|
98
|
+
),
|
|
99
|
+
"idx_soft_prompts_profile_active_category": (
|
|
100
|
+
"CREATE INDEX IF NOT EXISTS "
|
|
101
|
+
"idx_soft_prompts_profile_active_category "
|
|
102
|
+
"ON soft_prompt_templates(profile_id, active, category, prompt_id)"
|
|
103
|
+
),
|
|
104
|
+
}
|
|
105
|
+
return statements[name]
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def apply(conn: sqlite3.Connection) -> None:
|
|
109
|
+
"""Add each index supported by the installed runtime schema.
|
|
110
|
+
|
|
111
|
+
Optional v3.2/v3.4.7 tables also declare these indexes in their owning
|
|
112
|
+
schema modules, so a partial bootstrap cannot permanently miss an index
|
|
113
|
+
when those tables are created later.
|
|
114
|
+
"""
|
|
115
|
+
for name, table, required_columns in _INDEXES:
|
|
116
|
+
if set(required_columns) <= _columns(conn, table):
|
|
117
|
+
conn.execute(_index_sql(name))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def verify(conn: sqlite3.Connection) -> bool:
|
|
121
|
+
"""Require each index whose backing table and columns are present."""
|
|
122
|
+
names = {
|
|
123
|
+
str(row[0])
|
|
124
|
+
for row in conn.execute(
|
|
125
|
+
"SELECT name FROM sqlite_master WHERE type='index'"
|
|
126
|
+
).fetchall()
|
|
127
|
+
}
|
|
128
|
+
return all(
|
|
129
|
+
name in names
|
|
130
|
+
for name, table, required_columns in _INDEXES
|
|
131
|
+
if set(required_columns) <= _columns(conn, table)
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def repair(conn: sqlite3.Connection) -> None:
|
|
136
|
+
"""Create only indexes supported by the current optional schema."""
|
|
137
|
+
apply(conn)
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
"""M030 — page-first Entity Explorer indexes for large installations."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import sqlite3
|
|
9
|
+
|
|
10
|
+
NAME = "M030_entity_explorer_indexes"
|
|
11
|
+
DB_TARGET = "memory"
|
|
12
|
+
|
|
13
|
+
DDL = """
|
|
14
|
+
CREATE INDEX IF NOT EXISTS idx_entities_profile_fact_count_id
|
|
15
|
+
ON canonical_entities(profile_id, fact_count DESC, entity_id ASC);
|
|
16
|
+
CREATE INDEX IF NOT EXISTS idx_entities_profile_type_fact_count_id
|
|
17
|
+
ON canonical_entities(
|
|
18
|
+
profile_id, entity_type COLLATE NOCASE, fact_count DESC, entity_id ASC
|
|
19
|
+
);
|
|
20
|
+
CREATE INDEX IF NOT EXISTS idx_entity_profiles_profile_entity_rank
|
|
21
|
+
ON entity_profiles(
|
|
22
|
+
profile_id, entity_id, last_compiled_at DESC,
|
|
23
|
+
project_name COLLATE NOCASE
|
|
24
|
+
);
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
_INDEXES: tuple[tuple[str, str, tuple[str, ...], str], ...] = (
|
|
28
|
+
(
|
|
29
|
+
"idx_entities_profile_fact_count_id",
|
|
30
|
+
"canonical_entities",
|
|
31
|
+
("profile_id", "fact_count", "entity_id"),
|
|
32
|
+
"CREATE INDEX IF NOT EXISTS idx_entities_profile_fact_count_id "
|
|
33
|
+
"ON canonical_entities(profile_id, fact_count DESC, entity_id ASC)",
|
|
34
|
+
),
|
|
35
|
+
(
|
|
36
|
+
"idx_entities_profile_type_fact_count_id",
|
|
37
|
+
"canonical_entities",
|
|
38
|
+
("profile_id", "entity_type", "fact_count", "entity_id"),
|
|
39
|
+
"CREATE INDEX IF NOT EXISTS idx_entities_profile_type_fact_count_id "
|
|
40
|
+
"ON canonical_entities("
|
|
41
|
+
"profile_id, entity_type COLLATE NOCASE, fact_count DESC, entity_id ASC"
|
|
42
|
+
")",
|
|
43
|
+
),
|
|
44
|
+
(
|
|
45
|
+
"idx_entity_profiles_profile_entity_rank",
|
|
46
|
+
"entity_profiles",
|
|
47
|
+
(
|
|
48
|
+
"profile_id",
|
|
49
|
+
"entity_id",
|
|
50
|
+
"last_compiled_at",
|
|
51
|
+
"project_name",
|
|
52
|
+
),
|
|
53
|
+
"CREATE INDEX IF NOT EXISTS idx_entity_profiles_profile_entity_rank "
|
|
54
|
+
"ON entity_profiles("
|
|
55
|
+
"profile_id, entity_id, last_compiled_at DESC, "
|
|
56
|
+
"project_name COLLATE NOCASE"
|
|
57
|
+
")",
|
|
58
|
+
),
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _columns(conn: sqlite3.Connection, table: str) -> set[str]:
|
|
63
|
+
return {
|
|
64
|
+
str(row[1])
|
|
65
|
+
for row in conn.execute(f"PRAGMA table_info({table})").fetchall()
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def apply(conn: sqlite3.Connection) -> None:
|
|
70
|
+
"""Create every index supported by the installed optional schema."""
|
|
71
|
+
for _, table, required_columns, sql in _INDEXES:
|
|
72
|
+
if set(required_columns) <= _columns(conn, table):
|
|
73
|
+
conn.execute(sql)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def verify(conn: sqlite3.Connection) -> bool:
|
|
77
|
+
"""Require indexes only when their complete backing schema exists."""
|
|
78
|
+
names = {
|
|
79
|
+
str(row[0])
|
|
80
|
+
for row in conn.execute(
|
|
81
|
+
"SELECT name FROM sqlite_master WHERE type='index'"
|
|
82
|
+
).fetchall()
|
|
83
|
+
}
|
|
84
|
+
return all(
|
|
85
|
+
name in names
|
|
86
|
+
for name, table, required_columns, _ in _INDEXES
|
|
87
|
+
if set(required_columns) <= _columns(conn, table)
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def repair(conn: sqlite3.Connection) -> None:
|
|
92
|
+
"""Safely restore a dropped index without replaying data migrations."""
|
|
93
|
+
apply(conn)
|
|
@@ -26,6 +26,8 @@ from . import (
|
|
|
26
26
|
M015_add_pinned_column,
|
|
27
27
|
M019_derivation_lineage,
|
|
28
28
|
M020_model_state_integrity,
|
|
29
|
+
M029_behavioral_history_indexes,
|
|
30
|
+
M030_entity_explorer_indexes,
|
|
29
31
|
)
|
|
30
32
|
|
|
31
33
|
# ---------------------------------------------------------------------------
|
|
@@ -77,6 +79,8 @@ __all__ = (
|
|
|
77
79
|
"M015_add_pinned_column",
|
|
78
80
|
"M019_derivation_lineage",
|
|
79
81
|
"M020_model_state_integrity",
|
|
82
|
+
"M029_behavioral_history_indexes",
|
|
83
|
+
"M030_entity_explorer_indexes",
|
|
80
84
|
# Legacy re-exports (backward compat):
|
|
81
85
|
"CURRENT_SCHEMA_VERSION",
|
|
82
86
|
"get_schema_version",
|