superlocalmemory 3.8.0 → 3.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +112 -0
- package/README.md +32 -120
- package/package.json +9 -2
- package/plugin/.claude-plugin/plugin.json +1 -2
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +2 -2
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +3 -5
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +2 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +3 -5
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +2 -1
- package/scripts/postinstall.js +7 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +494 -9
- package/src/superlocalmemory/cli/daemon.py +7 -0
- package/src/superlocalmemory/cli/loop_cmd.py +2 -7
- package/src/superlocalmemory/cli/main.py +72 -7
- package/src/superlocalmemory/cli/setup_wizard.py +142 -16
- package/src/superlocalmemory/cli/version_banner.py +17 -3
- package/src/superlocalmemory/core/backend_orchestrator.py +18 -16
- package/src/superlocalmemory/core/component_healer.py +144 -0
- package/src/superlocalmemory/core/component_registry.py +487 -0
- package/src/superlocalmemory/core/config.py +21 -0
- package/src/superlocalmemory/core/embedding_worker.py +4 -5
- package/src/superlocalmemory/core/embeddings.py +132 -45
- package/src/superlocalmemory/core/engine.py +29 -22
- package/src/superlocalmemory/core/engine_ingestion.py +332 -45
- package/src/superlocalmemory/core/ingestion_command.py +154 -25
- package/src/superlocalmemory/core/injection.py +12 -7
- package/src/superlocalmemory/core/maintenance.py +43 -0
- package/src/superlocalmemory/core/maintenance_scheduler.py +44 -6
- package/src/superlocalmemory/core/recall_pipeline.py +42 -4
- package/src/superlocalmemory/core/store_pipeline.py +195 -20
- package/src/superlocalmemory/hooks/hook_handlers.py +6 -1
- package/src/superlocalmemory/hooks/portable_kit.py +34 -2
- package/src/superlocalmemory/learning/model_rollback.py +3 -0
- package/src/superlocalmemory/learning/ranker_retrain_online.py +2 -0
- package/src/superlocalmemory/learning/reward.py +50 -0
- package/src/superlocalmemory/learning/source_quality.py +523 -1
- package/src/superlocalmemory/loops/ledger.py +25 -5
- package/src/superlocalmemory/mcp/_daemon_proxy.py +6 -2
- package/src/superlocalmemory/mcp/_pool_adapter.py +4 -1
- package/src/superlocalmemory/mcp/server.py +11 -30
- package/src/superlocalmemory/mcp/tools_active.py +1 -1
- package/src/superlocalmemory/mcp/tools_core.py +21 -5
- package/src/superlocalmemory/mcp/tools_learning.py +2 -2
- package/src/superlocalmemory/retrieval/bridge_discovery.py +14 -0
- package/src/superlocalmemory/retrieval/engine.py +53 -21
- package/src/superlocalmemory/retrieval/reranker.py +3 -4
- package/src/superlocalmemory/retrieval/spreading_activation.py +68 -38
- package/src/superlocalmemory/server/config_file.py +90 -0
- package/src/superlocalmemory/server/origin.py +50 -0
- package/src/superlocalmemory/server/routes/backup.py +293 -70
- package/src/superlocalmemory/server/routes/behavioral.py +342 -61
- package/src/superlocalmemory/server/routes/brain.py +57 -16
- package/src/superlocalmemory/server/routes/config_api.py +84 -82
- package/src/superlocalmemory/server/routes/entity.py +100 -23
- package/src/superlocalmemory/server/routes/evolution.py +103 -100
- package/src/superlocalmemory/server/routes/learning.py +286 -105
- package/src/superlocalmemory/server/routes/learning_telemetry.py +153 -0
- package/src/superlocalmemory/server/routes/memories.py +8 -3
- package/src/superlocalmemory/server/routes/mesh.py +121 -32
- package/src/superlocalmemory/server/routes/ratelimit.py +33 -25
- package/src/superlocalmemory/server/routes/stats.py +93 -155
- package/src/superlocalmemory/server/routes/token.py +3 -13
- package/src/superlocalmemory/server/routes/v3_api.py +184 -20
- package/src/superlocalmemory/server/unified_daemon.py +732 -41
- package/src/superlocalmemory/storage/embedding_migrator.py +235 -0
- package/src/superlocalmemory/storage/migration_runner.py +79 -1
- package/src/superlocalmemory/storage/migrations/M010_evolution_config.py +5 -0
- package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +270 -0
- package/src/superlocalmemory/storage/migrations/M029_behavioral_history_indexes.py +137 -0
- package/src/superlocalmemory/storage/migrations/M030_entity_explorer_indexes.py +93 -0
- package/src/superlocalmemory/storage/migrations/__init__.py +4 -0
- package/src/superlocalmemory/storage/schema.py +49 -1
- package/src/superlocalmemory/storage/schema_v32.py +2 -0
- package/src/superlocalmemory/storage/schema_v347.py +4 -0
- package/src/superlocalmemory/ui/index.html +6 -8
- package/src/superlocalmemory/ui/js/core.js +52 -9
- package/src/superlocalmemory/ui/js/dashboard.js +169 -82
- package/src/superlocalmemory/ui/js/od-backup.js +156 -65
- package/src/superlocalmemory/ui/js/od-brain.js +88 -51
- package/src/superlocalmemory/ui/js/od-components.js +147 -0
- package/src/superlocalmemory/ui/js/od-entities.js +65 -22
- package/src/superlocalmemory/ui/js/od-graph.js +46 -4
- package/src/superlocalmemory/ui/js/od-health.js +18 -0
- package/src/superlocalmemory/ui/js/od-memories.js +84 -5
- package/src/superlocalmemory/ui/js/od-mesh.js +23 -9
- package/src/superlocalmemory/ui/js/od-operations.js +36 -0
- package/src/superlocalmemory/ui/js/od-settings.js +186 -63
- package/src/superlocalmemory/ui/js/od-shell.js +249 -33
- package/src/superlocalmemory/ui/js/od-skills.js +44 -17
- package/src/superlocalmemory/ui/js/settings.js +15 -1
- package/plugin-src/.mcp.json +0 -12
- package/plugin-src/agents/slm-governance-advisor.md +0 -80
- package/plugin-src/agents/slm-loop-runner.md +0 -71
- package/plugin-src/agents/slm-memory-advisor.md +0 -49
- package/plugin-src/agents/slm-optimize-advisor.md +0 -44
- package/plugin-src/commands/slm-loop.md +0 -31
- package/plugin-src/hooks/.gitkeep +0 -0
- package/plugin-src/hooks/hooks.json +0 -102
- package/plugin-src/manifest.json +0 -30
- package/plugin-src/requirements.txt +0 -1
- package/plugin-src/rules/CLAUDE.md.fragment +0 -44
- package/plugin-src/scripts/ensure-venv.bat +0 -122
- package/plugin-src/scripts/ensure-venv.sh +0 -105
- package/plugin-src/scripts/slm-launch +0 -62
- package/plugin-src/scripts/slm-launch.bat +0 -23
- package/plugin-src/settings.json +0 -25
- package/plugin-src/skills/slm-governance/SKILL.md +0 -248
- package/plugin-src/skills/slm-loop/SKILL.md +0 -99
- package/plugin-src/skills/slm-mesh/SKILL.md +0 -282
- package/plugin-src/skills/slm-profile/SKILL.md +0 -148
- package/plugin-src/skills/slm-scope/SKILL.md +0 -176
|
@@ -26,6 +26,23 @@ if TYPE_CHECKING:
|
|
|
26
26
|
|
|
27
27
|
logger = logging.getLogger(__name__)
|
|
28
28
|
|
|
29
|
+
# ---------------------------------------------------------------------------
|
|
30
|
+
# Backfill constants
|
|
31
|
+
# ---------------------------------------------------------------------------
|
|
32
|
+
|
|
33
|
+
#: Default batch size for backfill_missing_embeddings.
|
|
34
|
+
_BACKFILL_BATCH_SIZE = 50
|
|
35
|
+
|
|
36
|
+
#: Max characters embedded per fact during backfill. The embedding model
|
|
37
|
+
#: (nomic-embed-text-v1.5) truncates at ~8192 tokens anyway, but a raw
|
|
38
|
+
#: oversized document (observed up to 107 KB on a real DB) makes the shared
|
|
39
|
+
#: single-worker embedder busy for 15-20s on ONE fact — starving foreground
|
|
40
|
+
#: recall during a self-heal pass. Bounding the input keeps every fact's embed
|
|
41
|
+
#: fast and the worker responsive; the leading slice captures the fact's gist
|
|
42
|
+
#: for semantic recall. Facts this large are documents that were almost
|
|
43
|
+
#: certainly NULL because they failed to embed at ingestion for the same reason.
|
|
44
|
+
_MAX_EMBED_CHARS = 8000
|
|
45
|
+
|
|
29
46
|
# Sentinel stored in config.json when no model has been set yet.
|
|
30
47
|
_NO_MODEL = ""
|
|
31
48
|
|
|
@@ -44,6 +61,24 @@ def _model_signature(config: SLMConfig) -> str:
|
|
|
44
61
|
return f"{emb.model_name}::{emb.dimension}"
|
|
45
62
|
|
|
46
63
|
|
|
64
|
+
def _normalize_signature(signature: str) -> str:
|
|
65
|
+
"""Normalize a signature for equivalence comparison.
|
|
66
|
+
|
|
67
|
+
v3.8.2 self-healing: the SAME embedding model has been recorded under
|
|
68
|
+
different name strings across releases — notably the HuggingFace org
|
|
69
|
+
prefix drifted (``nomic-ai/nomic-embed-text-v1.5`` vs the bare
|
|
70
|
+
``nomic-embed-text-v1.5``). A prefix-only difference does NOT change the
|
|
71
|
+
embedding vector space, so it must not trigger a full multi-hour re-embed
|
|
72
|
+
when a non-technical user upgrades. This collapses the model name to its
|
|
73
|
+
basename (segment after the last ``/``) while keeping the ``::dimension``
|
|
74
|
+
suffix — a genuine model change (different basename OR dimension) still
|
|
75
|
+
differs and still triggers migration.
|
|
76
|
+
"""
|
|
77
|
+
model, sep, dim = signature.partition("::")
|
|
78
|
+
model = model.rsplit("/", 1)[-1].strip()
|
|
79
|
+
return f"{model}{sep}{dim}" if sep else model
|
|
80
|
+
|
|
81
|
+
|
|
47
82
|
def _read_stored_signature(config_dir: Path) -> str:
|
|
48
83
|
"""Read the last-used embedding model signature from config.json."""
|
|
49
84
|
config_path = config_dir / "config.json"
|
|
@@ -88,6 +123,19 @@ def check_embedding_migration(config: SLMConfig) -> bool:
|
|
|
88
123
|
if stored_sig == current_sig:
|
|
89
124
|
return False
|
|
90
125
|
|
|
126
|
+
# v3.8.2 self-healing: a prefix-only model-name drift (e.g. the nomic-ai/
|
|
127
|
+
# org prefix appearing/disappearing between releases) is the SAME vector
|
|
128
|
+
# space — absorb the transition by refreshing the stored signature to the
|
|
129
|
+
# current form, with NO re-embed. This spares non-technical users a
|
|
130
|
+
# multi-hour full re-index on a cosmetic upgrade.
|
|
131
|
+
if _normalize_signature(stored_sig) == _normalize_signature(current_sig):
|
|
132
|
+
_write_stored_signature(config.base_dir, current_sig)
|
|
133
|
+
logger.info(
|
|
134
|
+
"Embedding signature normalized (no re-embed): %s ~= %s",
|
|
135
|
+
stored_sig, current_sig,
|
|
136
|
+
)
|
|
137
|
+
return False
|
|
138
|
+
|
|
91
139
|
logger.warning(
|
|
92
140
|
"Embedding model changed: %s -> %s. Re-indexing required.",
|
|
93
141
|
stored_sig, current_sig,
|
|
@@ -177,3 +225,190 @@ def run_embedding_migration(
|
|
|
177
225
|
reindexed, total,
|
|
178
226
|
)
|
|
179
227
|
return reindexed
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
# ---------------------------------------------------------------------------
|
|
231
|
+
# Backfill: embed facts that were NEVER embedded (embedding IS NULL)
|
|
232
|
+
# ---------------------------------------------------------------------------
|
|
233
|
+
|
|
234
|
+
def _count_null_embeddings(
|
|
235
|
+
db: Any,
|
|
236
|
+
profile_id: str,
|
|
237
|
+
all_profiles: bool,
|
|
238
|
+
) -> int:
|
|
239
|
+
"""Return count of atomic_facts rows with NULL embedding."""
|
|
240
|
+
if all_profiles:
|
|
241
|
+
rows = db.execute(
|
|
242
|
+
"SELECT count(*) AS c FROM atomic_facts WHERE embedding IS NULL",
|
|
243
|
+
)
|
|
244
|
+
else:
|
|
245
|
+
rows = db.execute(
|
|
246
|
+
"SELECT count(*) AS c FROM atomic_facts "
|
|
247
|
+
"WHERE embedding IS NULL AND profile_id = ?",
|
|
248
|
+
(profile_id,),
|
|
249
|
+
)
|
|
250
|
+
return int(rows[0]["c"]) if rows else 0
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def backfill_missing_embeddings(
|
|
254
|
+
config: "SLMConfig",
|
|
255
|
+
db: Any,
|
|
256
|
+
embedder: Any,
|
|
257
|
+
batch_size: int = _BACKFILL_BATCH_SIZE,
|
|
258
|
+
limit: int | None = None,
|
|
259
|
+
all_profiles: bool = False,
|
|
260
|
+
) -> dict[str, int]:
|
|
261
|
+
"""Embed atomic_facts rows whose ``embedding`` column is NULL.
|
|
262
|
+
|
|
263
|
+
Unlike :func:`run_embedding_migration` (which re-embeds on model-signature
|
|
264
|
+
change), this function handles facts that were *never* embedded — for
|
|
265
|
+
example facts stored while the embedder was unavailable.
|
|
266
|
+
|
|
267
|
+
Resumable and idempotent: re-running after a partial run only processes
|
|
268
|
+
the remaining NULLs. Fail-open per-fact: a single bad fact logs a warning
|
|
269
|
+
and is skipped; the batch continues.
|
|
270
|
+
|
|
271
|
+
Writes mirror :func:`run_embedding_migration` exactly:
|
|
272
|
+
* ``atomic_facts.embedding`` ← ``json.dumps(vector)``
|
|
273
|
+
* ``embedding_metadata`` ← upserted row with current model name + dimension
|
|
274
|
+
|
|
275
|
+
Args:
|
|
276
|
+
config: Active SLMConfig (provides profile_id, model name, dimension).
|
|
277
|
+
db: DatabaseManager (or duck-compatible object with ``.execute()``).
|
|
278
|
+
embedder: Object implementing ``embed_batch(texts) -> list[vec|None]``
|
|
279
|
+
and (optionally) ``embed(text) -> vec|None``. Pass ``None`` to
|
|
280
|
+
make this a no-op (returns zero counts).
|
|
281
|
+
batch_size: Facts per embed_batch() call. Defaults to 50.
|
|
282
|
+
limit: Maximum facts to embed in this call. ``None`` means no cap —
|
|
283
|
+
all NULL-embedding facts are processed. Use a bounded limit for
|
|
284
|
+
the maintenance self-healing path so each pass is quick.
|
|
285
|
+
all_profiles: When ``True``, processes facts from every profile in the
|
|
286
|
+
database. When ``False`` (default), scopes to
|
|
287
|
+
``config.active_profile``.
|
|
288
|
+
|
|
289
|
+
Returns:
|
|
290
|
+
``{"scanned": int, "embedded": int, "remaining_null": int}``
|
|
291
|
+
|
|
292
|
+
*scanned*: total NULL-embedding facts found before applying *limit*.
|
|
293
|
+
*embedded*: facts successfully written in this call.
|
|
294
|
+
*remaining_null*: NULL count after the call (includes facts not yet
|
|
295
|
+
reached because of *limit*).
|
|
296
|
+
"""
|
|
297
|
+
profile_id = config.active_profile
|
|
298
|
+
|
|
299
|
+
if embedder is None:
|
|
300
|
+
logger.warning(
|
|
301
|
+
"backfill_missing_embeddings: no embedder available — skipping."
|
|
302
|
+
)
|
|
303
|
+
return {"scanned": 0, "embedded": 0, "remaining_null": 0}
|
|
304
|
+
|
|
305
|
+
# ------------------------------------------------------------------
|
|
306
|
+
# 1. Fetch all NULL-embedding facts (cheap query; only reads IDs + content)
|
|
307
|
+
# ------------------------------------------------------------------
|
|
308
|
+
if all_profiles:
|
|
309
|
+
rows = db.execute(
|
|
310
|
+
"SELECT fact_id, content, profile_id FROM atomic_facts "
|
|
311
|
+
"WHERE embedding IS NULL ORDER BY created_at",
|
|
312
|
+
)
|
|
313
|
+
else:
|
|
314
|
+
rows = db.execute(
|
|
315
|
+
"SELECT fact_id, content, profile_id FROM atomic_facts "
|
|
316
|
+
"WHERE embedding IS NULL AND profile_id = ? ORDER BY created_at",
|
|
317
|
+
(profile_id,),
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
facts: list[tuple[str, str, str]] = [
|
|
321
|
+
(dict(r)["fact_id"], dict(r)["content"], dict(r)["profile_id"])
|
|
322
|
+
for r in rows
|
|
323
|
+
]
|
|
324
|
+
scanned = len(facts)
|
|
325
|
+
|
|
326
|
+
if scanned == 0:
|
|
327
|
+
return {"scanned": 0, "embedded": 0, "remaining_null": 0}
|
|
328
|
+
|
|
329
|
+
# Apply call-level limit (resumability: next call picks up where this left off)
|
|
330
|
+
if limit is not None:
|
|
331
|
+
facts = facts[:limit]
|
|
332
|
+
|
|
333
|
+
current_model = config.embedding.model_name
|
|
334
|
+
current_dim = config.embedding.dimension
|
|
335
|
+
embedded = 0
|
|
336
|
+
|
|
337
|
+
# ------------------------------------------------------------------
|
|
338
|
+
# 2. Batch embed and write back
|
|
339
|
+
# ------------------------------------------------------------------
|
|
340
|
+
for batch_start in range(0, len(facts), batch_size):
|
|
341
|
+
batch = facts[batch_start : batch_start + batch_size]
|
|
342
|
+
# Bound per-fact input so an oversized document doesn't monopolize the
|
|
343
|
+
# shared embedding worker (starving foreground recall during self-heal).
|
|
344
|
+
texts = [(content or "")[:_MAX_EMBED_CHARS] for _, content, _ in batch]
|
|
345
|
+
fact_ids = [fid for fid, _, _ in batch]
|
|
346
|
+
prof_ids = [pid for _, _, pid in batch]
|
|
347
|
+
|
|
348
|
+
# Attempt batch embed; fall back to per-fact on batch failure.
|
|
349
|
+
try:
|
|
350
|
+
vectors: list[Any] = embedder.embed_batch(texts)
|
|
351
|
+
except Exception as exc:
|
|
352
|
+
logger.warning(
|
|
353
|
+
"backfill: batch embed failed for facts %d-%d: %s — "
|
|
354
|
+
"retrying per-fact.",
|
|
355
|
+
batch_start,
|
|
356
|
+
batch_start + len(batch),
|
|
357
|
+
exc,
|
|
358
|
+
)
|
|
359
|
+
vectors = []
|
|
360
|
+
for text in texts:
|
|
361
|
+
try:
|
|
362
|
+
vec = embedder.embed(text)
|
|
363
|
+
vectors.append(vec)
|
|
364
|
+
except Exception as per_fact_exc:
|
|
365
|
+
logger.warning(
|
|
366
|
+
"backfill: per-fact embed failed for '%s...': %s",
|
|
367
|
+
text[:40],
|
|
368
|
+
per_fact_exc,
|
|
369
|
+
)
|
|
370
|
+
vectors.append(None)
|
|
371
|
+
|
|
372
|
+
# Write each successfully-embedded fact back to the DB.
|
|
373
|
+
for fid, vec, pid in zip(fact_ids, vectors, prof_ids):
|
|
374
|
+
if vec is None:
|
|
375
|
+
logger.warning(
|
|
376
|
+
"backfill: null vector for fact %s — skipping.", fid[:16]
|
|
377
|
+
)
|
|
378
|
+
continue
|
|
379
|
+
try:
|
|
380
|
+
embedding_json = json.dumps(vec)
|
|
381
|
+
# Mirror run_embedding_migration's write path exactly.
|
|
382
|
+
db.execute(
|
|
383
|
+
"UPDATE atomic_facts SET embedding = ? WHERE fact_id = ?",
|
|
384
|
+
(embedding_json, fid),
|
|
385
|
+
)
|
|
386
|
+
# Upsert embedding_metadata. NULL-embedding facts have no row
|
|
387
|
+
# here yet, so we INSERT; if a row somehow exists, update it.
|
|
388
|
+
db.execute(
|
|
389
|
+
"INSERT INTO embedding_metadata"
|
|
390
|
+
" (fact_id, profile_id, model_name, dimension)"
|
|
391
|
+
" VALUES (?, ?, ?, ?)"
|
|
392
|
+
" ON CONFLICT(fact_id) DO UPDATE SET"
|
|
393
|
+
" model_name = excluded.model_name",
|
|
394
|
+
(fid, pid, current_model, current_dim),
|
|
395
|
+
)
|
|
396
|
+
embedded += 1
|
|
397
|
+
except Exception as exc:
|
|
398
|
+
logger.warning(
|
|
399
|
+
"backfill: failed to write fact %s: %s", fid[:16], exc
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
# ------------------------------------------------------------------
|
|
403
|
+
# 3. Count remaining NULLs (accounts for the limit; tells caller how
|
|
404
|
+
# many passes remain before full convergence).
|
|
405
|
+
# ------------------------------------------------------------------
|
|
406
|
+
remaining = _count_null_embeddings(db, profile_id, all_profiles)
|
|
407
|
+
|
|
408
|
+
logger.info(
|
|
409
|
+
"Embedding backfill: %d/%d facts embedded, %d remaining NULL.",
|
|
410
|
+
embedded,
|
|
411
|
+
scanned,
|
|
412
|
+
remaining,
|
|
413
|
+
)
|
|
414
|
+
return {"scanned": scanned, "embedded": embedded, "remaining_null": remaining}
|
|
@@ -113,6 +113,15 @@ from superlocalmemory.storage.migrations import (
|
|
|
113
113
|
from superlocalmemory.storage.migrations import (
|
|
114
114
|
M027_transferable_patterns_profile as _M027,
|
|
115
115
|
)
|
|
116
|
+
from superlocalmemory.storage.migrations import (
|
|
117
|
+
M028_fact_entity_associations as _M028,
|
|
118
|
+
)
|
|
119
|
+
from superlocalmemory.storage.migrations import (
|
|
120
|
+
M029_behavioral_history_indexes as _M029,
|
|
121
|
+
)
|
|
122
|
+
from superlocalmemory.storage.migrations import (
|
|
123
|
+
M030_entity_explorer_indexes as _M030,
|
|
124
|
+
)
|
|
116
125
|
|
|
117
126
|
# Map migration name → module (used for the optional ``verify(conn)`` hook
|
|
118
127
|
# that lets the runner detect "already applied" state when an idempotent
|
|
@@ -144,6 +153,9 @@ _MODULES = {
|
|
|
144
153
|
_M025.NAME: _M025,
|
|
145
154
|
_M026.NAME: _M026,
|
|
146
155
|
_M027.NAME: _M027,
|
|
156
|
+
_M028.NAME: _M028,
|
|
157
|
+
_M029.NAME: _M029,
|
|
158
|
+
_M030.NAME: _M030,
|
|
147
159
|
}
|
|
148
160
|
|
|
149
161
|
logger = logging.getLogger(__name__)
|
|
@@ -218,6 +230,10 @@ MIGRATIONS: list[Migration] = [
|
|
|
218
230
|
# position proxy when the column is absent, so a failed deferred apply never
|
|
219
231
|
# crashes the trainer — it just keeps the old label path.
|
|
220
232
|
DEFERRED_MIGRATIONS: list[Migration] = [
|
|
233
|
+
# M028 captures an atomic_facts rowid high-water mark before readiness.
|
|
234
|
+
# atomic_facts/canonical_entities are bootstrapped by MemoryEngine.
|
|
235
|
+
Migration(name=_M028.NAME, db_target="memory", ddl=_M028.DDL,
|
|
236
|
+
dependencies=(_M018.NAME,)),
|
|
221
237
|
Migration(name=_M006.NAME, db_target="memory", ddl=_M006.DDL),
|
|
222
238
|
# M011 extends atomic_facts + creates memory_archive / memory_merge_log.
|
|
223
239
|
# atomic_facts is bootstrapped at engine init, so M011 defers alongside M006.
|
|
@@ -256,6 +272,10 @@ DEFERRED_MIGRATIONS: list[Migration] = [
|
|
|
256
272
|
# consolidation run, not during engine init or apply_all. apply() is a no-op
|
|
257
273
|
# when the table is absent (first install after the schema change).
|
|
258
274
|
Migration(name=_M027.NAME, db_target="learning", ddl=_M027.DDL),
|
|
275
|
+
# M029 indexes behavioral tables bootstrapped during engine initialization.
|
|
276
|
+
Migration(name=_M029.NAME, db_target="memory", ddl=_M029.DDL),
|
|
277
|
+
# M030 bounds Entity Explorer pagination and profile-summary ranking.
|
|
278
|
+
Migration(name=_M030.NAME, db_target="memory", ddl=_M030.DDL),
|
|
259
279
|
]
|
|
260
280
|
|
|
261
281
|
|
|
@@ -401,7 +421,65 @@ def _apply_single(
|
|
|
401
421
|
)
|
|
402
422
|
logger.warning(detail)
|
|
403
423
|
return ("failed", detail)
|
|
404
|
-
|
|
424
|
+
# A matching migration-log row is not proof that the promised
|
|
425
|
+
# schema still exists. Existing installs can retain a migration log
|
|
426
|
+
# while a partial restore drops an additive table or index.
|
|
427
|
+
#
|
|
428
|
+
# Never replay a historical migration merely because verify()
|
|
429
|
+
# fails. Some migrations rebuild tables and transform data; replay
|
|
430
|
+
# would be destructive (M002 is the canonical example). Only a
|
|
431
|
+
# module-supplied repair(conn) hook is allowed to reconcile a
|
|
432
|
+
# completed migration's end-state.
|
|
433
|
+
mod = _MODULES.get(migration.name)
|
|
434
|
+
verify_fn = (
|
|
435
|
+
getattr(mod, "verify", None) if mod is not None else None
|
|
436
|
+
)
|
|
437
|
+
if verify_fn is None:
|
|
438
|
+
return ("skipped", "already complete")
|
|
439
|
+
try:
|
|
440
|
+
schema_complete = bool(verify_fn(conn))
|
|
441
|
+
except sqlite3.Error as exc:
|
|
442
|
+
return (
|
|
443
|
+
"failed",
|
|
444
|
+
f"schema verification failed for {migration.name}: {exc}",
|
|
445
|
+
)
|
|
446
|
+
if schema_complete:
|
|
447
|
+
return ("skipped", "already complete (schema verified)")
|
|
448
|
+
if dry_run:
|
|
449
|
+
return (
|
|
450
|
+
"skipped",
|
|
451
|
+
"dry-run: would repair missing migration end-state",
|
|
452
|
+
)
|
|
453
|
+
repair_fn = (
|
|
454
|
+
getattr(mod, "repair", None) if mod is not None else None
|
|
455
|
+
)
|
|
456
|
+
if not callable(repair_fn):
|
|
457
|
+
detail = (
|
|
458
|
+
f"schema incomplete for completed migration "
|
|
459
|
+
f"{migration.name}; automatic replay is disabled"
|
|
460
|
+
)
|
|
461
|
+
logger.warning(detail)
|
|
462
|
+
return ("failed", detail)
|
|
463
|
+
try:
|
|
464
|
+
repair_fn(conn)
|
|
465
|
+
except sqlite3.Error as exc:
|
|
466
|
+
return (
|
|
467
|
+
"failed",
|
|
468
|
+
f"safe repair failed for {migration.name}: {exc}",
|
|
469
|
+
)
|
|
470
|
+
try:
|
|
471
|
+
if not bool(verify_fn(conn)):
|
|
472
|
+
return (
|
|
473
|
+
"failed",
|
|
474
|
+
f"safe repair did not restore {migration.name}",
|
|
475
|
+
)
|
|
476
|
+
except sqlite3.Error as exc:
|
|
477
|
+
return (
|
|
478
|
+
"failed",
|
|
479
|
+
f"post-repair verification failed for "
|
|
480
|
+
f"{migration.name}: {exc}",
|
|
481
|
+
)
|
|
482
|
+
return ("applied", "missing end-state repaired safely")
|
|
405
483
|
# status is 'failed' or 'in_progress' → retry from scratch.
|
|
406
484
|
if dry_run:
|
|
407
485
|
return ("skipped", f"dry-run: would retry (status={status})")
|
|
@@ -50,6 +50,11 @@ def verify(conn: sqlite3.Connection) -> bool:
|
|
|
50
50
|
return _REQUIRED_TABLES.issubset(names)
|
|
51
51
|
|
|
52
52
|
|
|
53
|
+
def repair(conn: sqlite3.Connection) -> None:
|
|
54
|
+
"""Restore additive Skill Evolution tables without replaying other migrations."""
|
|
55
|
+
conn.executescript(DDL)
|
|
56
|
+
|
|
57
|
+
|
|
53
58
|
DDL = """
|
|
54
59
|
CREATE TABLE IF NOT EXISTS evolution_config (
|
|
55
60
|
profile_id TEXT PRIMARY KEY,
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
"""M028 — normalized fact/entity associations for O(1) ingest effects.
|
|
5
|
+
|
|
6
|
+
The table is both an association index and an idempotency ledger. Its
|
|
7
|
+
composite primary key means a fact can contribute to an entity's ``fact_count``
|
|
8
|
+
once, regardless of ingestion retries or how many operations converge on the
|
|
9
|
+
same consolidated fact.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import sqlite3
|
|
16
|
+
from datetime import UTC, datetime
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
NAME = "M028_fact_entity_associations"
|
|
20
|
+
DB_TARGET = "memory"
|
|
21
|
+
|
|
22
|
+
DDL = """
|
|
23
|
+
CREATE TABLE IF NOT EXISTS fact_entity_associations (
|
|
24
|
+
profile_id TEXT NOT NULL,
|
|
25
|
+
fact_id TEXT NOT NULL,
|
|
26
|
+
entity_id TEXT NOT NULL,
|
|
27
|
+
first_operation_id TEXT NOT NULL DEFAULT '',
|
|
28
|
+
count_applied INTEGER NOT NULL DEFAULT 0
|
|
29
|
+
CHECK (count_applied IN (0, 1)),
|
|
30
|
+
created_at TEXT NOT NULL DEFAULT (
|
|
31
|
+
strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
|
|
32
|
+
),
|
|
33
|
+
PRIMARY KEY (profile_id, fact_id, entity_id),
|
|
34
|
+
FOREIGN KEY (fact_id) REFERENCES atomic_facts(fact_id) ON DELETE CASCADE,
|
|
35
|
+
FOREIGN KEY (entity_id)
|
|
36
|
+
REFERENCES canonical_entities(entity_id) ON DELETE CASCADE
|
|
37
|
+
);
|
|
38
|
+
CREATE INDEX IF NOT EXISTS idx_fact_entity_associations_entity
|
|
39
|
+
ON fact_entity_associations(profile_id, entity_id, fact_id);
|
|
40
|
+
CREATE TABLE IF NOT EXISTS fact_entity_association_repair_state (
|
|
41
|
+
repair_key TEXT PRIMARY KEY,
|
|
42
|
+
state TEXT NOT NULL DEFAULT 'pending'
|
|
43
|
+
CHECK (state IN ('pending', 'running', 'retrying', 'complete')),
|
|
44
|
+
target_fact_rowid INTEGER NOT NULL DEFAULT -1,
|
|
45
|
+
last_fact_rowid INTEGER NOT NULL DEFAULT 0,
|
|
46
|
+
scanned INTEGER NOT NULL DEFAULT 0,
|
|
47
|
+
inserted INTEGER NOT NULL DEFAULT 0,
|
|
48
|
+
last_error TEXT NOT NULL DEFAULT '',
|
|
49
|
+
updated_at TEXT NOT NULL
|
|
50
|
+
);
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
|
|
55
|
+
return conn.execute(
|
|
56
|
+
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?",
|
|
57
|
+
(table,),
|
|
58
|
+
).fetchone() is not None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def apply(conn: sqlite3.Connection) -> None:
|
|
62
|
+
"""Install schema and capture a constant-time historical rowid boundary."""
|
|
63
|
+
conn.executescript(DDL)
|
|
64
|
+
association_columns = {
|
|
65
|
+
row[1]
|
|
66
|
+
for row in conn.execute(
|
|
67
|
+
"PRAGMA table_info(fact_entity_associations)"
|
|
68
|
+
).fetchall()
|
|
69
|
+
}
|
|
70
|
+
if "count_applied" not in association_columns:
|
|
71
|
+
conn.execute(
|
|
72
|
+
"ALTER TABLE fact_entity_associations "
|
|
73
|
+
"ADD COLUMN count_applied INTEGER NOT NULL DEFAULT 0 "
|
|
74
|
+
"CHECK (count_applied IN (0, 1))"
|
|
75
|
+
)
|
|
76
|
+
repair_columns = {
|
|
77
|
+
row[1]
|
|
78
|
+
for row in conn.execute(
|
|
79
|
+
"PRAGMA table_info(fact_entity_association_repair_state)"
|
|
80
|
+
).fetchall()
|
|
81
|
+
}
|
|
82
|
+
if "target_fact_rowid" not in repair_columns:
|
|
83
|
+
conn.execute(
|
|
84
|
+
"ALTER TABLE fact_entity_association_repair_state "
|
|
85
|
+
"ADD COLUMN target_fact_rowid INTEGER NOT NULL DEFAULT -1"
|
|
86
|
+
)
|
|
87
|
+
target = int(conn.execute(
|
|
88
|
+
"SELECT COALESCE(MAX(rowid), 0) FROM atomic_facts"
|
|
89
|
+
).fetchone()[0])
|
|
90
|
+
conn.executescript(
|
|
91
|
+
"INSERT OR IGNORE INTO fact_entity_association_repair_state "
|
|
92
|
+
"(repair_key,state,target_fact_rowid,updated_at) "
|
|
93
|
+
f"VALUES ('historical-backfill','pending',{target},'');"
|
|
94
|
+
"UPDATE fact_entity_association_repair_state "
|
|
95
|
+
f"SET target_fact_rowid={target} "
|
|
96
|
+
"WHERE repair_key='historical-backfill' AND target_fact_rowid < 0;"
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _now() -> str:
|
|
101
|
+
return datetime.now(UTC).isoformat()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _connect(db_path: Path) -> sqlite3.Connection:
|
|
105
|
+
conn = sqlite3.connect(str(db_path), timeout=5)
|
|
106
|
+
conn.row_factory = sqlite3.Row
|
|
107
|
+
conn.execute("PRAGMA busy_timeout=5000")
|
|
108
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
109
|
+
return conn
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def get_repair_status(db_path: Path) -> dict[str, int | str]:
|
|
113
|
+
"""Read durable backfill progress without inferring it from schema."""
|
|
114
|
+
conn = _connect(Path(db_path))
|
|
115
|
+
try:
|
|
116
|
+
row = conn.execute(
|
|
117
|
+
"SELECT state,target_fact_rowid,last_fact_rowid,scanned,inserted,"
|
|
118
|
+
"last_error,updated_at "
|
|
119
|
+
"FROM fact_entity_association_repair_state "
|
|
120
|
+
"WHERE repair_key='historical-backfill'"
|
|
121
|
+
).fetchone()
|
|
122
|
+
if row is None:
|
|
123
|
+
return {
|
|
124
|
+
"state": "pending", "target_fact_rowid": 0,
|
|
125
|
+
"last_fact_rowid": 0,
|
|
126
|
+
"scanned": 0, "inserted": 0,
|
|
127
|
+
"last_error": "", "updated_at": "",
|
|
128
|
+
}
|
|
129
|
+
return dict(row)
|
|
130
|
+
finally:
|
|
131
|
+
conn.close()
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _entity_ids(raw: object) -> tuple[str, ...]:
|
|
135
|
+
try:
|
|
136
|
+
values = json.loads(str(raw or "[]"))
|
|
137
|
+
except (TypeError, ValueError):
|
|
138
|
+
return ()
|
|
139
|
+
if not isinstance(values, list):
|
|
140
|
+
return ()
|
|
141
|
+
return tuple(dict.fromkeys(str(value) for value in values if value))
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _repair_batch(conn: sqlite3.Connection, batch_size: int) -> dict[str, int | bool]:
|
|
145
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
146
|
+
try:
|
|
147
|
+
status = conn.execute(
|
|
148
|
+
"SELECT last_fact_rowid,target_fact_rowid "
|
|
149
|
+
"FROM fact_entity_association_repair_state "
|
|
150
|
+
"WHERE repair_key='historical-backfill'"
|
|
151
|
+
).fetchone()
|
|
152
|
+
cursor = int(status["last_fact_rowid"] or 0)
|
|
153
|
+
target = int(status["target_fact_rowid"])
|
|
154
|
+
rows = conn.execute(
|
|
155
|
+
"SELECT rowid,fact_id,profile_id,canonical_entities_json "
|
|
156
|
+
"FROM atomic_facts WHERE rowid>? AND rowid<=? "
|
|
157
|
+
"ORDER BY rowid LIMIT ?",
|
|
158
|
+
(cursor, target, batch_size),
|
|
159
|
+
).fetchall()
|
|
160
|
+
if not rows:
|
|
161
|
+
conn.execute(
|
|
162
|
+
"UPDATE fact_entity_association_repair_state "
|
|
163
|
+
"SET state='complete',last_error='',updated_at=? "
|
|
164
|
+
"WHERE repair_key='historical-backfill'",
|
|
165
|
+
(_now(),),
|
|
166
|
+
)
|
|
167
|
+
conn.commit()
|
|
168
|
+
return {"scanned": 0, "inserted": 0, "complete": True}
|
|
169
|
+
inserted = 0
|
|
170
|
+
for row in rows:
|
|
171
|
+
for entity_id in _entity_ids(row["canonical_entities_json"]):
|
|
172
|
+
result = conn.execute(
|
|
173
|
+
"INSERT OR IGNORE INTO fact_entity_associations "
|
|
174
|
+
"(profile_id,fact_id,entity_id,first_operation_id,"
|
|
175
|
+
"count_applied) "
|
|
176
|
+
"SELECT ?,?,?,?,? FROM canonical_entities "
|
|
177
|
+
"WHERE profile_id=? AND entity_id=?",
|
|
178
|
+
(
|
|
179
|
+
row["profile_id"], row["fact_id"], entity_id,
|
|
180
|
+
"migration-backfill", 0,
|
|
181
|
+
row["profile_id"], entity_id,
|
|
182
|
+
),
|
|
183
|
+
)
|
|
184
|
+
inserted += max(0, result.rowcount)
|
|
185
|
+
conn.execute(
|
|
186
|
+
"UPDATE fact_entity_association_repair_state SET "
|
|
187
|
+
"state='running',last_fact_rowid=?,scanned=scanned+?,"
|
|
188
|
+
"inserted=inserted+?,last_error='',updated_at=? "
|
|
189
|
+
"WHERE repair_key='historical-backfill'",
|
|
190
|
+
(int(rows[-1]["rowid"]), len(rows), inserted, _now()),
|
|
191
|
+
)
|
|
192
|
+
conn.commit()
|
|
193
|
+
return {"scanned": len(rows), "inserted": inserted, "complete": False}
|
|
194
|
+
except Exception:
|
|
195
|
+
conn.rollback()
|
|
196
|
+
raise
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def repair_fact_entity_associations(
|
|
200
|
+
db_path: Path,
|
|
201
|
+
*,
|
|
202
|
+
batch_size: int = 250,
|
|
203
|
+
max_batches: int = 1,
|
|
204
|
+
) -> dict[str, int | bool]:
|
|
205
|
+
"""Run bounded, restartable short-transaction backfill batches."""
|
|
206
|
+
if batch_size < 1 or max_batches < 1:
|
|
207
|
+
raise ValueError("batch_size and max_batches must be positive")
|
|
208
|
+
totals = {"scanned": 0, "inserted": 0, "complete": False}
|
|
209
|
+
conn = _connect(Path(db_path))
|
|
210
|
+
try:
|
|
211
|
+
for _ in range(max_batches):
|
|
212
|
+
result = _repair_batch(conn, batch_size)
|
|
213
|
+
totals["scanned"] += int(result["scanned"])
|
|
214
|
+
totals["inserted"] += int(result["inserted"])
|
|
215
|
+
totals["complete"] = bool(result["complete"])
|
|
216
|
+
if totals["complete"]:
|
|
217
|
+
break
|
|
218
|
+
return totals
|
|
219
|
+
except sqlite3.Error as exc:
|
|
220
|
+
try:
|
|
221
|
+
conn.execute(
|
|
222
|
+
"UPDATE fact_entity_association_repair_state SET "
|
|
223
|
+
"state='retrying',last_error=?,updated_at=? "
|
|
224
|
+
"WHERE repair_key='historical-backfill'",
|
|
225
|
+
(type(exc).__name__, _now()),
|
|
226
|
+
)
|
|
227
|
+
conn.commit()
|
|
228
|
+
except sqlite3.Error:
|
|
229
|
+
pass
|
|
230
|
+
raise
|
|
231
|
+
finally:
|
|
232
|
+
conn.close()
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def verify(conn: sqlite3.Connection) -> bool:
|
|
236
|
+
"""Return true when the normalized association contract is indexed."""
|
|
237
|
+
if not (
|
|
238
|
+
_table_exists(conn, "fact_entity_associations")
|
|
239
|
+
and _table_exists(conn, "fact_entity_association_repair_state")
|
|
240
|
+
):
|
|
241
|
+
return False
|
|
242
|
+
association_columns = {
|
|
243
|
+
row[1]
|
|
244
|
+
for row in conn.execute(
|
|
245
|
+
"PRAGMA table_info(fact_entity_associations)"
|
|
246
|
+
).fetchall()
|
|
247
|
+
}
|
|
248
|
+
repair_columns = {
|
|
249
|
+
row[1]
|
|
250
|
+
for row in conn.execute(
|
|
251
|
+
"PRAGMA table_info(fact_entity_association_repair_state)"
|
|
252
|
+
).fetchall()
|
|
253
|
+
}
|
|
254
|
+
if (
|
|
255
|
+
"count_applied" not in association_columns
|
|
256
|
+
or "target_fact_rowid" not in repair_columns
|
|
257
|
+
):
|
|
258
|
+
return False
|
|
259
|
+
indexes = {
|
|
260
|
+
row[1]
|
|
261
|
+
for row in conn.execute(
|
|
262
|
+
"PRAGMA index_list(fact_entity_associations)"
|
|
263
|
+
).fetchall()
|
|
264
|
+
}
|
|
265
|
+
return "idx_fact_entity_associations_entity" in indexes
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def repair(conn: sqlite3.Connection) -> None:
|
|
269
|
+
"""Restore additive M028 schema without recapturing its high-water mark."""
|
|
270
|
+
apply(conn)
|