superlocalmemory 4.0.10 → 4.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/.claude-plugin/marketplace.json +12 -2
  2. package/CHANGELOG.md +244 -0
  3. package/README.md +40 -75
  4. package/package.json +6 -3
  5. package/plugin/.claude-plugin/plugin.json +2 -2
  6. package/plugin/CLAUDE.md +3 -3
  7. package/plugin/agents/slm-governance-advisor.md +1 -1
  8. package/plugin/agents/slm-loop-runner.md +4 -4
  9. package/plugin/agents/slm-memory-advisor.md +1 -1
  10. package/plugin/agents/slm-optimize-advisor.md +1 -1
  11. package/plugin/requirements.txt +1 -1
  12. package/plugin/skills/slm-cache/SKILL.md +1 -1
  13. package/plugin/skills/slm-compress/SKILL.md +1 -1
  14. package/plugin/skills/slm-governance/SKILL.md +1 -1
  15. package/plugin/skills/slm-graph/SKILL.md +1 -1
  16. package/plugin/skills/slm-loop/SKILL.md +2 -2
  17. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  18. package/plugin/skills/slm-profile/SKILL.md +5 -5
  19. package/plugin/skills/slm-recall/SKILL.md +102 -15
  20. package/plugin/skills/slm-remember/SKILL.md +35 -3
  21. package/plugin/skills/slm-scope/SKILL.md +1 -1
  22. package/plugin/skills/slm-session/SKILL.md +29 -3
  23. package/plugin/skills/slm-status/SKILL.md +1 -1
  24. package/plugin-src/rules/AGENTS.md +16 -8
  25. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  26. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-governance/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  29. package/plugin-src/skills/slm-loop/SKILL.md +2 -2
  30. package/plugin-src/skills/slm-mesh/SKILL.md +1 -1
  31. package/plugin-src/skills/slm-profile/SKILL.md +5 -5
  32. package/plugin-src/skills/slm-recall/SKILL.md +102 -15
  33. package/plugin-src/skills/slm-remember/SKILL.md +35 -3
  34. package/plugin-src/skills/slm-scope/SKILL.md +1 -1
  35. package/plugin-src/skills/slm-session/SKILL.md +29 -3
  36. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  37. package/pyproject.toml +1 -1
  38. package/src/superlocalmemory/__init__.py +1 -1
  39. package/src/superlocalmemory/cli/commands.py +357 -18
  40. package/src/superlocalmemory/cli/daemon.py +30 -0
  41. package/src/superlocalmemory/cli/db_migrate.py +71 -1
  42. package/src/superlocalmemory/cli/gdpr_cmd.py +15 -2
  43. package/src/superlocalmemory/cli/main.py +24 -2
  44. package/src/superlocalmemory/code_graph/database.py +44 -0
  45. package/src/superlocalmemory/compliance/gdpr.py +449 -39
  46. package/src/superlocalmemory/core/admission.py +231 -11
  47. package/src/superlocalmemory/core/backend_orchestrator.py +190 -84
  48. package/src/superlocalmemory/core/config.py +90 -11
  49. package/src/superlocalmemory/core/consolidation_engine.py +34 -0
  50. package/src/superlocalmemory/core/engine.py +140 -11
  51. package/src/superlocalmemory/core/graph_analyzer.py +76 -112
  52. package/src/superlocalmemory/core/graph_metrics.py +597 -0
  53. package/src/superlocalmemory/core/graph_pruner.py +121 -0
  54. package/src/superlocalmemory/core/maintenance_scheduler.py +205 -0
  55. package/src/superlocalmemory/core/mode_capability.py +111 -0
  56. package/src/superlocalmemory/core/ollama_validator.py +315 -0
  57. package/src/superlocalmemory/core/projection_drain.py +380 -0
  58. package/src/superlocalmemory/core/recall_pipeline.py +390 -3
  59. package/src/superlocalmemory/core/recall_worker.py +6 -3
  60. package/src/superlocalmemory/core/scale_autopromote.py +196 -0
  61. package/src/superlocalmemory/core/scale_engine.py +16 -2
  62. package/src/superlocalmemory/core/score_contract.py +21 -1
  63. package/src/superlocalmemory/core/session_identity.py +85 -0
  64. package/src/superlocalmemory/core/status_contract.py +108 -0
  65. package/src/superlocalmemory/core/worker_pool.py +4 -4
  66. package/src/superlocalmemory/core/working_memory.py +288 -0
  67. package/src/superlocalmemory/encoding/cognitive_consolidator.py +36 -6
  68. package/src/superlocalmemory/encoding/context_generator.py +1 -1
  69. package/src/superlocalmemory/encoding/entity_resolver.py +38 -0
  70. package/src/superlocalmemory/encoding/fact_extractor.py +18 -14
  71. package/src/superlocalmemory/encoding/prospective_markers.py +262 -0
  72. package/src/superlocalmemory/encoding/type_router.py +12 -12
  73. package/src/superlocalmemory/evolution/mutation_generator.py +30 -4
  74. package/src/superlocalmemory/graph/cozo_adjacency.py +122 -0
  75. package/src/superlocalmemory/graph/cozo_backend.py +103 -138
  76. package/src/superlocalmemory/hooks/portable_kit.py +10 -2
  77. package/src/superlocalmemory/learning/bandit.py +43 -0
  78. package/src/superlocalmemory/learning/consolidation_worker.py +54 -0
  79. package/src/superlocalmemory/learning/database.py +60 -3
  80. package/src/superlocalmemory/learning/entity_compiler.py +21 -58
  81. package/src/superlocalmemory/learning/feedback.py +3 -1
  82. package/src/superlocalmemory/learning/outcomes.py +47 -16
  83. package/src/superlocalmemory/learning/pattern_miner.py +28 -3
  84. package/src/superlocalmemory/learning/pattern_miner_constants.py +43 -0
  85. package/src/superlocalmemory/learning/pcos.py +291 -0
  86. package/src/superlocalmemory/learning/reward_from_outcomes.py +365 -0
  87. package/src/superlocalmemory/learning/reward_proxy.py +100 -10
  88. package/src/superlocalmemory/learning/signal_kinds.py +79 -0
  89. package/src/superlocalmemory/mcp/profiles.py +14 -2
  90. package/src/superlocalmemory/mcp/tools_active.py +2 -1
  91. package/src/superlocalmemory/mcp/tools_core.py +31 -3
  92. package/src/superlocalmemory/mcp/tools_v28.py +20 -1
  93. package/src/superlocalmemory/parameterization/pattern_extractor.py +14 -1
  94. package/src/superlocalmemory/parameterization/soft_prompt_generator.py +98 -0
  95. package/src/superlocalmemory/retrieval/bm25_channel.py +64 -3
  96. package/src/superlocalmemory/retrieval/channel_status.py +117 -0
  97. package/src/superlocalmemory/retrieval/engine.py +106 -11
  98. package/src/superlocalmemory/retrieval/entity_channel.py +210 -256
  99. package/src/superlocalmemory/retrieval/graph_adjacency.py +219 -0
  100. package/src/superlocalmemory/retrieval/scope_policy.py +20 -0
  101. package/src/superlocalmemory/retrieval/semantic_channel.py +47 -5
  102. package/src/superlocalmemory/retrieval/spreading.py +288 -0
  103. package/src/superlocalmemory/server/api.py +24 -5
  104. package/src/superlocalmemory/server/bandit_loops.py +17 -1
  105. package/src/superlocalmemory/server/rbac_enforce.py +26 -6
  106. package/src/superlocalmemory/server/recall_health.py +87 -10
  107. package/src/superlocalmemory/server/recall_serializer.py +9 -0
  108. package/src/superlocalmemory/server/routes/behavioral.py +75 -10
  109. package/src/superlocalmemory/server/routes/compliance.py +98 -18
  110. package/src/superlocalmemory/server/routes/config_api.py +186 -4
  111. package/src/superlocalmemory/server/routes/evolution.py +178 -0
  112. package/src/superlocalmemory/server/routes/ingest.py +8 -0
  113. package/src/superlocalmemory/server/routes/learning_telemetry.py +2 -1
  114. package/src/superlocalmemory/server/routes/memories.py +49 -7
  115. package/src/superlocalmemory/server/routes/timeline.py +4 -0
  116. package/src/superlocalmemory/server/routes/v3_api.py +191 -15
  117. package/src/superlocalmemory/server/ui.py +20 -4
  118. package/src/superlocalmemory/server/unified_daemon.py +241 -7
  119. package/src/superlocalmemory/storage/_migration_internals.py +54 -2
  120. package/src/superlocalmemory/storage/_schema_version.py +24 -3
  121. package/src/superlocalmemory/storage/database.py +477 -59
  122. package/src/superlocalmemory/storage/embedding_codec.py +71 -0
  123. package/src/superlocalmemory/storage/lineage_retention.py +236 -0
  124. package/src/superlocalmemory/storage/logical_edges.py +43 -2
  125. package/src/superlocalmemory/storage/migration_runner.py +119 -0
  126. package/src/superlocalmemory/storage/migrations/M043_quarantine_display_summaries.py +60 -36
  127. package/src/superlocalmemory/storage/migrations/M044_play_carries_its_own_evidence.py +127 -0
  128. package/src/superlocalmemory/storage/migrations/M045_fact_outcome_score.py +158 -0
  129. package/src/superlocalmemory/storage/migrations/M046_prospective_memory_has_its_own_name.py +620 -0
  130. package/src/superlocalmemory/storage/migrations/M047_fisher_vectors_are_stored_like_every_other_vector.py +306 -0
  131. package/src/superlocalmemory/storage/migrations/M048_upcoming_holds_only_what_is_upcoming.py +207 -0
  132. package/src/superlocalmemory/storage/migrations/M049_a_schema_version_marker_is_one_row.py +201 -0
  133. package/src/superlocalmemory/storage/migrations.py +18 -2
  134. package/src/superlocalmemory/storage/models.py +40 -1
  135. package/src/superlocalmemory/storage/projection_outbox.py +346 -0
  136. package/src/superlocalmemory/storage/retention_policy.py +860 -0
  137. package/src/superlocalmemory/storage/schema.py +35 -1
  138. package/src/superlocalmemory/storage/write_coordinator.py +19 -2
  139. package/src/superlocalmemory/trust/scorer.py +43 -1
  140. package/src/superlocalmemory/ui/index.html +9 -18
  141. package/src/superlocalmemory/ui/js/event-delegation.js +12 -1
  142. package/src/superlocalmemory/ui/js/od-health.js +28 -6
  143. package/src/superlocalmemory/ui/js/od-memories.js +19 -0
  144. package/src/superlocalmemory/ui/js/od-settings.js +87 -1
  145. package/src/superlocalmemory/ui/js/recall-lab.js +78 -3
@@ -8,14 +8,45 @@ from __future__ import annotations
8
8
 
9
9
  import json
10
10
  import logging
11
+ from pathlib import Path
11
12
  import os
12
13
  from fastapi import APIRouter, HTTPException, Request
13
14
  from fastapi.responses import JSONResponse
15
+ from superlocalmemory.core.config import CANONICAL_RECALL_LIMIT
16
+ from superlocalmemory.core.status_contract import (
17
+ COUNT_QUERIES,
18
+ counts_from_sqlite,
19
+ projection_queue_depth,
20
+ store_size_mb,
21
+ )
14
22
  from superlocalmemory.server.routes.helpers import SLM_VERSION, get_read_connection
15
23
  from superlocalmemory.server.route_mutations import authorize_route_mutation
16
24
 
17
25
  logger = logging.getLogger(__name__)
18
26
 
27
+ def _signal_session_id() -> str:
28
+ """A name for the caller, for matching an outcome back to this recall.
29
+
30
+ The agent id the request arrived under, when it arrived under one. Falls
31
+ back to the workspace, which keeps dashboard and scripted traffic separable
32
+ from an agent's. Never empty: an unnamed recall leaves no record.
33
+ """
34
+ try:
35
+ from superlocalmemory.mcp.agent_context import get_current_agent_id
36
+
37
+ agent = str(get_current_agent_id() or "").strip()
38
+ if agent:
39
+ return f"agent:{agent}"
40
+ except Exception: # noqa: BLE001 -- naming the caller must never fail a read
41
+ pass
42
+ try:
43
+ from superlocalmemory.server.routes.helpers import get_active_profile
44
+
45
+ return f"api:{get_active_profile()}"
46
+ except Exception: # noqa: BLE001
47
+ return "api:default"
48
+
49
+
19
50
  router = APIRouter(prefix="/api/v3", tags=["v3"])
20
51
 
21
52
 
@@ -102,20 +133,15 @@ async def dashboard(request: Request):
102
133
 
103
134
  # Read stats directly from SQLite (dashboard doesn't load engine)
104
135
  memory_count = 0
105
- fact_count = 0
136
+ counts = dict.fromkeys(COUNT_QUERIES, 0)
137
+ queue_depth = 0
106
138
  db_path = config.base_dir / "memory.db"
107
139
  if db_path.exists():
108
140
  try:
109
141
  conn = get_read_connection(db_path)
142
+ counts = counts_from_sqlite(conn, active_profile)
143
+ queue_depth = projection_queue_depth(conn)
110
144
  cursor = conn.cursor()
111
- try:
112
- cursor.execute(
113
- "SELECT COUNT(*) FROM atomic_facts WHERE profile_id = ?",
114
- (active_profile,),
115
- )
116
- fact_count = cursor.fetchone()[0]
117
- except Exception:
118
- pass
119
145
  try:
120
146
  try:
121
147
  cursor.execute(
@@ -143,10 +169,23 @@ async def dashboard(request: Request):
143
169
  "provider": config.llm.provider or "none",
144
170
  "model": config.llm.model or "",
145
171
  "memory_count": memory_count,
146
- "fact_count": fact_count,
147
172
  "profile": active_profile,
148
173
  "base_dir": str(config.base_dir),
149
174
  "version": SLM_VERSION,
175
+ # The counts and the store's own address were missing here while
176
+ # every other status surface carried them, so the one surface a
177
+ # person actually looks at could not answer "is the graph healthy".
178
+ "db_path": str(db_path),
179
+ "db_size_mb": store_size_mb(db_path),
180
+ "profile_generation": get_profile_runtime(
181
+ request.app.state,
182
+ ).snapshot.generation,
183
+ # Facts stored but not yet in the graph and vector projections. A
184
+ # number that does not fall is a projection that has stopped
185
+ # keeping up, which is otherwise invisible: nothing errors, the
186
+ # memory is safely in SQLite, and recall just quietly gets worse.
187
+ "projection_queue_depth": queue_depth,
188
+ **counts,
150
189
  }
151
190
  payload.update(dashboard_mode_fields(config.mode))
152
191
  return payload
@@ -495,12 +534,45 @@ async def set_full_config(request: Request):
495
534
  _emb_fields = ("embedding_provider", "embedding_endpoint", "embedding_key",
496
535
  "embedding_model", "embedding_dimension")
497
536
  if any(k in body for k in _emb_fields):
537
+ _old_emb = config.embedding
538
+ _new_provider = body.get("embedding_provider", "")
539
+ _new_model = body.get("embedding_model", "")
540
+ _new_dim = int(body.get("embedding_dimension", 0) or 0)
541
+ # The same range the other save route enforces. Without it a
542
+ # dashboard save with no dimension field stored a width of zero.
543
+ if _new_dim and not (64 <= _new_dim <= 8192):
544
+ return JSONResponse(
545
+ {"error": f"Dimension must be 64-8192, got {_new_dim}"},
546
+ status_code=400,
547
+ )
548
+ # The SECOND way to change the embedding model, and it was
549
+ # unguarded. Switching mode from the dashboard carries the embedding
550
+ # fields, so a width that the store cannot hold arrived here
551
+ # untouched while the other route refused it — one door bolted, the
552
+ # other open.
553
+ if not bool(body.get("force")):
554
+ _refusal = _refuse_incompatible_embedding(
555
+ config, config.embedding, _new_model, _new_dim,
556
+ new_provider=_new_provider,
557
+ )
558
+ if _refusal is not None:
559
+ return _refusal
498
560
  config.embedding = EmbeddingConfig(
499
- provider=body.get("embedding_provider", ""),
561
+ provider=_new_provider,
500
562
  api_endpoint=body.get("embedding_endpoint", ""),
501
563
  api_key=body.get("embedding_key", ""),
502
- model_name=body.get("embedding_model", ""),
503
- dimension=int(body.get("embedding_dimension", 0) or 0),
564
+ model_name=_new_model,
565
+ dimension=_new_dim or _old_emb.dimension,
566
+ # Not naming these reset them to defaults, so every embedding
567
+ # save from the dashboard silently put the local model back to
568
+ # whatever ships — the exact defect the other route had.
569
+ ollama_model=(
570
+ _new_model if _new_provider == "ollama" and _new_model
571
+ else _old_emb.ollama_model
572
+ ),
573
+ ollama_base_url=_old_emb.ollama_base_url,
574
+ api_version=_old_emb.api_version,
575
+ deployment_name=_old_emb.deployment_name,
504
576
  )
505
577
 
506
578
  # When the mode actually changed, apply the new mode's structural presets
@@ -571,6 +643,85 @@ async def get_embedding_config(request: Request):
571
643
  return _internal_error()
572
644
 
573
645
 
646
+ def _refuse_incompatible_embedding(
647
+ config, old_emb, new_model, new_dim, new_provider=None,
648
+ ):
649
+ """None when the change is safe, otherwise the 409 to return instead.
650
+
651
+ Fail-open on anything it cannot determine: a store with no vectors yet, a
652
+ model server that is not running, an unreadable database. Refusing on
653
+ "I could not tell" would block a legitimate first-time setup, and the
654
+ dimension a caller declares is still checked against the store either way.
655
+ """
656
+ from fastapi.responses import JSONResponse as _JSON
657
+
658
+ try:
659
+ from superlocalmemory.core.ollama_validator import (
660
+ EMBEDDING,
661
+ stored_embedding_dimension,
662
+ validate_ollama_model,
663
+ )
664
+
665
+ db_path = Path(config.base_dir) / "memory.db"
666
+ stored = stored_embedding_dimension(db_path)
667
+ if stored is None:
668
+ return None
669
+
670
+ # The provider being SAVED, falling back to the current one when the
671
+ # caller is not changing it. Reading only the current provider meant
672
+ # that SWITCHING to a local model never probed at all — and switching
673
+ # is exactly when the width changes.
674
+ effective_provider = (
675
+ new_provider
676
+ if new_provider is not None
677
+ else getattr(old_emb, "provider", "")
678
+ )
679
+ measured = None
680
+ if effective_provider == "ollama" or getattr(old_emb, "provider", "") == "ollama":
681
+ # The model being SAVED, not the one already configured. Probing the
682
+ # old one always matched the stored width and therefore always
683
+ # allowed the change — the guard measured the thing it was not
684
+ # protecting against.
685
+ probe = validate_ollama_model(
686
+ new_model or getattr(old_emb, "ollama_model", ""),
687
+ EMBEDDING,
688
+ base_url=getattr(old_emb, "ollama_base_url", "")
689
+ or "http://localhost:11434",
690
+ )
691
+ measured = probe.dimension if probe.ok else None
692
+
693
+ declared = int(new_dim or 0)
694
+ if measured is None and declared <= 0:
695
+ # Nothing to compare: the server could not be asked and the caller
696
+ # named no width. Allowing is the fail-open the first-time setup
697
+ # needs; the other route's probe still guards the common path.
698
+ return None
699
+ effective = measured if measured is not None else declared
700
+ if effective == stored:
701
+ return None
702
+
703
+ return _JSON(
704
+ {
705
+ "error": "embedding_width_mismatch",
706
+ "stored_dimension": stored,
707
+ "requested_dimension": effective,
708
+ "model_name": new_model,
709
+ "detail": (
710
+ f"{new_model} produces {effective}-dimensional vectors and "
711
+ f"this store holds {stored}-dimensional ones. Vectors of "
712
+ f"different widths cannot be compared, so every memory "
713
+ f"already stored would become unfindable by meaning. "
714
+ f"Rebuild them first with: slm db migrate — or resend with "
715
+ f"force=true if they have already been rebuilt."
716
+ ),
717
+ },
718
+ status_code=409,
719
+ )
720
+ except Exception: # noqa: BLE001 - never block a save on the check failing
721
+ logger.exception("embedding width pre-check failed; allowing the save")
722
+ return None
723
+
724
+
574
725
  @router.put("/embedding/config")
575
726
  async def set_embedding_config(request: Request):
576
727
  """Update embedding configuration independently of mode switch."""
@@ -591,13 +742,34 @@ async def set_embedding_config(request: Request):
591
742
  new_key = body.get("api_key", config.embedding.api_key)
592
743
 
593
744
  old_emb = config.embedding
745
+
746
+ # A width that disagrees with what the store already holds is refused
747
+ # here, at the moment of writing, not merely offered as a check the
748
+ # caller may or may not have run. Vectors of different widths cannot be
749
+ # compared, so the store would keep answering similarity questions and
750
+ # every answer would be noise. ``force=true`` is the escape hatch for
751
+ # somebody who has already re-embedded.
752
+ if not bool(body.get("force")):
753
+ refusal = _refuse_incompatible_embedding(
754
+ config, old_emb, new_model, new_dim, new_provider=new_provider,
755
+ )
756
+ if refusal is not None:
757
+ return refusal
758
+
594
759
  config.embedding = EmbeddingConfig(
595
760
  model_name=new_model,
596
761
  dimension=new_dim,
597
762
  provider=new_provider,
598
763
  api_endpoint=new_endpoint,
599
764
  api_key=new_key,
600
- ollama_model=old_emb.ollama_model,
765
+ # In Ollama mode the embedder resolves its model from
766
+ # ``ollama_model``, so keeping the old value here made a rename a
767
+ # no-op that still answered "success". A caller naming a model gets
768
+ # that model.
769
+ ollama_model=(
770
+ new_model if new_provider == "ollama" and new_model
771
+ else old_emb.ollama_model
772
+ ),
601
773
  ollama_base_url=old_emb.ollama_base_url,
602
774
  api_version=old_emb.api_version,
603
775
  deployment_name=old_emb.deployment_name,
@@ -1018,7 +1190,7 @@ async def recall_trace(request: Request):
1018
1190
  try:
1019
1191
  body = await request.json()
1020
1192
  query = body.get("query", "")
1021
- limit = body.get("limit", 10)
1193
+ limit = body.get("limit", CANONICAL_RECALL_LIMIT)
1022
1194
  window = body.get("window", "") or ""
1023
1195
  as_of_raw = (body.get("as_of", "") or "").strip()
1024
1196
  raw_known_as_of = body.get("known_as_of", "")
@@ -1073,6 +1245,10 @@ async def recall_trace(request: Request):
1073
1245
  window=window or None, as_of=_as_of,
1074
1246
  known_as_of=_known_as_of, valid_at=_valid_at,
1075
1247
  include_unknown=include_unknown,
1248
+ # Whoever asked, by the name they arrived under. Without a name
1249
+ # the record of this recall is discarded and no outcome
1250
+ # reported afterwards can be matched back to it.
1251
+ session_id=_signal_session_id(),
1076
1252
  ),
1077
1253
  )
1078
1254
  elapsed_ms = round((_time.monotonic() - t0) * 1000, 1)
@@ -231,14 +231,30 @@ def create_app() -> FastAPI:
231
231
  "<p><a href='/api/docs'>API Documentation</a></p>"
232
232
  "</body></html>"
233
233
  )
234
- from superlocalmemory.server.asset_versions import render_index
235
234
  from superlocalmemory import __version__ as _v
236
235
 
237
236
  # __SLM_VERSION__ was substituted only by the unified daemon, so the
238
237
  # dashboard's upgrade detector did nothing when served from here.
239
- return render_index(
240
- index_path, UI_DIR, substitutions={"__SLM_VERSION__": _v},
241
- )
238
+ # Asset versioning is cosmetic. It must never be why this page 500s.
239
+ #
240
+ # The import is deferred (house style, keeps startup lean), which means
241
+ # it resolves at REQUEST time — so when `pip install -e .` replaced the
242
+ # installed package underneath a running daemon, this route began
243
+ # answering "Internal Server Error" on the dashboard while every other
244
+ # endpoint was fine. A stale hand-written version string is a trifle; a
245
+ # blank page is not. Fall back to the file as written.
246
+ try:
247
+ from superlocalmemory.server.asset_versions import render_index
248
+
249
+ return render_index(
250
+ index_path, UI_DIR, substitutions={"__SLM_VERSION__": _v},
251
+ )
252
+ except Exception as exc: # noqa: BLE001 — serve the page regardless
253
+ logger.warning(
254
+ "asset version rewrite unavailable, serving index.html as "
255
+ "written: %s: %s", type(exc).__name__, exc,
256
+ )
257
+ return index_path.read_text().replace("__SLM_VERSION__", _v)
242
258
 
243
259
  @application.get("/favicon.ico", include_in_schema=False)
244
260
  async def favicon():
@@ -511,6 +511,49 @@ _MIGRATION_EXEMPT_PATH_PREFIXES: tuple[str, ...] = (
511
511
  )
512
512
 
513
513
 
514
+ def _serving_blocked_by(migration_result: dict) -> list[str]:
515
+ """Failed migrations that should stop this daemon serving. Fail-closed.
516
+
517
+ A failed migration used to 503 every route without asking what had failed.
518
+ For a missing table that is right. For a data invariant that ordinary use can
519
+ re-violate it is not: one drifted row made the whole store unreachable until
520
+ somebody restarted it by hand, and the restart fixed nothing that a
521
+ background repair would not have fixed on its own.
522
+
523
+ A migration may answer for itself by exposing ``blocks_serving(conn)``.
524
+ Anything that does not is treated as blocking, so this cannot quietly open a
525
+ door for a migration nobody has thought about.
526
+ """
527
+ failed = list(migration_result.get("failed") or [])
528
+ if not failed:
529
+ return []
530
+ try:
531
+ import sqlite3
532
+
533
+ from superlocalmemory.infra.data_root import state_path
534
+ from superlocalmemory.storage._migration_internals import _MODULES
535
+ except Exception: # noqa: BLE001 — never let this decide by crashing
536
+ return failed
537
+
538
+ blocking: list[str] = []
539
+ for name in failed:
540
+ decide = getattr(_MODULES.get(name), "blocks_serving", None)
541
+ if not callable(decide):
542
+ blocking.append(name)
543
+ continue
544
+ try:
545
+ db = state_path("memory.db")
546
+ conn = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
547
+ try:
548
+ if decide(conn):
549
+ blocking.append(name)
550
+ finally:
551
+ conn.close()
552
+ except Exception: # noqa: BLE001 — unknown means blocking
553
+ blocking.append(name)
554
+ return blocking
555
+
556
+
514
557
  def _is_migration_exempt_path(path: str) -> bool:
515
558
  """Return True for health, status, and repair paths that must stay reachable
516
559
  even when the daemon reports a schema migration failure.
@@ -1804,6 +1847,41 @@ def _stop_deployment_retention(application) -> bool:
1804
1847
  return True
1805
1848
 
1806
1849
 
1850
+ def _start_embedder_warmup(engine: object) -> "threading.Thread | None":
1851
+ """Load the embedding model in the background. Never blocks startup.
1852
+
1853
+ Returns the thread so a test can join it; ``None`` when there is nothing to
1854
+ warm. Every failure is "not warmed", never a failed startup: a daemon that
1855
+ cannot embed still serves keyword recall and still stores memories, and the
1856
+ materializer fills the vectors in afterwards either way.
1857
+ """
1858
+ embedder = getattr(engine, "_embedder", None)
1859
+ if embedder is None or not hasattr(embedder, "embed"):
1860
+ return None
1861
+
1862
+ from superlocalmemory.core.engine import _is_remote_embedder
1863
+
1864
+ if _is_remote_embedder(embedder):
1865
+ # A hosted embedder has no model to load and warming it would spend a
1866
+ # request, and money, on a sentence nobody asked about.
1867
+ return None
1868
+
1869
+ def _warm() -> None:
1870
+ started = time.time()
1871
+ try:
1872
+ embedder.embed("slm embedder warm-up")
1873
+ except Exception as exc: # pragma: no cover — warming is best effort
1874
+ logger.debug("embedder warm-up failed (%s) — writes will defer", exc)
1875
+ return
1876
+ logger.info(
1877
+ "Embedding model warm and ready (%.1fs)", time.time() - started,
1878
+ )
1879
+
1880
+ thread = threading.Thread(target=_warm, daemon=True, name="slm-embed-warmup")
1881
+ thread.start()
1882
+ return thread
1883
+
1884
+
1807
1885
  @asynccontextmanager
1808
1886
  async def lifespan(application: FastAPI):
1809
1887
  """Initialize engine, workers, and optional services on startup."""
@@ -2088,6 +2166,40 @@ async def lifespan(application: FastAPI):
2088
2166
  engine = MemoryEngine(config)
2089
2167
  engine.initialize()
2090
2168
 
2169
+ # Load the embedding model now, off the request path, the way the
2170
+ # cross-encoder is already warmed at startup.
2171
+ #
2172
+ # A write embeds inline so the memory it stores can be found by asking a
2173
+ # question rather than only by quoting its own words, and it gives that
2174
+ # one second before deferring to the materializer. Loading the model
2175
+ # takes 9.9-11.0 s here; once loaded an embed is 42 ms. So on a daemon
2176
+ # that had not embedded yet, the first writes each waited the full second
2177
+ # and stored no vector regardless -- and the model only ever loaded
2178
+ # because some *recall* eventually paid for it. Whoever recalled first
2179
+ # wore the cold start.
2180
+ #
2181
+ # This belongs to the daemon and not to engine wiring: the worker is a
2182
+ # subprocess per engine, so warming from wiring would have every `slm
2183
+ # status` spawn one and load a model it will never use.
2184
+ _start_embedder_warmup(engine)
2185
+
2186
+ # Tell the hook subprocesses that skill evolution is on. The hook reads
2187
+ # this env var as its fast-path signal and nothing ever set it, so the
2188
+ # feature was off for everyone who had switched it on in config: the
2189
+ # hook checked, found nothing, and returned False. Hooks are launched as
2190
+ # children of this process and inherit its environment, which is the
2191
+ # only channel between the two.
2192
+ try:
2193
+ if getattr(getattr(config, "evolution", None), "enabled", False):
2194
+ os.environ["SLM_EVOLUTION_ENABLED"] = "1"
2195
+ else:
2196
+ # Cleared as well as set: a daemon restarted with the setting
2197
+ # turned off must not leave the previous run's answer behind in
2198
+ # an environment the next hook inherits.
2199
+ os.environ.pop("SLM_EVOLUTION_ENABLED", None)
2200
+ except Exception as _evo_exc: # pragma: no cover — never block startup
2201
+ logger.debug("evolution flag not exported: %s", _evo_exc)
2202
+
2091
2203
  # Refresh migration state now that the engine is initialised. Any
2092
2204
  # schema work the engine's own bootstrap may have applied (e.g.
2093
2205
  # runtime-table creation) is captured here so the dashboard and
@@ -2175,6 +2287,30 @@ async def lifespan(application: FastAPI):
2175
2287
  "deferred migration runner crashed (non-fatal): %s", _dexc,
2176
2288
  )
2177
2289
 
2290
+ # Move an existing store onto the graph and vector backends on the
2291
+ # first start after an upgrade. They have shipped as required
2292
+ # dependencies since 3.7 and sat unused, because building the
2293
+ # projections was three manual commands almost nobody ran. Runs after
2294
+ # the deferred migrations so it projects the converted store, and never
2295
+ # fatal: if the libraries will not import or the projection does not
2296
+ # match, the daemon serves from SQLite and says so.
2297
+ try:
2298
+ from superlocalmemory.core.scale_autopromote import (
2299
+ auto_promote_scale_backends,
2300
+ )
2301
+ _promotion = auto_promote_scale_backends(config)
2302
+ application.state.scale_autopromotion = _promotion.as_dict()
2303
+ if _promotion.promoted and _promotion.restart_required:
2304
+ logger.info(
2305
+ "graph and vector backends are promoted and serve after the "
2306
+ "next restart",
2307
+ )
2308
+ except Exception as _pexc: # pragma: no cover — defensive
2309
+ logger.warning("automatic backend promotion crashed (non-fatal): %s", _pexc)
2310
+ application.state.scale_autopromotion = {
2311
+ "attempted": True, "promoted": False, "reason": str(_pexc),
2312
+ }
2313
+
2178
2314
  # S9-DASH-02: start the outcome-queue worker so recall →
2179
2315
  # pending_outcomes is actually produced. Before v3.4.22 this
2180
2316
  # producer had zero callers and the closed-loop pipeline was
@@ -3041,6 +3177,18 @@ async def lifespan(application: FastAPI):
3041
3177
  await _cancel_fact_entity_association_repair(application)
3042
3178
  await _cancel_source_quality_repair(application)
3043
3179
 
3180
+ # Stop the projection drain. Its queue is durable, so an interrupted pass
3181
+ # costs a repeat of idempotent work and nothing else — but a live worker
3182
+ # writing into RocksDB and Lance while the process tears down around it has
3183
+ # no upside.
3184
+ try:
3185
+ from superlocalmemory.core.backend_orchestrator import get_orchestrator
3186
+ _orch = get_orchestrator()
3187
+ if _orch is not None:
3188
+ _orch.stop()
3189
+ except Exception as exc: # pragma: no cover — defensive
3190
+ logger.warning("projection drain shutdown failed: %s", exc)
3191
+
3044
3192
  # Cancel the cross-platform sync loop (H-CONC-2) so adapter file I/O does
3045
3193
  # not outlive the daemon.
3046
3194
  try:
@@ -3904,7 +4052,7 @@ def _register_dashboard_routes(application: FastAPI) -> None:
3904
4052
  @application.middleware("http")
3905
4053
  async def _migration_readiness_gate(request, call_next):
3906
4054
  migration_result = getattr(application.state, "migration_result", None)
3907
- if migration_result and migration_result.get("failed"):
4055
+ if migration_result and _serving_blocked_by(migration_result):
3908
4056
  if not _is_migration_exempt_path(request.url.path):
3909
4057
  from fastapi.responses import JSONResponse
3910
4058
  return JSONResponse(
@@ -4072,12 +4220,26 @@ def _register_dashboard_routes(application: FastAPI) -> None:
4072
4220
  # 4.0.10: asset ?v= strings are now derived from file content instead of
4073
4221
  # being hand-written literals that tracked nothing. See
4074
4222
  # server/asset_versions.py — including what that does and does not fix.
4075
- from superlocalmemory.server.asset_versions import render_index
4223
+ # Asset versioning is cosmetic. It must never be why this page 500s.
4224
+ #
4225
+ # The import is deferred (house style, keeps startup lean), which means
4226
+ # it resolves at REQUEST time — so when `pip install -e .` replaced the
4227
+ # installed package underneath a running daemon, this route began
4228
+ # answering "Internal Server Error" on the dashboard while every other
4229
+ # endpoint was fine. A stale hand-written version string is a trifle; a
4230
+ # blank page is not. Fall back to the file as written.
4231
+ try:
4232
+ from superlocalmemory.server.asset_versions import render_index
4076
4233
 
4077
- return render_index(
4078
- index_path, UI_DIR,
4079
- substitutions={"__SLM_VERSION__": _SLM_VERSION},
4080
- )
4234
+ return render_index(
4235
+ index_path, UI_DIR, substitutions={"__SLM_VERSION__": _SLM_VERSION},
4236
+ )
4237
+ except Exception as exc: # noqa: BLE001 — serve the page regardless
4238
+ logger.warning(
4239
+ "asset version rewrite unavailable, serving index.html as "
4240
+ "written: %s: %s", type(exc).__name__, exc,
4241
+ )
4242
+ return index_path.read_text().replace("__SLM_VERSION__", _SLM_VERSION)
4081
4243
 
4082
4244
  @application.get("/favicon.ico", include_in_schema=False)
4083
4245
  async def favicon():
@@ -4159,7 +4321,14 @@ def _register_daemon_routes(application: FastAPI) -> None:
4159
4321
  (migration_result or {}).get("failed", []) or []
4160
4322
  )
4161
4323
  migration_details = (migration_result or {}).get("details", {}) or {}
4162
- migrations_ready = bool(migration_result) and not migration_failures
4324
+ # Ready means "can serve", so it keys off the failures that actually
4325
+ # stop this daemon serving -- not off every failure. A data invariant
4326
+ # that ordinary use re-violated leaves every route working; reporting
4327
+ # not-ready for it told operators to restart, which fixed nothing a
4328
+ # background repair would not have fixed. Everything still shows up in
4329
+ # migration_failures and migration_failure_reasons below, named.
4330
+ migration_blocking = _serving_blocked_by(migration_result or {})
4331
+ migrations_ready = bool(migration_result) and not migration_blocking
4163
4332
  if migration_details.get("_crash"):
4164
4333
  migrations_ready = False
4165
4334
  writer_runtime = getattr(
@@ -4177,6 +4346,23 @@ def _register_daemon_routes(application: FastAPI) -> None:
4177
4346
  "embedding": embedding_ready,
4178
4347
  "recall_health": recall_health.get("recall_healthy") is True,
4179
4348
  "migration_failures": migration_failures,
4349
+ # Which of those are the reason this daemon will not serve, as
4350
+ # opposed to the ones it is reporting while serving normally.
4351
+ "migration_blocking": migration_blocking,
4352
+ # WHY each one failed, not just which. The runner already produces
4353
+ # a precise sentence per migration -- "safe repair did not restore
4354
+ # M043_...", "schema verification failed ... : <sqlite error>" --
4355
+ # and this endpoint computed it and then dropped it on the floor.
4356
+ # A migration recorded ``complete`` in migration_log can still be
4357
+ # reported failed here, because a completed migration is re-checked
4358
+ # by its own verify() on every start; with only a name to go on,
4359
+ # that reads as the health check contradicting the database. It is
4360
+ # not: they are answering different questions, and this is the
4361
+ # sentence that says which. Reported as #125.
4362
+ "migration_failure_reasons": {
4363
+ name: str(migration_details.get(name, "(no detail recorded)"))
4364
+ for name in migration_failures
4365
+ },
4180
4366
  }
4181
4367
  readiness["retrieval"] = bool(
4182
4368
  readiness["embedding"] and readiness["recall_health"]
@@ -4255,6 +4441,12 @@ def _register_daemon_routes(application: FastAPI) -> None:
4255
4441
  # reveal staleness on its own. Loopback-only, alongside the other
4256
4442
  # operational metadata.
4257
4443
  "version_integrity": _version_integrity_payload(),
4444
+ # How far behind the second graph store is. A drain that stops
4445
+ # advancing is the failure that does not announce itself: every
4446
+ # other signal here stays green while the graph quietly diverges
4447
+ # from the record, and the only symptom is answers that are subtly
4448
+ # worse. A depth that does not fall is the thing to alert on.
4449
+ "projection": _projection_health(),
4258
4450
  }
4259
4451
 
4260
4452
  @application.get("/recall")
@@ -4909,6 +5101,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
4909
5101
  fact_count = 0
4910
5102
  entity_count = 0
4911
5103
  edge_count = 0
5104
+ projection_queue_depth = 0
4912
5105
  if engine is not None:
4913
5106
  try:
4914
5107
  fact_count = engine._db.get_fact_count(profile_snapshot.profile_id)
@@ -4925,6 +5118,11 @@ def _register_daemon_routes(application: FastAPI) -> None:
4925
5118
  edge_count = int(dict(edges[0])["c"]) if edges else 0
4926
5119
  except Exception:
4927
5120
  logger.debug("daemon status count query failed", exc_info=True)
5121
+ try:
5122
+ from superlocalmemory.storage import projection_outbox
5123
+ projection_queue_depth = projection_outbox.depth(engine._db)
5124
+ except Exception:
5125
+ logger.debug("projection queue depth unavailable", exc_info=True)
4928
5126
  db_path = getattr(config, "db_path", None)
4929
5127
  db_size_mb = (
4930
5128
  round(db_path.stat().st_size / 1024 / 1024, 2)
@@ -4950,6 +5148,10 @@ def _register_daemon_routes(application: FastAPI) -> None:
4950
5148
  "legacy_port": _LEGACY_PORT,
4951
5149
  "profile": profile_snapshot.profile_id,
4952
5150
  "profile_generation": profile_snapshot.generation,
5151
+ # Facts stored but not yet in the graph and vector projections. The
5152
+ # CLI and MCP read their own status from here, so this is where the
5153
+ # number has to be for all three surfaces to agree.
5154
+ "projection_queue_depth": projection_queue_depth,
4953
5155
  # F2 fix: expose M028 backfill progress so operators can monitor
4954
5156
  # the post-upgrade fact/entity association repair state.
4955
5157
  "m028_backfill": getattr(
@@ -5561,6 +5763,38 @@ def _terminalize_orphan_operation(engine, operation_id: str) -> None:
5561
5763
  )
5562
5764
 
5563
5765
 
5766
+ def _projection_health() -> dict:
5767
+ """Queue depth, stall count, and whether the worker is running.
5768
+
5769
+ Never raises: a health endpoint that fails because one of its fields could
5770
+ not be computed is worse than the missing field.
5771
+
5772
+ Takes no application. It used to accept one and never read it -- the
5773
+ orchestrator is a process singleton -- which made the signature claim a
5774
+ dependency the body did not have, and made a test that handed it a broken
5775
+ application look like it was exercising the failure path when it was only
5776
+ observing whatever the process had already built. Compare
5777
+ ``_ops_failure_counts`` directly below, which takes an application because it
5778
+ genuinely reads one.
5779
+ """
5780
+ try:
5781
+ from superlocalmemory.core.backend_orchestrator import get_orchestrator
5782
+
5783
+ orchestrator = get_orchestrator()
5784
+ if orchestrator is None:
5785
+ return {"available": False}
5786
+ health = dict(orchestrator.outbox_health())
5787
+ health["available"] = True
5788
+ # One boolean an alert can key on without knowing what a healthy depth
5789
+ # looks like on this store.
5790
+ health["behind"] = bool(health.get("depth", 0)) or bool(
5791
+ health.get("stalled", 0)
5792
+ )
5793
+ return health
5794
+ except Exception as exc: # noqa: BLE001
5795
+ return {"available": False, "error": str(exc)[:120]}
5796
+
5797
+
5564
5798
  def _ops_failure_counts(engine, application) -> dict:
5565
5799
  """Return operational failure counts for /status and /health.
5566
5800