superlocalmemory 3.8.3 → 3.8.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/CHANGELOG.md +42 -0
  2. package/README.md +3 -2
  3. package/package.json +1 -1
  4. package/plugin/.claude-plugin/plugin.json +1 -1
  5. package/plugin/CLAUDE.md +3 -3
  6. package/plugin/agents/slm-governance-advisor.md +1 -1
  7. package/plugin/agents/slm-loop-runner.md +1 -1
  8. package/plugin/agents/slm-memory-advisor.md +1 -1
  9. package/plugin/agents/slm-optimize-advisor.md +1 -1
  10. package/plugin/requirements.txt +1 -1
  11. package/plugin/skills/slm-cache/SKILL.md +1 -1
  12. package/plugin/skills/slm-compress/SKILL.md +1 -1
  13. package/plugin/skills/slm-governance/SKILL.md +1 -1
  14. package/plugin/skills/slm-graph/SKILL.md +1 -1
  15. package/plugin/skills/slm-loop/SKILL.md +1 -1
  16. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  17. package/plugin/skills/slm-profile/SKILL.md +1 -1
  18. package/plugin/skills/slm-recall/SKILL.md +1 -1
  19. package/plugin/skills/slm-remember/SKILL.md +1 -1
  20. package/plugin/skills/slm-scope/SKILL.md +1 -1
  21. package/plugin/skills/slm-session/SKILL.md +1 -1
  22. package/plugin/skills/slm-status/SKILL.md +1 -1
  23. package/plugin-src/rules/AGENTS.md +1 -1
  24. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  25. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  26. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-recall/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-remember/SKILL.md +1 -1
  29. package/plugin-src/skills/slm-session/SKILL.md +1 -1
  30. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  31. package/pyproject.toml +1 -1
  32. package/src/superlocalmemory/__init__.py +1 -1
  33. package/src/superlocalmemory/access/rbac.py +68 -76
  34. package/src/superlocalmemory/cli/commands.py +19 -0
  35. package/src/superlocalmemory/cli/ingest_cmd.py +11 -1
  36. package/src/superlocalmemory/cli/main.py +30 -0
  37. package/src/superlocalmemory/cli/pending_store.py +39 -14
  38. package/src/superlocalmemory/core/backend_orchestrator.py +93 -0
  39. package/src/superlocalmemory/core/config.py +78 -0
  40. package/src/superlocalmemory/core/consolidation_engine.py +79 -73
  41. package/src/superlocalmemory/core/engine.py +92 -11
  42. package/src/superlocalmemory/core/fact_consolidator.py +148 -30
  43. package/src/superlocalmemory/core/graph_pruner.py +436 -39
  44. package/src/superlocalmemory/core/ingestion_command.py +160 -31
  45. package/src/superlocalmemory/core/maintenance_scheduler.py +61 -1
  46. package/src/superlocalmemory/core/recall_pipeline.py +3 -0
  47. package/src/superlocalmemory/core/registry.py +5 -1
  48. package/src/superlocalmemory/core/remote_mode.py +3 -1
  49. package/src/superlocalmemory/core/scale_engine.py +41 -18
  50. package/src/superlocalmemory/core/store_pipeline.py +18 -4
  51. package/src/superlocalmemory/encoding/entity_resolver.py +18 -11
  52. package/src/superlocalmemory/hooks/_outcome_common.py +9 -2
  53. package/src/superlocalmemory/hooks/adapter_base.py +58 -44
  54. package/src/superlocalmemory/hooks/ide_connector.py +26 -8
  55. package/src/superlocalmemory/hooks/portable_kit.py +105 -9
  56. package/src/superlocalmemory/hooks/prewarm_auth.py +21 -2
  57. package/src/superlocalmemory/infra/auth_middleware.py +3 -1
  58. package/src/superlocalmemory/infra/cloud_backup.py +26 -27
  59. package/src/superlocalmemory/infra/event_bus.py +250 -88
  60. package/src/superlocalmemory/learning/consolidation_cycle.py +33 -16
  61. package/src/superlocalmemory/learning/entity_compiler.py +148 -132
  62. package/src/superlocalmemory/learning/memory_merge.py +97 -82
  63. package/src/superlocalmemory/learning/reward_archive.py +98 -90
  64. package/src/superlocalmemory/learning/reward_boost.py +40 -30
  65. package/src/superlocalmemory/mcp/http_transport.py +335 -3
  66. package/src/superlocalmemory/retrieval/engine.py +7 -1
  67. package/src/superlocalmemory/retrieval/entity_channel.py +25 -1
  68. package/src/superlocalmemory/retrieval/reranker.py +98 -15
  69. package/src/superlocalmemory/retrieval/spreading_activation.py +20 -12
  70. package/src/superlocalmemory/retrieval/vector_store.py +84 -69
  71. package/src/superlocalmemory/server/loopback.py +91 -0
  72. package/src/superlocalmemory/server/origin.py +9 -4
  73. package/src/superlocalmemory/server/routes/backup.py +6 -2
  74. package/src/superlocalmemory/server/routes/behavioral.py +6 -12
  75. package/src/superlocalmemory/server/routes/compliance.py +20 -23
  76. package/src/superlocalmemory/server/routes/config_api.py +83 -0
  77. package/src/superlocalmemory/server/routes/helpers.py +24 -13
  78. package/src/superlocalmemory/server/routes/memories.py +67 -68
  79. package/src/superlocalmemory/server/routes/mesh.py +7 -2
  80. package/src/superlocalmemory/server/routes/profiles.py +20 -21
  81. package/src/superlocalmemory/server/routes/rbac.py +0 -1
  82. package/src/superlocalmemory/server/routes/tiers.py +42 -30
  83. package/src/superlocalmemory/server/routes/v3_api.py +67 -77
  84. package/src/superlocalmemory/server/unified_daemon.py +200 -31
  85. package/src/superlocalmemory/server/write_identity.py +22 -4
  86. package/src/superlocalmemory/storage/database.py +109 -19
  87. package/src/superlocalmemory/storage/deferred_writes.py +153 -0
  88. package/src/superlocalmemory/storage/embedding_migrator.py +19 -0
  89. package/src/superlocalmemory/storage/memory_write.py +119 -0
  90. package/src/superlocalmemory/storage/migration_runner.py +7 -0
  91. package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +113 -78
  92. package/src/superlocalmemory/storage/migrations/M031_dead_letter_operations.py +80 -0
  93. package/src/superlocalmemory/storage/write_lock.py +88 -0
@@ -252,6 +252,36 @@ def main() -> None:
252
252
  "--dry-run", action="store_true", dest="dry_run", default=False,
253
253
  help="Show what would be written without making changes",
254
254
  )
255
+ # Workstream C — transport flexibility (3.8.4)
256
+ connect_p.add_argument(
257
+ "--transport",
258
+ choices=["stdio", "http", "http-mcp-remote"],
259
+ default="stdio",
260
+ dest="transport",
261
+ help=(
262
+ "MCP transport to write into the IDE config. "
263
+ "stdio (default, zero regression) | "
264
+ "http (native Streamable-HTTP, requires SLM daemon) | "
265
+ "http-mcp-remote (stdio bridge via mcp-remote for stdio-only clients)"
266
+ ),
267
+ )
268
+ connect_p.add_argument(
269
+ "--port",
270
+ type=int,
271
+ default=8765,
272
+ dest="daemon_port",
273
+ help="SLM daemon port for http/http-mcp-remote transport (default: 8765)",
274
+ )
275
+ connect_p.add_argument(
276
+ "--verify",
277
+ action="store_true",
278
+ default=False,
279
+ dest="verify",
280
+ help=(
281
+ "After writing the config, probe the daemon health endpoint to confirm "
282
+ "the transport is reachable (only meaningful for --transport http)"
283
+ ),
284
+ )
255
285
 
256
286
  migrate_p = sub.add_parser("migrate", help="Migrate data from V2 to V3 schema")
257
287
  migrate_p.add_argument(
@@ -40,6 +40,13 @@ _MAX_RETRIES = 3
40
40
  _STUCK_DAYS = 7
41
41
  _MAX_RETRY_DELAY_SECONDS = 3600
42
42
 
43
+ # Fix F: hard cap on pending.db retries. After this many failed attempts the
44
+ # item transitions to status='dead_letter' (next_retry_at=NULL) so it is
45
+ # permanently excluded from the work queue. Value chosen to survive transient
46
+ # engine unavailability (12h at 30-min drain intervals) while still providing
47
+ # a definitive signal for genuinely poisoned content.
48
+ _MAX_RETRY_COUNT: int = 20
49
+
43
50
  _SCHEMA = """
44
51
  CREATE TABLE IF NOT EXISTS pending_memories (
45
52
  id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -180,9 +187,17 @@ def mark_done(row_id: int, base_dir: Path | None = None) -> None:
180
187
  def mark_failed(row_id: int, error: str, base_dir: Path | None = None) -> None:
181
188
  """Mark a pending memory as failed with error message.
182
189
 
183
- Unprocessed evidence is never deleted or terminally hidden. Failures stay
184
- pending with bounded exponential backoff; M018 idempotency makes repeated
185
- replay safe once the canonical operation has been created.
190
+ Fix F: when the retry counter reaches _MAX_RETRY_COUNT the item transitions
191
+ to ``status='dead_letter'`` with ``next_retry_at=NULL``. Dead-lettered
192
+ items are excluded from the work queue (``get_pending`` filters
193
+ ``status='pending'``) and are not resurrected by the startup
194
+ ``UPDATE…WHERE status='failed'`` sweep (which targets only 'failed', not
195
+ 'dead_letter'). This gives operators a stable, inspectable record of
196
+ genuinely poisoned content instead of an infinite retry loop.
197
+
198
+ Below the cap, failures stay pending with bounded exponential backoff;
199
+ M018 idempotency makes repeated replay safe once the canonical operation
200
+ has been created.
186
201
  """
187
202
  conn = _get_db(base_dir)
188
203
  try:
@@ -193,17 +208,27 @@ def mark_failed(row_id: int, error: str, base_dir: Path | None = None) -> None:
193
208
  if row is None:
194
209
  return
195
210
  next_count = int(row[0] or 0) + 1
196
- delay = 0 if next_count == 1 else min(
197
- 2 ** min(next_count - 1, 12),
198
- _MAX_RETRY_DELAY_SECONDS,
199
- )
200
- conn.execute(
201
- "UPDATE pending_memories SET error = ?, "
202
- "retry_count = retry_count + 1, "
203
- "status = 'pending', next_retry_at = ? "
204
- "WHERE id = ?",
205
- (error, time.time() + delay, row_id),
206
- )
211
+ if next_count >= _MAX_RETRY_COUNT:
212
+ # Fix F: permanently exclude from work queue — dead-letter.
213
+ conn.execute(
214
+ "UPDATE pending_memories SET error = ?, "
215
+ "retry_count = retry_count + 1, "
216
+ "status = 'dead_letter', next_retry_at = NULL "
217
+ "WHERE id = ?",
218
+ (error, row_id),
219
+ )
220
+ else:
221
+ delay = 0 if next_count == 1 else min(
222
+ 2 ** min(next_count - 1, 12),
223
+ _MAX_RETRY_DELAY_SECONDS,
224
+ )
225
+ conn.execute(
226
+ "UPDATE pending_memories SET error = ?, "
227
+ "retry_count = retry_count + 1, "
228
+ "status = 'pending', next_retry_at = ? "
229
+ "WHERE id = ?",
230
+ (error, time.time() + delay, row_id),
231
+ )
207
232
  conn.commit()
208
233
  finally:
209
234
  conn.close()
@@ -100,6 +100,11 @@ class BackendOrchestrator:
100
100
  "Scale Engine remains on Local Core (state=%s)",
101
101
  getattr(self._config, "scale_engine_state", "local_core"),
102
102
  )
103
+ # v3.8.5: schedule a background check that auto-promotes to
104
+ # Cozo+LanceDB only once the DB is large enough that they beat the
105
+ # SQLite graph. A no-op (and never even starts the build) for the
106
+ # vast majority of installs, which sit far below the threshold.
107
+ self._maybe_schedule_auto_promote()
103
108
  return
104
109
 
105
110
  # 3. Initialize CozoDB if available
@@ -129,6 +134,94 @@ class BackendOrchestrator:
129
134
  "active" if self._cozo and self._cozo_status() == "active" else "off",
130
135
  "active" if self._lancedb and self._lancedb_status() == "active" else "off")
131
136
 
137
+ def _maybe_schedule_auto_promote(self) -> None:
138
+ """Schedule a delayed, one-shot scale auto-promote check (v3.8.5).
139
+
140
+ Fires well after boot warmup so it never competes for CPU / the write
141
+ lock during the startup window — the daemon keeps serving canonical
142
+ SQLite throughout. A no-op (never even starts the build) unless
143
+ auto-promotion is enabled AND the DB has grown past the threshold where
144
+ a graph DB actually beats the well-indexed SQLite graph.
145
+ """
146
+ import os
147
+ import threading
148
+
149
+ cfg = self._config
150
+ if not getattr(cfg, "scale_auto_promote_enabled", True):
151
+ return
152
+ if getattr(cfg, "scale_engine_state", "local_core") != "local_core":
153
+ return
154
+ try:
155
+ delay = float(os.environ.get("SLM_AUTO_PROMOTE_DELAY_S", "300"))
156
+ except (TypeError, ValueError):
157
+ delay = 300.0
158
+ timer = threading.Timer(delay, self._auto_promote_if_at_scale)
159
+ timer.daemon = True
160
+ timer.start()
161
+
162
+ def _count_default_edges(self) -> int:
163
+ """graph_edges count for the default profile (fail-soft → 0)."""
164
+ try:
165
+ rows = self._db.execute(
166
+ "SELECT COUNT(*) AS c FROM graph_edges WHERE profile_id = 'default'"
167
+ )
168
+ return int(rows[0]["c"]) if rows else 0
169
+ except Exception:
170
+ return 0
171
+
172
+ def _auto_promote_if_at_scale(self) -> None:
173
+ """Build + promote the Cozo/Lance projection iff the DB is at scale.
174
+
175
+ Uses the SAME staged parity gate as the manual CLI path
176
+ (prepare → verify → promote). Any failure leaves canonical SQLite
177
+ selected — the projection is derived data, never the source of truth.
178
+ The promoted backends only serve after the next daemon restart, so this
179
+ logs a clear, actionable message rather than swapping under a live
180
+ process.
181
+ """
182
+ try:
183
+ import os
184
+
185
+ cfg = self._config
186
+ if getattr(cfg, "scale_engine_state", "local_core") != "local_core":
187
+ return
188
+ threshold = int(
189
+ os.environ.get("SLM_AUTO_PROMOTE_MIN_EDGES", "")
190
+ or getattr(cfg, "scale_auto_promote_min_edges", 1_000_000)
191
+ )
192
+ edges = self._count_default_edges()
193
+ if edges < threshold:
194
+ logger.info(
195
+ "Scale auto-promote: %d edges < threshold %d — Local Core "
196
+ "(SQLite) stays optimal; no projection built.",
197
+ edges, threshold,
198
+ )
199
+ return
200
+ logger.info(
201
+ "Scale auto-promote: %d edges >= threshold %d — building "
202
+ "Cozo+LanceDB projection in the background (SQLite keeps serving).",
203
+ edges, threshold,
204
+ )
205
+ from superlocalmemory.core.scale_engine import ScaleEngineManager
206
+
207
+ mgr = ScaleEngineManager(cfg, profile_id="default")
208
+ prepared = mgr.prepare()
209
+ stage_id = prepared.get("stage_id")
210
+ mgr.verify(stage_id)
211
+ mgr.promote(stage_id)
212
+ logger.warning(
213
+ "Scale Engine AUTO-PROMOTED to Cozo+LanceDB at %d edges. RESTART "
214
+ "the daemon (`slm restart`) to activate the backends; until then "
215
+ "it keeps serving canonical SQLite.",
216
+ edges,
217
+ )
218
+ except Exception as exc:
219
+ # Derived-data failure must never take down Local Core.
220
+ logger.warning(
221
+ "Scale auto-promote skipped — staying on Local Core / SQLite: %s",
222
+ exc,
223
+ )
224
+
132
225
  def _recover_interrupted_scale_promotion(self) -> None:
133
226
  """Repair an interrupted promotion; never auto-mutate a legacy root."""
134
227
  try:
@@ -960,6 +960,29 @@ class HealthConfig:
960
960
  enable_structured_logging: bool = True
961
961
 
962
962
 
963
+ # ---------------------------------------------------------------------------
964
+ # Graph Pruning Config (Workstream G — #84)
965
+ # ---------------------------------------------------------------------------
966
+
967
+ @dataclass
968
+ class GraphPruningConfig:
969
+ """Graph thinning parameters. Exposed via dashboard + CLI.
970
+
971
+ Defaults reproduce the previous hard-coded behaviour so existing
972
+ deployments see zero behavioural change after upgrade.
973
+ """
974
+
975
+ #: Maximum in- or out-degree per node. Maps to the legacy module-level
976
+ #: ``_MAX_DEGREE_PER_NODE = 100`` in graph_pruner.py.
977
+ max_degree_per_node: int = 100
978
+ #: Discard edges with weight strictly below this floor.
979
+ #: 0.0 = no floor (current behaviour, always-safe default).
980
+ min_edge_weight: float = 0.0
981
+ #: Master kill-switch for all graph pruning. ``False`` skips the entire
982
+ #: prune step (useful for debugging graph bloat).
983
+ enabled: bool = True
984
+
985
+
963
986
  # ---------------------------------------------------------------------------
964
987
  # Master Config
965
988
  # ---------------------------------------------------------------------------
@@ -1008,8 +1031,19 @@ class SLMConfig:
1008
1031
  # Scale Engine is installed separately from activation. Existing roots
1009
1032
  # stay on SQLite until a staged parity check promotes these projections.
1010
1033
  scale_engine_state: str = "local_core" # local_core | prepared | verified | promoted
1034
+ # v3.8.5: auto-promote Cozo+LanceDB when a DB grows past the scale at which
1035
+ # they actually help. Below the threshold the well-indexed SQLite graph is
1036
+ # faster (measured ~1.7ms/traversal at 208K edges), so normal installs stay
1037
+ # on Local Core and never pay the projection/migration cost. The threshold
1038
+ # is deliberately high: the graph DB win appears at millions of edges, not
1039
+ # hundreds of thousands. Auto-promotion is background, uses the same staged
1040
+ # parity gate as the manual path, and falls back to SQLite on any failure.
1041
+ scale_auto_promote_enabled: bool = True
1042
+ scale_auto_promote_min_edges: int = 1_000_000
1011
1043
  evolution: EvolutionConfig = field(default_factory=EvolutionConfig)
1012
1044
  health: HealthConfig = field(default_factory=HealthConfig)
1045
+ # v3.8.4-G: Graph thinning parameters (#84)
1046
+ graph_pruning: GraphPruningConfig = field(default_factory=GraphPruningConfig)
1013
1047
 
1014
1048
  # v3.4.3: Daemon configuration
1015
1049
  daemon_idle_timeout: int = 0 # 0 = 24/7 (no auto-kill). >0 = seconds before auto-kill.
@@ -1122,6 +1156,15 @@ class SLMConfig:
1122
1156
  state if state in {"local_core", "prepared", "verified", "promoted"}
1123
1157
  else "local_core"
1124
1158
  )
1159
+ config.scale_auto_promote_enabled = bool(
1160
+ data.get("scale_auto_promote_enabled", True)
1161
+ )
1162
+ try:
1163
+ config.scale_auto_promote_min_edges = int(
1164
+ data.get("scale_auto_promote_min_edges", 1_000_000)
1165
+ )
1166
+ except (TypeError, ValueError):
1167
+ config.scale_auto_promote_min_edges = 1_000_000
1125
1168
 
1126
1169
  # V3.3 config fields (additive — defaults work if missing from JSON)
1127
1170
  fg = data.get("forgetting", {})
@@ -1174,6 +1217,31 @@ class SLMConfig:
1174
1217
  if k in HealthConfig.__dataclass_fields__
1175
1218
  })
1176
1219
 
1220
+ # v3.8.4-G: Graph pruning config (#84) — additive, no migration needed.
1221
+ # Old configs without this section silently fall back to GraphPruningConfig()
1222
+ # defaults, which reproduce the existing _MAX_DEGREE_PER_NODE=100 behaviour.
1223
+ gp_raw = data.get("graph_pruning", {})
1224
+ if gp_raw:
1225
+ try:
1226
+ gp_kwargs = {
1227
+ k: v for k, v in gp_raw.items()
1228
+ if k in GraphPruningConfig.__dataclass_fields__
1229
+ }
1230
+ gp = GraphPruningConfig(**gp_kwargs)
1231
+ # Validate and clamp out-of-range values
1232
+ max_deg = gp.max_degree_per_node if gp.max_degree_per_node >= 1 else 100
1233
+ min_w = max(0.0, min(1.0, gp.min_edge_weight))
1234
+ config.graph_pruning = GraphPruningConfig(
1235
+ max_degree_per_node=max_deg,
1236
+ min_edge_weight=min_w,
1237
+ enabled=gp.enabled,
1238
+ )
1239
+ except (ValueError, TypeError) as exc:
1240
+ logger.warning(
1241
+ "graph_pruning config invalid (%s) — using defaults", exc
1242
+ )
1243
+ config.graph_pruning = GraphPruningConfig()
1244
+
1177
1245
  # V3.4.65: Injection config (additive — defaults if missing from JSON)
1178
1246
  inj = data.get("injection", {}) or {}
1179
1247
  config.injection = InjectionConfig(
@@ -1271,6 +1339,8 @@ class SLMConfig:
1271
1339
  "graph_backend": self.graph_backend,
1272
1340
  "vector_backend": self.vector_backend,
1273
1341
  "scale_engine_state": self.scale_engine_state,
1342
+ "scale_auto_promote_enabled": self.scale_auto_promote_enabled,
1343
+ "scale_auto_promote_min_edges": self.scale_auto_promote_min_edges,
1274
1344
  "base_dir": str(self.base_dir), # V3.5.9: persist so load() can restore custom paths
1275
1345
  "llm": {
1276
1346
  "provider": self.llm.provider,
@@ -1329,6 +1399,14 @@ class SLMConfig:
1329
1399
  # Multi-scope memory: behaviour defaults
1330
1400
  data["scope"] = self.scope.as_dict()
1331
1401
 
1402
+ # v3.8.4-G: Persist graph pruning config (#84).
1403
+ # Always written — additive key, never overwrites unrelated sections.
1404
+ data["graph_pruning"] = {
1405
+ "max_degree_per_node": self.graph_pruning.max_degree_per_node,
1406
+ "min_edge_weight": self.graph_pruning.min_edge_weight,
1407
+ "enabled": self.graph_pruning.enabled,
1408
+ }
1409
+
1332
1410
  # Preserve existing V3.3 config sections that aren't in for_mode()
1333
1411
  for key in ("forgetting", "quantization", "sagq", "embedding_signature", "auto_invoke"):
1334
1412
  if key in existing:
@@ -708,93 +708,99 @@ class ConsolidationEngine:
708
708
  Uses 'custom' category because the soft_prompt_templates CHECK
709
709
  constraint does not include 'skill_evolution'. The content is
710
710
  prefixed with [SKILL_EVOLUTION] for easy filtering.
711
- """
712
- import sqlite3 as _sqlite3
713
711
 
714
- db_path = str(self._db.db_path)
712
+ Concurrency fix (v3.8.4): all writes routed through memory_write()
713
+ so the process write lock (get_write_lock) serialises in-process
714
+ writers and proper busy_timeout handles cross-process races.
715
+ No slow ops are held inside the write lock — this method has none.
716
+ """
717
+ from superlocalmemory.storage.memory_write import memory_read, memory_write
715
718
 
716
- conn = _sqlite3.connect(db_path, timeout=10)
717
- conn.row_factory = _sqlite3.Row
719
+ db_path = self._db.db_path
718
720
 
719
- # Fetch promoted evolutions
721
+ # Phase 1: read-only fetch — does NOT hold the write lock.
720
722
  try:
721
- promoted_rows = conn.execute(
722
- "SELECT id, skill_name, parent_skill_id, evolution_type, "
723
- "mutation_summary, created_at "
724
- "FROM skill_evolution_log "
725
- "WHERE status = 'promoted' "
726
- "ORDER BY created_at DESC LIMIT 20",
727
- ).fetchall()
728
- except _sqlite3.OperationalError:
723
+ with memory_read(db_path) as rconn:
724
+ promoted_rows = [
725
+ dict(r)
726
+ for r in rconn.execute(
727
+ "SELECT id, skill_name, parent_skill_id, evolution_type, "
728
+ "mutation_summary, created_at "
729
+ "FROM skill_evolution_log "
730
+ "WHERE status = 'promoted' "
731
+ "ORDER BY created_at DESC LIMIT 20",
732
+ ).fetchall()
733
+ ]
734
+ except Exception:
729
735
  # Table may not exist yet
730
- conn.close()
731
736
  return {"created": 0, "message": "skill_evolution_log table not found"}
732
737
 
733
- created_count = 0
738
+ if not promoted_rows:
739
+ return {"promoted_skills_found": 0, "soft_prompts_created": 0}
740
+
734
741
  now = datetime.now(timezone.utc).isoformat()
742
+ created_count = 0
735
743
 
736
- for row in promoted_rows:
737
- r = dict(row)
738
- skill_name = r["skill_name"]
739
- parent = r.get("parent_skill_id") or skill_name
740
- evo_type = r["evolution_type"]
741
- summary = r.get("mutation_summary", "")
742
- evo_id = r["id"]
743
-
744
- # Build prompt content
745
- content = (
746
- f"[SKILL_EVOLUTION] Evolved skill: '{skill_name}' "
747
- f"({'replaces' if evo_type == 'fix' else 'extends'} '{parent}' "
748
- f"via {evo_type}). {summary}. "
749
- f"Use the evolved version for better results."
750
- )
744
+ # Phase 2: short write transaction — hold the write lock for the
745
+ # INSERT/UPDATE loop only (pure SQL, no network / embed calls).
746
+ with memory_write(db_path) as conn:
747
+ for r in promoted_rows:
748
+ skill_name = r["skill_name"]
749
+ parent = r.get("parent_skill_id") or skill_name
750
+ evo_type = r["evolution_type"]
751
+ summary_txt = r.get("mutation_summary", "")
752
+ evo_id = r["id"]
753
+
754
+ content = (
755
+ f"[SKILL_EVOLUTION] Evolved skill: '{skill_name}' "
756
+ f"({'replaces' if evo_type == 'fix' else 'extends'} '{parent}' "
757
+ f"via {evo_type}). {summary_txt}. "
758
+ f"Use the evolved version for better results."
759
+ )
760
+ prompt_id = f"evo-{evo_id}"
751
761
 
752
- # Use a deterministic prompt_id based on the evolution record
753
- prompt_id = f"evo-{evo_id}"
762
+ existing = conn.execute(
763
+ "SELECT prompt_id FROM soft_prompt_templates WHERE prompt_id = ?",
764
+ (prompt_id,),
765
+ ).fetchone()
754
766
 
755
- # Check if prompt already exists
756
- existing = conn.execute(
757
- "SELECT prompt_id FROM soft_prompt_templates WHERE prompt_id = ?",
758
- (prompt_id,),
759
- ).fetchone()
767
+ if existing:
768
+ # M-REPLACE: Update existing record instead of INSERT OR REPLACE
769
+ # to avoid silently dropping columns with defaults.
770
+ try:
771
+ conn.execute(
772
+ "UPDATE soft_prompt_templates "
773
+ "SET content = ?, updated_at = ? "
774
+ "WHERE prompt_id = ?",
775
+ (content, now, prompt_id),
776
+ )
777
+ except Exception as upd_exc:
778
+ logger.debug(
779
+ "Failed to update soft prompt %s: %s", prompt_id, upd_exc
780
+ )
781
+ continue
760
782
 
761
- if existing:
762
- # M-REPLACE: Update existing record instead of INSERT OR REPLACE
763
- # to avoid silently dropping columns with defaults
764
783
  try:
765
- conn.execute(
766
- "UPDATE soft_prompt_templates "
767
- "SET content = ?, updated_at = ? "
768
- "WHERE prompt_id = ?",
769
- (content, now, prompt_id),
784
+ cur = conn.execute(
785
+ "INSERT OR IGNORE INTO soft_prompt_templates "
786
+ "(prompt_id, profile_id, category, content, source_pattern_ids, "
787
+ " confidence, effectiveness, token_count, retention_score, "
788
+ " active, version, created_at, updated_at) "
789
+ "VALUES (?, ?, 'custom', ?, ?, 0.8, 0.5, ?, 1.0, 1, 1, ?, ?)",
790
+ (
791
+ prompt_id, profile_id, content,
792
+ json.dumps([evo_id]),
793
+ len(content.split()),
794
+ now, now,
795
+ ),
796
+ )
797
+ if cur.rowcount > 0:
798
+ created_count += 1
799
+ except Exception as ins_exc:
800
+ # Unique constraint on (profile_id, category) WHERE active=1
801
+ logger.debug(
802
+ "Skipping soft prompt for %s: %s", skill_name, ins_exc
770
803
  )
771
- except _sqlite3.OperationalError as upd_exc:
772
- logger.debug("Failed to update soft prompt %s: %s", prompt_id, upd_exc)
773
- continue
774
-
775
- try:
776
- conn.execute(
777
- "INSERT OR IGNORE INTO soft_prompt_templates "
778
- "(prompt_id, profile_id, category, content, source_pattern_ids, "
779
- " confidence, effectiveness, token_count, retention_score, "
780
- " active, version, created_at, updated_at) "
781
- "VALUES (?, ?, 'custom', ?, ?, 0.8, 0.5, ?, 1.0, 1, 1, ?, ?)",
782
- (prompt_id, profile_id, content,
783
- json.dumps([evo_id]),
784
- len(content.split()), # Rough token estimate
785
- now, now),
786
- )
787
- if conn.total_changes:
788
- created_count += 1
789
- except _sqlite3.IntegrityError:
790
- # Unique constraint on (profile_id, category) WHERE active=1
791
- logger.debug(
792
- "Skipping soft prompt for %s: unique constraint on active custom",
793
- skill_name,
794
- )
795
-
796
- conn.commit()
797
- conn.close()
798
804
 
799
805
  return {
800
806
  "promoted_skills_found": len(promoted_rows),
@@ -18,6 +18,8 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
18
18
  from __future__ import annotations
19
19
 
20
20
  import logging
21
+ import os
22
+ import threading
21
23
  from pathlib import Path
22
24
  from typing import Any
23
25
 
@@ -26,7 +28,7 @@ from superlocalmemory.core.engine_capabilities import Capabilities, CapabilityEr
26
28
  from superlocalmemory.core.modes import get_capabilities
27
29
  from superlocalmemory.learning.outcome_queue import RecallEvent, enqueue_recall
28
30
  from superlocalmemory.storage.models import (
29
- AtomicFact, MemoryRecord, Mode, RecallResponse,
31
+ AtomicFact, FactType, MemoryRecord, Mode, RecallResponse,
30
32
  )
31
33
 
32
34
  logger = logging.getLogger(__name__)
@@ -47,6 +49,34 @@ def _verify_ingestion_schema(memory_db: Path) -> bool:
47
49
  connection.close()
48
50
 
49
51
 
52
+ # ---------------------------------------------------------------------------
53
+ # Workstream D (3.8.4) — warm-guard sync embed helpers
54
+ # ---------------------------------------------------------------------------
55
+
56
+ def _is_remote_embedder(embedder: object) -> bool:
57
+ """Return True if *embedder* makes remote HTTP calls (cloud / OpenAI-compatible).
58
+
59
+ Remote embedders (100–400ms round-trip) must never block the store_fast()
60
+ write path synchronously — they stay on the async materializer path.
61
+
62
+ Local embedders:
63
+ - EmbeddingService (subprocess, local ONNX/sentence-transformers): has
64
+ ``_config`` with ``is_cloud=False`` and ``is_openai_compatible=False``.
65
+ - OllamaEmbedder (localhost HTTP, ~73ms): has NO ``_config`` attribute.
66
+
67
+ Remote embedders:
68
+ - EmbeddingService with ``_config.is_cloud=True`` (Azure, etc.)
69
+ - EmbeddingService with ``_config.is_openai_compatible=True``
70
+ """
71
+ cfg = getattr(embedder, "_config", None)
72
+ if cfg is None:
73
+ return False # OllamaEmbedder — local
74
+ return bool(
75
+ getattr(cfg, "is_cloud", False)
76
+ or getattr(cfg, "is_openai_compatible", False)
77
+ )
78
+
79
+
50
80
  class MemoryEngine:
51
81
  """Main orchestrator for the SuperLocalMemory V3 memory system.
52
82
 
@@ -103,6 +133,11 @@ class MemoryEngine:
103
133
  self._consolidation_engine = None
104
134
  self._maintenance_scheduler = None
105
135
  self._hooks = HookRegistry()
136
+ # Workstream D (3.8.4): single-worker pool reused across store_fast() calls.
137
+ # Lazy-created on first warm-guard attempt; avoids per-call thread churn.
138
+ self._store_fast_embed_pool: object | None = None
139
+ # Lock guards the lazy-init to prevent TOCTOU race on concurrent first calls.
140
+ self._store_fast_embed_pool_lock = threading.Lock()
106
141
 
107
142
  # -- Public properties (Phase 2+ access) --------------------------------
108
143
 
@@ -525,9 +560,6 @@ class MemoryEngine:
525
560
  import re as _re
526
561
  import uuid as _uuid
527
562
  from datetime import datetime, timezone
528
- from superlocalmemory.storage.models import (
529
- AtomicFact, FactType, MemoryRecord,
530
- )
531
563
  from superlocalmemory.core.engine_ingestion import content_passes_admission
532
564
  if not content_passes_admission(content):
533
565
  return []
@@ -568,15 +600,64 @@ class MemoryEngine:
568
600
  r"\b([A-Z][a-z]+(?:\s[A-Z][a-z]+){0,3})\b", fact_text)}
569
601
  | {m.group(1) for m in _re.finditer(r"\b([A-Z]{2,})\b", fact_text)}
570
602
  )
571
- # Queryable admission must never acquire the embedding worker lock.
572
- # On a warm daemon that looked cheap, but on a clean Mode A install the
573
- # background model load owns that lock for up to 180s and turned the
574
- # receipt-first path into a hidden synchronous wait. The canonical
575
- # materializer below runs the complete pipeline and promotes this same
576
- # fact with its embedding, Fisher parameters, entities and graph edges.
577
- # Until then it is deliberately BM25/entity/date recallable.
603
+ # Workstream D (3.8.4) warm-guard synchronous embed.
604
+ #
605
+ # Original contract (3.8.2): queryable admission NEVER acquires the
606
+ # embedding worker lock. On a clean Mode-A install the background model
607
+ # load owns that lock for up to 180s, turning the receipt-first path into
608
+ # a hidden synchronous wait. The canonical materializer promotes this
609
+ # fact with its full pipeline (embedding, Fisher, entities, graph edges).
610
+ #
611
+ # 3.8.4 extension: when the embedder is PROVABLY warm (_available is True)
612
+ # AND is a local embedder (not a remote cloud/OpenAI endpoint), compute the
613
+ # embedding synchronously with a hard 500ms cap. On timeout or any
614
+ # exception, fall through to emb=None — the materializer fills it async.
615
+ # This preserves the 3.8.2 invariant for cold start while eliminating the
616
+ # semantic-channel blind spot on warm daemons (the top UX complaint).
578
617
  emb = None
579
618
  fmean = fvar = None
619
+ _embedder_ref = self._embedder
620
+ if (
621
+ _embedder_ref is not None
622
+ and getattr(_embedder_ref, "_available", None) is True
623
+ and not _is_remote_embedder(_embedder_ref)
624
+ ):
625
+ import concurrent.futures as _cf
626
+ # Lazy-init the pool once per engine instance — avoids per-call
627
+ # thread churn and the associated resource leak from discard-on-exit.
628
+ # Double-checked locking guards against TOCTOU on concurrent first calls.
629
+ if self._store_fast_embed_pool is None:
630
+ with self._store_fast_embed_pool_lock:
631
+ if self._store_fast_embed_pool is None:
632
+ self._store_fast_embed_pool = _cf.ThreadPoolExecutor(
633
+ max_workers=1,
634
+ thread_name_prefix="slm-sg-embed",
635
+ )
636
+ try:
637
+ _timeout_s = int(os.environ.get("SLM_STORE_FAST_EMBED_TIMEOUT_MS", 500)) / 1000.0
638
+ except (ValueError, TypeError):
639
+ _timeout_s = 0.5 # default 500 ms
640
+ try:
641
+ _future = self._store_fast_embed_pool.submit(_embedder_ref.embed, fact_text)
642
+ try:
643
+ emb = _future.result(timeout=_timeout_s)
644
+ if emb:
645
+ fmean, fvar = _embedder_ref.compute_fisher_params(emb)
646
+ except _cf.TimeoutError:
647
+ logger.debug(
648
+ "store_fast: warm-guard embed timed out (>%.0fms) — deferring to materializer",
649
+ _timeout_s * 1000,
650
+ )
651
+ emb = None
652
+ except Exception as _exc:
653
+ logger.debug(
654
+ "store_fast: warm-guard embed failed (%s) — deferring to materializer",
655
+ _exc,
656
+ )
657
+ emb = None
658
+ except Exception as _exc:
659
+ logger.debug("store_fast: warm-guard pool submit failed (%s)", _exc)
660
+ emb = None
580
661
  fact = AtomicFact(
581
662
  fact_id=_uuid.uuid4().hex[:16], memory_id=record.memory_id,
582
663
  profile_id=self._profile_id, content=fact_text,