superlocalmemory 3.8.3 → 3.8.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/CHANGELOG.md +76 -0
  2. package/README.md +3 -2
  3. package/package.json +1 -1
  4. package/plugin/.claude-plugin/plugin.json +1 -1
  5. package/plugin/CLAUDE.md +3 -3
  6. package/plugin/agents/slm-governance-advisor.md +1 -1
  7. package/plugin/agents/slm-loop-runner.md +1 -1
  8. package/plugin/agents/slm-memory-advisor.md +1 -1
  9. package/plugin/agents/slm-optimize-advisor.md +1 -1
  10. package/plugin/requirements.txt +1 -1
  11. package/plugin/skills/slm-cache/SKILL.md +1 -1
  12. package/plugin/skills/slm-compress/SKILL.md +1 -1
  13. package/plugin/skills/slm-governance/SKILL.md +1 -1
  14. package/plugin/skills/slm-graph/SKILL.md +1 -1
  15. package/plugin/skills/slm-loop/SKILL.md +1 -1
  16. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  17. package/plugin/skills/slm-profile/SKILL.md +1 -1
  18. package/plugin/skills/slm-recall/SKILL.md +1 -1
  19. package/plugin/skills/slm-remember/SKILL.md +1 -1
  20. package/plugin/skills/slm-scope/SKILL.md +1 -1
  21. package/plugin/skills/slm-session/SKILL.md +1 -1
  22. package/plugin/skills/slm-status/SKILL.md +1 -1
  23. package/plugin-src/rules/AGENTS.md +1 -1
  24. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  25. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  26. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-recall/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-remember/SKILL.md +1 -1
  29. package/plugin-src/skills/slm-session/SKILL.md +1 -1
  30. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  31. package/pyproject.toml +9 -4
  32. package/src/superlocalmemory/__init__.py +1 -1
  33. package/src/superlocalmemory/access/rbac.py +68 -76
  34. package/src/superlocalmemory/cli/commands.py +158 -404
  35. package/src/superlocalmemory/cli/ingest_cmd.py +11 -1
  36. package/src/superlocalmemory/cli/main.py +30 -0
  37. package/src/superlocalmemory/cli/pending_store.py +39 -14
  38. package/src/superlocalmemory/core/backend_orchestrator.py +93 -0
  39. package/src/superlocalmemory/core/component_registry.py +4 -2
  40. package/src/superlocalmemory/core/config.py +78 -0
  41. package/src/superlocalmemory/core/consolidation_engine.py +79 -73
  42. package/src/superlocalmemory/core/embeddings.py +33 -6
  43. package/src/superlocalmemory/core/engine.py +186 -60
  44. package/src/superlocalmemory/core/engine_ingestion.py +150 -63
  45. package/src/superlocalmemory/core/fact_consolidator.py +148 -30
  46. package/src/superlocalmemory/core/graph_pruner.py +436 -39
  47. package/src/superlocalmemory/core/ingestion_command.py +273 -32
  48. package/src/superlocalmemory/core/maintenance_scheduler.py +61 -1
  49. package/src/superlocalmemory/core/mutations.py +32 -10
  50. package/src/superlocalmemory/core/recall_pipeline.py +111 -74
  51. package/src/superlocalmemory/core/registry.py +5 -1
  52. package/src/superlocalmemory/core/remember_admission.py +152 -0
  53. package/src/superlocalmemory/core/remember_runtime.py +712 -0
  54. package/src/superlocalmemory/core/remote_mode.py +3 -1
  55. package/src/superlocalmemory/core/scale_engine.py +41 -18
  56. package/src/superlocalmemory/core/store_pipeline.py +18 -4
  57. package/src/superlocalmemory/encoding/entity_resolver.py +18 -11
  58. package/src/superlocalmemory/graph/cozo_backend.py +5 -5
  59. package/src/superlocalmemory/hooks/_outcome_common.py +9 -2
  60. package/src/superlocalmemory/hooks/adapter_base.py +58 -44
  61. package/src/superlocalmemory/hooks/ide_connector.py +26 -8
  62. package/src/superlocalmemory/hooks/portable_kit.py +105 -9
  63. package/src/superlocalmemory/hooks/prewarm_auth.py +21 -2
  64. package/src/superlocalmemory/infra/auth_middleware.py +3 -1
  65. package/src/superlocalmemory/infra/cloud_backup.py +26 -27
  66. package/src/superlocalmemory/infra/event_bus.py +250 -88
  67. package/src/superlocalmemory/learning/bandit.py +50 -1
  68. package/src/superlocalmemory/learning/consolidation_cycle.py +33 -16
  69. package/src/superlocalmemory/learning/entity_compiler.py +148 -132
  70. package/src/superlocalmemory/learning/memory_merge.py +97 -82
  71. package/src/superlocalmemory/learning/reward_archive.py +98 -90
  72. package/src/superlocalmemory/learning/reward_boost.py +40 -30
  73. package/src/superlocalmemory/learning/source_quality.py +38 -35
  74. package/src/superlocalmemory/mcp/_daemon_proxy.py +38 -15
  75. package/src/superlocalmemory/mcp/http_transport.py +335 -3
  76. package/src/superlocalmemory/mcp/tools_active.py +4 -41
  77. package/src/superlocalmemory/mcp/tools_core.py +26 -87
  78. package/src/superlocalmemory/mcp/tools_evolution.py +5 -10
  79. package/src/superlocalmemory/optimize/proxy/capture.py +196 -8
  80. package/src/superlocalmemory/retrieval/engine.py +15 -4
  81. package/src/superlocalmemory/retrieval/entity_channel.py +25 -1
  82. package/src/superlocalmemory/retrieval/reranker.py +130 -22
  83. package/src/superlocalmemory/retrieval/spreading_activation.py +20 -12
  84. package/src/superlocalmemory/retrieval/vector_store.py +84 -69
  85. package/src/superlocalmemory/server/loopback.py +85 -0
  86. package/src/superlocalmemory/server/origin.py +9 -4
  87. package/src/superlocalmemory/server/profile_runtime.py +14 -0
  88. package/src/superlocalmemory/server/routes/abstraction.py +2 -4
  89. package/src/superlocalmemory/server/routes/agents.py +3 -5
  90. package/src/superlocalmemory/server/routes/backup.py +6 -2
  91. package/src/superlocalmemory/server/routes/behavioral.py +11 -25
  92. package/src/superlocalmemory/server/routes/brain.py +6 -9
  93. package/src/superlocalmemory/server/routes/compliance.py +20 -23
  94. package/src/superlocalmemory/server/routes/config_api.py +83 -0
  95. package/src/superlocalmemory/server/routes/entity.py +3 -7
  96. package/src/superlocalmemory/server/routes/evolution.py +3 -5
  97. package/src/superlocalmemory/server/routes/helpers.py +57 -25
  98. package/src/superlocalmemory/server/routes/insights.py +2 -4
  99. package/src/superlocalmemory/server/routes/learning.py +2 -5
  100. package/src/superlocalmemory/server/routes/lifecycle.py +2 -4
  101. package/src/superlocalmemory/server/routes/memories.py +119 -98
  102. package/src/superlocalmemory/server/routes/mesh.py +7 -2
  103. package/src/superlocalmemory/server/routes/profiles.py +20 -21
  104. package/src/superlocalmemory/server/routes/rbac.py +0 -1
  105. package/src/superlocalmemory/server/routes/tiers.py +28 -35
  106. package/src/superlocalmemory/server/routes/timeline.py +2 -4
  107. package/src/superlocalmemory/server/routes/v3_api.py +85 -93
  108. package/src/superlocalmemory/server/unified_daemon.py +400 -140
  109. package/src/superlocalmemory/server/write_identity.py +22 -4
  110. package/src/superlocalmemory/storage/admission_codec.py +119 -0
  111. package/src/superlocalmemory/storage/admission_journal.py +728 -0
  112. package/src/superlocalmemory/storage/database.py +168 -19
  113. package/src/superlocalmemory/storage/deferred_writes.py +209 -0
  114. package/src/superlocalmemory/storage/embedding_migrator.py +19 -0
  115. package/src/superlocalmemory/storage/memory_write.py +115 -0
  116. package/src/superlocalmemory/storage/migration_runner.py +44 -0
  117. package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +113 -78
  118. package/src/superlocalmemory/storage/migrations/M031_dead_letter_operations.py +80 -0
  119. package/src/superlocalmemory/storage/migrations/M032_write_coordinator_admission.py +188 -0
  120. package/src/superlocalmemory/storage/read_connection.py +115 -0
  121. package/src/superlocalmemory/storage/write_coordinator.py +756 -0
  122. package/src/superlocalmemory/storage/write_lock.py +88 -0
  123. package/src/superlocalmemory/ui/index.html +1 -1
  124. package/src/superlocalmemory/ui/js/auto-settings.js +14 -1
  125. package/src/superlocalmemory/ui/js/od-settings.js +9 -3
@@ -281,15 +281,25 @@ def _write_tool_events(events: list[dict]) -> int:
281
281
 
282
282
  v3.4.10: Preserves input_summary, output_summary, and project_path
283
283
  from enriched sources (ECC observations, enriched hook).
284
+
285
+ Concurrency note
286
+ ----------------
287
+ ``slm ingest`` is a SEPARATE OS process — the daemon's threading.RLock
288
+ write-lock cannot help here. PRAGMA busy_timeout is the only cross-process
289
+ lever: it makes SQLite WAIT for the single writer (daemon / hook) to finish
290
+ instead of immediately raising SQLITE_BUSY. The ``timeout`` kwarg to
291
+ sqlite3.connect is the Python-level retry budget (same duration).
284
292
  """
285
293
  db_path = Path(MEMORY_DB)
286
294
  if not db_path.exists():
287
295
  return 0
288
296
 
289
- conn = sqlite3.connect(str(db_path), timeout=10)
297
+ _busy_ms = 10_000
298
+ conn = sqlite3.connect(str(db_path), timeout=_busy_ms / 1000.0)
290
299
  count = 0
291
300
 
292
301
  try:
302
+ conn.execute(f"PRAGMA busy_timeout={_busy_ms}")
293
303
  for ev in events:
294
304
  try:
295
305
  conn.execute(
@@ -252,6 +252,36 @@ def main() -> None:
252
252
  "--dry-run", action="store_true", dest="dry_run", default=False,
253
253
  help="Show what would be written without making changes",
254
254
  )
255
+ # Workstream C — transport flexibility (3.8.4)
256
+ connect_p.add_argument(
257
+ "--transport",
258
+ choices=["stdio", "http", "http-mcp-remote"],
259
+ default="stdio",
260
+ dest="transport",
261
+ help=(
262
+ "MCP transport to write into the IDE config. "
263
+ "stdio (default, zero regression) | "
264
+ "http (native Streamable-HTTP, requires SLM daemon) | "
265
+ "http-mcp-remote (stdio bridge via mcp-remote for stdio-only clients)"
266
+ ),
267
+ )
268
+ connect_p.add_argument(
269
+ "--port",
270
+ type=int,
271
+ default=8765,
272
+ dest="daemon_port",
273
+ help="SLM daemon port for http/http-mcp-remote transport (default: 8765)",
274
+ )
275
+ connect_p.add_argument(
276
+ "--verify",
277
+ action="store_true",
278
+ default=False,
279
+ dest="verify",
280
+ help=(
281
+ "After writing the config, probe the daemon health endpoint to confirm "
282
+ "the transport is reachable (only meaningful for --transport http)"
283
+ ),
284
+ )
255
285
 
256
286
  migrate_p = sub.add_parser("migrate", help="Migrate data from V2 to V3 schema")
257
287
  migrate_p.add_argument(
@@ -40,6 +40,13 @@ _MAX_RETRIES = 3
40
40
  _STUCK_DAYS = 7
41
41
  _MAX_RETRY_DELAY_SECONDS = 3600
42
42
 
43
+ # Fix F: hard cap on pending.db retries. After this many failed attempts the
44
+ # item transitions to status='dead_letter' (next_retry_at=NULL) so it is
45
+ # permanently excluded from the work queue. Value chosen to survive transient
46
+ # engine unavailability (12h at 30-min drain intervals) while still providing
47
+ # a definitive signal for genuinely poisoned content.
48
+ _MAX_RETRY_COUNT: int = 20
49
+
43
50
  _SCHEMA = """
44
51
  CREATE TABLE IF NOT EXISTS pending_memories (
45
52
  id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -180,9 +187,17 @@ def mark_done(row_id: int, base_dir: Path | None = None) -> None:
180
187
  def mark_failed(row_id: int, error: str, base_dir: Path | None = None) -> None:
181
188
  """Mark a pending memory as failed with error message.
182
189
 
183
- Unprocessed evidence is never deleted or terminally hidden. Failures stay
184
- pending with bounded exponential backoff; M018 idempotency makes repeated
185
- replay safe once the canonical operation has been created.
190
+ Fix F: when the retry counter reaches _MAX_RETRY_COUNT the item transitions
191
+ to ``status='dead_letter'`` with ``next_retry_at=NULL``. Dead-lettered
192
+ items are excluded from the work queue (``get_pending`` filters
193
+ ``status='pending'``) and are not resurrected by the startup
194
+ ``UPDATE…WHERE status='failed'`` sweep (which targets only 'failed', not
195
+ 'dead_letter'). This gives operators a stable, inspectable record of
196
+ genuinely poisoned content instead of an infinite retry loop.
197
+
198
+ Below the cap, failures stay pending with bounded exponential backoff;
199
+ M018 idempotency makes repeated replay safe once the canonical operation
200
+ has been created.
186
201
  """
187
202
  conn = _get_db(base_dir)
188
203
  try:
@@ -193,17 +208,27 @@ def mark_failed(row_id: int, error: str, base_dir: Path | None = None) -> None:
193
208
  if row is None:
194
209
  return
195
210
  next_count = int(row[0] or 0) + 1
196
- delay = 0 if next_count == 1 else min(
197
- 2 ** min(next_count - 1, 12),
198
- _MAX_RETRY_DELAY_SECONDS,
199
- )
200
- conn.execute(
201
- "UPDATE pending_memories SET error = ?, "
202
- "retry_count = retry_count + 1, "
203
- "status = 'pending', next_retry_at = ? "
204
- "WHERE id = ?",
205
- (error, time.time() + delay, row_id),
206
- )
211
+ if next_count >= _MAX_RETRY_COUNT:
212
+ # Fix F: permanently exclude from work queue — dead-letter.
213
+ conn.execute(
214
+ "UPDATE pending_memories SET error = ?, "
215
+ "retry_count = retry_count + 1, "
216
+ "status = 'dead_letter', next_retry_at = NULL "
217
+ "WHERE id = ?",
218
+ (error, row_id),
219
+ )
220
+ else:
221
+ delay = 0 if next_count == 1 else min(
222
+ 2 ** min(next_count - 1, 12),
223
+ _MAX_RETRY_DELAY_SECONDS,
224
+ )
225
+ conn.execute(
226
+ "UPDATE pending_memories SET error = ?, "
227
+ "retry_count = retry_count + 1, "
228
+ "status = 'pending', next_retry_at = ? "
229
+ "WHERE id = ?",
230
+ (error, time.time() + delay, row_id),
231
+ )
207
232
  conn.commit()
208
233
  finally:
209
234
  conn.close()
@@ -100,6 +100,11 @@ class BackendOrchestrator:
100
100
  "Scale Engine remains on Local Core (state=%s)",
101
101
  getattr(self._config, "scale_engine_state", "local_core"),
102
102
  )
103
+ # v3.8.5: schedule a background check that auto-promotes to
104
+ # Cozo+LanceDB only once the DB is large enough that they beat the
105
+ # SQLite graph. A no-op (and never even starts the build) for the
106
+ # vast majority of installs, which sit far below the threshold.
107
+ self._maybe_schedule_auto_promote()
103
108
  return
104
109
 
105
110
  # 3. Initialize CozoDB if available
@@ -129,6 +134,94 @@ class BackendOrchestrator:
129
134
  "active" if self._cozo and self._cozo_status() == "active" else "off",
130
135
  "active" if self._lancedb and self._lancedb_status() == "active" else "off")
131
136
 
137
+ def _maybe_schedule_auto_promote(self) -> None:
138
+ """Schedule a delayed, one-shot scale auto-promote check (v3.8.5).
139
+
140
+ Fires well after boot warmup so it never competes for CPU / the write
141
+ lock during the startup window — the daemon keeps serving canonical
142
+ SQLite throughout. A no-op (never even starts the build) unless
143
+ auto-promotion is enabled AND the DB has grown past the threshold where
144
+ a graph DB actually beats the well-indexed SQLite graph.
145
+ """
146
+ import os
147
+ import threading
148
+
149
+ cfg = self._config
150
+ if not getattr(cfg, "scale_auto_promote_enabled", True):
151
+ return
152
+ if getattr(cfg, "scale_engine_state", "local_core") != "local_core":
153
+ return
154
+ try:
155
+ delay = float(os.environ.get("SLM_AUTO_PROMOTE_DELAY_S", "300"))
156
+ except (TypeError, ValueError):
157
+ delay = 300.0
158
+ timer = threading.Timer(delay, self._auto_promote_if_at_scale)
159
+ timer.daemon = True
160
+ timer.start()
161
+
162
+ def _count_default_edges(self) -> int:
163
+ """graph_edges count for the default profile (fail-soft → 0)."""
164
+ try:
165
+ rows = self._db.execute(
166
+ "SELECT COUNT(*) AS c FROM graph_edges WHERE profile_id = 'default'"
167
+ )
168
+ return int(rows[0]["c"]) if rows else 0
169
+ except Exception:
170
+ return 0
171
+
172
+ def _auto_promote_if_at_scale(self) -> None:
173
+ """Build + promote the Cozo/Lance projection iff the DB is at scale.
174
+
175
+ Uses the SAME staged parity gate as the manual CLI path
176
+ (prepare → verify → promote). Any failure leaves canonical SQLite
177
+ selected — the projection is derived data, never the source of truth.
178
+ The promoted backends only serve after the next daemon restart, so this
179
+ logs a clear, actionable message rather than swapping under a live
180
+ process.
181
+ """
182
+ try:
183
+ import os
184
+
185
+ cfg = self._config
186
+ if getattr(cfg, "scale_engine_state", "local_core") != "local_core":
187
+ return
188
+ threshold = int(
189
+ os.environ.get("SLM_AUTO_PROMOTE_MIN_EDGES", "")
190
+ or getattr(cfg, "scale_auto_promote_min_edges", 1_000_000)
191
+ )
192
+ edges = self._count_default_edges()
193
+ if edges < threshold:
194
+ logger.info(
195
+ "Scale auto-promote: %d edges < threshold %d — Local Core "
196
+ "(SQLite) stays optimal; no projection built.",
197
+ edges, threshold,
198
+ )
199
+ return
200
+ logger.info(
201
+ "Scale auto-promote: %d edges >= threshold %d — building "
202
+ "Cozo+LanceDB projection in the background (SQLite keeps serving).",
203
+ edges, threshold,
204
+ )
205
+ from superlocalmemory.core.scale_engine import ScaleEngineManager
206
+
207
+ mgr = ScaleEngineManager(cfg, profile_id="default")
208
+ prepared = mgr.prepare()
209
+ stage_id = prepared.get("stage_id")
210
+ mgr.verify(stage_id)
211
+ mgr.promote(stage_id)
212
+ logger.warning(
213
+ "Scale Engine AUTO-PROMOTED to Cozo+LanceDB at %d edges. RESTART "
214
+ "the daemon (`slm restart`) to activate the backends; until then "
215
+ "it keeps serving canonical SQLite.",
216
+ edges,
217
+ )
218
+ except Exception as exc:
219
+ # Derived-data failure must never take down Local Core.
220
+ logger.warning(
221
+ "Scale auto-promote skipped — staying on Local Core / SQLite: %s",
222
+ exc,
223
+ )
224
+
132
225
  def _recover_interrupted_scale_promotion(self) -> None:
133
226
  """Repair an interrupted promotion; never auto-mutate a legacy root."""
134
227
  try:
@@ -265,8 +265,10 @@ def probe_reranker_model(config: Any = None) -> Component:
265
265
  auto_fixable=enabled,
266
266
  fix_cmd="slm doctor --fix",
267
267
  )
268
- if not enabled and comp.status == STATUS_MISSING:
269
- # Not enabled absent is expected, not a problem.
268
+ if not enabled:
269
+ # A cached reranker is still inactive when the operator disabled the
270
+ # channel. The dashboard must describe configured runtime state, not
271
+ # machine-specific HuggingFace-cache state.
270
272
  return replace(comp, status=STATUS_OK,
271
273
  detail="disabled (retrieval.use_cross_encoder=false)",
272
274
  fix_cmd="", auto_fixable=False)
@@ -960,6 +960,29 @@ class HealthConfig:
960
960
  enable_structured_logging: bool = True
961
961
 
962
962
 
963
+ # ---------------------------------------------------------------------------
964
+ # Graph Pruning Config (Workstream G — #84)
965
+ # ---------------------------------------------------------------------------
966
+
967
+ @dataclass
968
+ class GraphPruningConfig:
969
+ """Graph thinning parameters. Exposed via dashboard + CLI.
970
+
971
+ Defaults reproduce the previous hard-coded behaviour so existing
972
+ deployments see zero behavioural change after upgrade.
973
+ """
974
+
975
+ #: Maximum in- or out-degree per node. Maps to the legacy module-level
976
+ #: ``_MAX_DEGREE_PER_NODE = 100`` in graph_pruner.py.
977
+ max_degree_per_node: int = 100
978
+ #: Discard edges with weight strictly below this floor.
979
+ #: 0.0 = no floor (current behaviour, always-safe default).
980
+ min_edge_weight: float = 0.0
981
+ #: Master kill-switch for all graph pruning. ``False`` skips the entire
982
+ #: prune step (useful for debugging graph bloat).
983
+ enabled: bool = True
984
+
985
+
963
986
  # ---------------------------------------------------------------------------
964
987
  # Master Config
965
988
  # ---------------------------------------------------------------------------
@@ -1008,8 +1031,19 @@ class SLMConfig:
1008
1031
  # Scale Engine is installed separately from activation. Existing roots
1009
1032
  # stay on SQLite until a staged parity check promotes these projections.
1010
1033
  scale_engine_state: str = "local_core" # local_core | prepared | verified | promoted
1034
+ # v3.8.5: auto-promote Cozo+LanceDB when a DB grows past the scale at which
1035
+ # they actually help. Below the threshold the well-indexed SQLite graph is
1036
+ # faster (measured ~1.7ms/traversal at 208K edges), so normal installs stay
1037
+ # on Local Core and never pay the projection/migration cost. The threshold
1038
+ # is deliberately high: the graph DB win appears at millions of edges, not
1039
+ # hundreds of thousands. Auto-promotion is background, uses the same staged
1040
+ # parity gate as the manual path, and falls back to SQLite on any failure.
1041
+ scale_auto_promote_enabled: bool = True
1042
+ scale_auto_promote_min_edges: int = 1_000_000
1011
1043
  evolution: EvolutionConfig = field(default_factory=EvolutionConfig)
1012
1044
  health: HealthConfig = field(default_factory=HealthConfig)
1045
+ # v3.8.4-G: Graph thinning parameters (#84)
1046
+ graph_pruning: GraphPruningConfig = field(default_factory=GraphPruningConfig)
1013
1047
 
1014
1048
  # v3.4.3: Daemon configuration
1015
1049
  daemon_idle_timeout: int = 0 # 0 = 24/7 (no auto-kill). >0 = seconds before auto-kill.
@@ -1122,6 +1156,15 @@ class SLMConfig:
1122
1156
  state if state in {"local_core", "prepared", "verified", "promoted"}
1123
1157
  else "local_core"
1124
1158
  )
1159
+ config.scale_auto_promote_enabled = bool(
1160
+ data.get("scale_auto_promote_enabled", True)
1161
+ )
1162
+ try:
1163
+ config.scale_auto_promote_min_edges = int(
1164
+ data.get("scale_auto_promote_min_edges", 1_000_000)
1165
+ )
1166
+ except (TypeError, ValueError):
1167
+ config.scale_auto_promote_min_edges = 1_000_000
1125
1168
 
1126
1169
  # V3.3 config fields (additive — defaults work if missing from JSON)
1127
1170
  fg = data.get("forgetting", {})
@@ -1174,6 +1217,31 @@ class SLMConfig:
1174
1217
  if k in HealthConfig.__dataclass_fields__
1175
1218
  })
1176
1219
 
1220
+ # v3.8.4-G: Graph pruning config (#84) — additive, no migration needed.
1221
+ # Old configs without this section silently fall back to GraphPruningConfig()
1222
+ # defaults, which reproduce the existing _MAX_DEGREE_PER_NODE=100 behaviour.
1223
+ gp_raw = data.get("graph_pruning", {})
1224
+ if gp_raw:
1225
+ try:
1226
+ gp_kwargs = {
1227
+ k: v for k, v in gp_raw.items()
1228
+ if k in GraphPruningConfig.__dataclass_fields__
1229
+ }
1230
+ gp = GraphPruningConfig(**gp_kwargs)
1231
+ # Validate and clamp out-of-range values
1232
+ max_deg = gp.max_degree_per_node if gp.max_degree_per_node >= 1 else 100
1233
+ min_w = max(0.0, min(1.0, gp.min_edge_weight))
1234
+ config.graph_pruning = GraphPruningConfig(
1235
+ max_degree_per_node=max_deg,
1236
+ min_edge_weight=min_w,
1237
+ enabled=gp.enabled,
1238
+ )
1239
+ except (ValueError, TypeError) as exc:
1240
+ logger.warning(
1241
+ "graph_pruning config invalid (%s) — using defaults", exc
1242
+ )
1243
+ config.graph_pruning = GraphPruningConfig()
1244
+
1177
1245
  # V3.4.65: Injection config (additive — defaults if missing from JSON)
1178
1246
  inj = data.get("injection", {}) or {}
1179
1247
  config.injection = InjectionConfig(
@@ -1271,6 +1339,8 @@ class SLMConfig:
1271
1339
  "graph_backend": self.graph_backend,
1272
1340
  "vector_backend": self.vector_backend,
1273
1341
  "scale_engine_state": self.scale_engine_state,
1342
+ "scale_auto_promote_enabled": self.scale_auto_promote_enabled,
1343
+ "scale_auto_promote_min_edges": self.scale_auto_promote_min_edges,
1274
1344
  "base_dir": str(self.base_dir), # V3.5.9: persist so load() can restore custom paths
1275
1345
  "llm": {
1276
1346
  "provider": self.llm.provider,
@@ -1329,6 +1399,14 @@ class SLMConfig:
1329
1399
  # Multi-scope memory: behaviour defaults
1330
1400
  data["scope"] = self.scope.as_dict()
1331
1401
 
1402
+ # v3.8.4-G: Persist graph pruning config (#84).
1403
+ # Always written — additive key, never overwrites unrelated sections.
1404
+ data["graph_pruning"] = {
1405
+ "max_degree_per_node": self.graph_pruning.max_degree_per_node,
1406
+ "min_edge_weight": self.graph_pruning.min_edge_weight,
1407
+ "enabled": self.graph_pruning.enabled,
1408
+ }
1409
+
1332
1410
  # Preserve existing V3.3 config sections that aren't in for_mode()
1333
1411
  for key in ("forgetting", "quantization", "sagq", "embedding_signature", "auto_invoke"):
1334
1412
  if key in existing:
@@ -708,93 +708,99 @@ class ConsolidationEngine:
708
708
  Uses 'custom' category because the soft_prompt_templates CHECK
709
709
  constraint does not include 'skill_evolution'. The content is
710
710
  prefixed with [SKILL_EVOLUTION] for easy filtering.
711
- """
712
- import sqlite3 as _sqlite3
713
711
 
714
- db_path = str(self._db.db_path)
712
+ Concurrency fix (v3.8.4): all writes routed through memory_write()
713
+ so the process write lock (get_write_lock) serialises in-process
714
+ writers and proper busy_timeout handles cross-process races.
715
+ No slow ops are held inside the write lock — this method has none.
716
+ """
717
+ from superlocalmemory.storage.memory_write import memory_read, memory_write
715
718
 
716
- conn = _sqlite3.connect(db_path, timeout=10)
717
- conn.row_factory = _sqlite3.Row
719
+ db_path = self._db.db_path
718
720
 
719
- # Fetch promoted evolutions
721
+ # Phase 1: read-only fetch — does NOT hold the write lock.
720
722
  try:
721
- promoted_rows = conn.execute(
722
- "SELECT id, skill_name, parent_skill_id, evolution_type, "
723
- "mutation_summary, created_at "
724
- "FROM skill_evolution_log "
725
- "WHERE status = 'promoted' "
726
- "ORDER BY created_at DESC LIMIT 20",
727
- ).fetchall()
728
- except _sqlite3.OperationalError:
723
+ with memory_read(db_path) as rconn:
724
+ promoted_rows = [
725
+ dict(r)
726
+ for r in rconn.execute(
727
+ "SELECT id, skill_name, parent_skill_id, evolution_type, "
728
+ "mutation_summary, created_at "
729
+ "FROM skill_evolution_log "
730
+ "WHERE status = 'promoted' "
731
+ "ORDER BY created_at DESC LIMIT 20",
732
+ ).fetchall()
733
+ ]
734
+ except Exception:
729
735
  # Table may not exist yet
730
- conn.close()
731
736
  return {"created": 0, "message": "skill_evolution_log table not found"}
732
737
 
733
- created_count = 0
738
+ if not promoted_rows:
739
+ return {"promoted_skills_found": 0, "soft_prompts_created": 0}
740
+
734
741
  now = datetime.now(timezone.utc).isoformat()
742
+ created_count = 0
735
743
 
736
- for row in promoted_rows:
737
- r = dict(row)
738
- skill_name = r["skill_name"]
739
- parent = r.get("parent_skill_id") or skill_name
740
- evo_type = r["evolution_type"]
741
- summary = r.get("mutation_summary", "")
742
- evo_id = r["id"]
743
-
744
- # Build prompt content
745
- content = (
746
- f"[SKILL_EVOLUTION] Evolved skill: '{skill_name}' "
747
- f"({'replaces' if evo_type == 'fix' else 'extends'} '{parent}' "
748
- f"via {evo_type}). {summary}. "
749
- f"Use the evolved version for better results."
750
- )
744
+ # Phase 2: short write transaction — hold the write lock for the
745
+ # INSERT/UPDATE loop only (pure SQL, no network / embed calls).
746
+ with memory_write(db_path) as conn:
747
+ for r in promoted_rows:
748
+ skill_name = r["skill_name"]
749
+ parent = r.get("parent_skill_id") or skill_name
750
+ evo_type = r["evolution_type"]
751
+ summary_txt = r.get("mutation_summary", "")
752
+ evo_id = r["id"]
753
+
754
+ content = (
755
+ f"[SKILL_EVOLUTION] Evolved skill: '{skill_name}' "
756
+ f"({'replaces' if evo_type == 'fix' else 'extends'} '{parent}' "
757
+ f"via {evo_type}). {summary_txt}. "
758
+ f"Use the evolved version for better results."
759
+ )
760
+ prompt_id = f"evo-{evo_id}"
751
761
 
752
- # Use a deterministic prompt_id based on the evolution record
753
- prompt_id = f"evo-{evo_id}"
762
+ existing = conn.execute(
763
+ "SELECT prompt_id FROM soft_prompt_templates WHERE prompt_id = ?",
764
+ (prompt_id,),
765
+ ).fetchone()
754
766
 
755
- # Check if prompt already exists
756
- existing = conn.execute(
757
- "SELECT prompt_id FROM soft_prompt_templates WHERE prompt_id = ?",
758
- (prompt_id,),
759
- ).fetchone()
767
+ if existing:
768
+ # M-REPLACE: Update existing record instead of INSERT OR REPLACE
769
+ # to avoid silently dropping columns with defaults.
770
+ try:
771
+ conn.execute(
772
+ "UPDATE soft_prompt_templates "
773
+ "SET content = ?, updated_at = ? "
774
+ "WHERE prompt_id = ?",
775
+ (content, now, prompt_id),
776
+ )
777
+ except Exception as upd_exc:
778
+ logger.debug(
779
+ "Failed to update soft prompt %s: %s", prompt_id, upd_exc
780
+ )
781
+ continue
760
782
 
761
- if existing:
762
- # M-REPLACE: Update existing record instead of INSERT OR REPLACE
763
- # to avoid silently dropping columns with defaults
764
783
  try:
765
- conn.execute(
766
- "UPDATE soft_prompt_templates "
767
- "SET content = ?, updated_at = ? "
768
- "WHERE prompt_id = ?",
769
- (content, now, prompt_id),
784
+ cur = conn.execute(
785
+ "INSERT OR IGNORE INTO soft_prompt_templates "
786
+ "(prompt_id, profile_id, category, content, source_pattern_ids, "
787
+ " confidence, effectiveness, token_count, retention_score, "
788
+ " active, version, created_at, updated_at) "
789
+ "VALUES (?, ?, 'custom', ?, ?, 0.8, 0.5, ?, 1.0, 1, 1, ?, ?)",
790
+ (
791
+ prompt_id, profile_id, content,
792
+ json.dumps([evo_id]),
793
+ len(content.split()),
794
+ now, now,
795
+ ),
796
+ )
797
+ if cur.rowcount > 0:
798
+ created_count += 1
799
+ except Exception as ins_exc:
800
+ # Unique constraint on (profile_id, category) WHERE active=1
801
+ logger.debug(
802
+ "Skipping soft prompt for %s: %s", skill_name, ins_exc
770
803
  )
771
- except _sqlite3.OperationalError as upd_exc:
772
- logger.debug("Failed to update soft prompt %s: %s", prompt_id, upd_exc)
773
- continue
774
-
775
- try:
776
- conn.execute(
777
- "INSERT OR IGNORE INTO soft_prompt_templates "
778
- "(prompt_id, profile_id, category, content, source_pattern_ids, "
779
- " confidence, effectiveness, token_count, retention_score, "
780
- " active, version, created_at, updated_at) "
781
- "VALUES (?, ?, 'custom', ?, ?, 0.8, 0.5, ?, 1.0, 1, 1, ?, ?)",
782
- (prompt_id, profile_id, content,
783
- json.dumps([evo_id]),
784
- len(content.split()), # Rough token estimate
785
- now, now),
786
- )
787
- if conn.total_changes:
788
- created_count += 1
789
- except _sqlite3.IntegrityError:
790
- # Unique constraint on (profile_id, category) WHERE active=1
791
- logger.debug(
792
- "Skipping soft prompt for %s: unique constraint on active custom",
793
- skill_name,
794
- )
795
-
796
- conn.commit()
797
- conn.close()
798
804
 
799
805
  return {
800
806
  "promoted_skills_found": len(promoted_rows),
@@ -250,11 +250,38 @@ class EmbeddingService:
250
250
  def dimension(self) -> int:
251
251
  return self._config.dimension
252
252
 
253
- def unload(self) -> None:
254
- """Kill the worker subprocess to free all memory."""
255
- with self._lock:
253
+ def unload(self, timeout: float = 1.0) -> bool:
254
+ """Release the worker without blocking daemon shutdown on an embed call.
255
+
256
+ An in-flight request owns ``_lock`` while it waits for the worker's
257
+ response. Shutdown must not wait behind a wedged response: callers
258
+ can continue teardown and the worker process will be handled by the
259
+ process supervisor if necessary.
260
+ """
261
+ if not self._lock.acquire(timeout=max(0.0, timeout)):
262
+ logger.warning("EmbeddingService: unload skipped; embed worker is busy")
263
+ return False
264
+ try:
256
265
  self._kill_worker()
257
266
  logger.info("EmbeddingService: worker killed (idle timeout)")
267
+ return True
268
+ finally:
269
+ self._lock.release()
270
+
271
+ def shutdown(self, timeout: float = 1.0) -> None:
272
+ """Force bounded process teardown even when an embed call owns the lock.
273
+
274
+ Shutdown is stronger than the idle-time ``unload`` operation. Once
275
+ the engine is closing, no new request may use this service, so it is
276
+ safe to detach and terminate a wedged child without waiting behind the
277
+ request lock.
278
+ """
279
+ acquired = self._lock.acquire(timeout=max(0.0, timeout))
280
+ try:
281
+ self._kill_worker(timeout=min(max(0.0, timeout), 1.0))
282
+ finally:
283
+ if acquired:
284
+ self._lock.release()
258
285
 
259
286
  # ------------------------------------------------------------------
260
287
  # Public API
@@ -596,7 +623,7 @@ class EmbeddingService:
596
623
  self._available = False
597
624
  self._worker_proc = None
598
625
 
599
- def _kill_worker(self) -> None:
626
+ def _kill_worker(self, timeout: float = 3.0) -> None:
600
627
  """Terminate the worker and close every owned pipe exactly once."""
601
628
  if self._idle_timer is not None:
602
629
  self._idle_timer.cancel()
@@ -610,7 +637,7 @@ class EmbeddingService:
610
637
  try:
611
638
  proc.stdin.write('{"cmd":"quit"}\n')
612
639
  proc.stdin.flush()
613
- proc.wait(timeout=3)
640
+ proc.wait(timeout=max(0.0, timeout))
614
641
  except Exception:
615
642
  try:
616
643
  returncode = proc.poll()
@@ -621,7 +648,7 @@ class EmbeddingService:
621
648
  if returncode is None or not isinstance(returncode, int):
622
649
  try:
623
650
  proc.kill()
624
- proc.wait(timeout=3)
651
+ proc.wait(timeout=max(0.0, timeout))
625
652
  except Exception:
626
653
  pass
627
654
  finally: