superlocalmemory 3.8.3 → 3.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -0
- package/README.md +3 -2
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +9 -4
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/access/rbac.py +68 -76
- package/src/superlocalmemory/cli/commands.py +158 -404
- package/src/superlocalmemory/cli/ingest_cmd.py +11 -1
- package/src/superlocalmemory/cli/main.py +30 -0
- package/src/superlocalmemory/cli/pending_store.py +39 -14
- package/src/superlocalmemory/core/backend_orchestrator.py +93 -0
- package/src/superlocalmemory/core/component_registry.py +4 -2
- package/src/superlocalmemory/core/config.py +78 -0
- package/src/superlocalmemory/core/consolidation_engine.py +79 -73
- package/src/superlocalmemory/core/embeddings.py +33 -6
- package/src/superlocalmemory/core/engine.py +186 -60
- package/src/superlocalmemory/core/engine_ingestion.py +150 -63
- package/src/superlocalmemory/core/fact_consolidator.py +148 -30
- package/src/superlocalmemory/core/graph_pruner.py +436 -39
- package/src/superlocalmemory/core/ingestion_command.py +273 -32
- package/src/superlocalmemory/core/maintenance_scheduler.py +61 -1
- package/src/superlocalmemory/core/mutations.py +32 -10
- package/src/superlocalmemory/core/recall_pipeline.py +111 -74
- package/src/superlocalmemory/core/registry.py +5 -1
- package/src/superlocalmemory/core/remember_admission.py +152 -0
- package/src/superlocalmemory/core/remember_runtime.py +712 -0
- package/src/superlocalmemory/core/remote_mode.py +3 -1
- package/src/superlocalmemory/core/scale_engine.py +41 -18
- package/src/superlocalmemory/core/store_pipeline.py +18 -4
- package/src/superlocalmemory/encoding/entity_resolver.py +18 -11
- package/src/superlocalmemory/graph/cozo_backend.py +5 -5
- package/src/superlocalmemory/hooks/_outcome_common.py +9 -2
- package/src/superlocalmemory/hooks/adapter_base.py +58 -44
- package/src/superlocalmemory/hooks/ide_connector.py +26 -8
- package/src/superlocalmemory/hooks/portable_kit.py +105 -9
- package/src/superlocalmemory/hooks/prewarm_auth.py +21 -2
- package/src/superlocalmemory/infra/auth_middleware.py +3 -1
- package/src/superlocalmemory/infra/cloud_backup.py +26 -27
- package/src/superlocalmemory/infra/event_bus.py +250 -88
- package/src/superlocalmemory/learning/bandit.py +50 -1
- package/src/superlocalmemory/learning/consolidation_cycle.py +33 -16
- package/src/superlocalmemory/learning/entity_compiler.py +148 -132
- package/src/superlocalmemory/learning/memory_merge.py +97 -82
- package/src/superlocalmemory/learning/reward_archive.py +98 -90
- package/src/superlocalmemory/learning/reward_boost.py +40 -30
- package/src/superlocalmemory/learning/source_quality.py +38 -35
- package/src/superlocalmemory/mcp/_daemon_proxy.py +38 -15
- package/src/superlocalmemory/mcp/http_transport.py +335 -3
- package/src/superlocalmemory/mcp/tools_active.py +4 -41
- package/src/superlocalmemory/mcp/tools_core.py +26 -87
- package/src/superlocalmemory/mcp/tools_evolution.py +5 -10
- package/src/superlocalmemory/optimize/proxy/capture.py +196 -8
- package/src/superlocalmemory/retrieval/engine.py +15 -4
- package/src/superlocalmemory/retrieval/entity_channel.py +25 -1
- package/src/superlocalmemory/retrieval/reranker.py +130 -22
- package/src/superlocalmemory/retrieval/spreading_activation.py +20 -12
- package/src/superlocalmemory/retrieval/vector_store.py +84 -69
- package/src/superlocalmemory/server/loopback.py +85 -0
- package/src/superlocalmemory/server/origin.py +9 -4
- package/src/superlocalmemory/server/profile_runtime.py +14 -0
- package/src/superlocalmemory/server/routes/abstraction.py +2 -4
- package/src/superlocalmemory/server/routes/agents.py +3 -5
- package/src/superlocalmemory/server/routes/backup.py +6 -2
- package/src/superlocalmemory/server/routes/behavioral.py +11 -25
- package/src/superlocalmemory/server/routes/brain.py +6 -9
- package/src/superlocalmemory/server/routes/compliance.py +20 -23
- package/src/superlocalmemory/server/routes/config_api.py +83 -0
- package/src/superlocalmemory/server/routes/entity.py +3 -7
- package/src/superlocalmemory/server/routes/evolution.py +3 -5
- package/src/superlocalmemory/server/routes/helpers.py +57 -25
- package/src/superlocalmemory/server/routes/insights.py +2 -4
- package/src/superlocalmemory/server/routes/learning.py +2 -5
- package/src/superlocalmemory/server/routes/lifecycle.py +2 -4
- package/src/superlocalmemory/server/routes/memories.py +119 -98
- package/src/superlocalmemory/server/routes/mesh.py +7 -2
- package/src/superlocalmemory/server/routes/profiles.py +20 -21
- package/src/superlocalmemory/server/routes/rbac.py +0 -1
- package/src/superlocalmemory/server/routes/tiers.py +28 -35
- package/src/superlocalmemory/server/routes/timeline.py +2 -4
- package/src/superlocalmemory/server/routes/v3_api.py +85 -93
- package/src/superlocalmemory/server/unified_daemon.py +400 -140
- package/src/superlocalmemory/server/write_identity.py +22 -4
- package/src/superlocalmemory/storage/admission_codec.py +119 -0
- package/src/superlocalmemory/storage/admission_journal.py +728 -0
- package/src/superlocalmemory/storage/database.py +168 -19
- package/src/superlocalmemory/storage/deferred_writes.py +209 -0
- package/src/superlocalmemory/storage/embedding_migrator.py +19 -0
- package/src/superlocalmemory/storage/memory_write.py +115 -0
- package/src/superlocalmemory/storage/migration_runner.py +44 -0
- package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +113 -78
- package/src/superlocalmemory/storage/migrations/M031_dead_letter_operations.py +80 -0
- package/src/superlocalmemory/storage/migrations/M032_write_coordinator_admission.py +188 -0
- package/src/superlocalmemory/storage/read_connection.py +115 -0
- package/src/superlocalmemory/storage/write_coordinator.py +756 -0
- package/src/superlocalmemory/storage/write_lock.py +88 -0
- package/src/superlocalmemory/ui/index.html +1 -1
- package/src/superlocalmemory/ui/js/auto-settings.js +14 -1
- package/src/superlocalmemory/ui/js/od-settings.js +9 -3
|
@@ -281,15 +281,25 @@ def _write_tool_events(events: list[dict]) -> int:
|
|
|
281
281
|
|
|
282
282
|
v3.4.10: Preserves input_summary, output_summary, and project_path
|
|
283
283
|
from enriched sources (ECC observations, enriched hook).
|
|
284
|
+
|
|
285
|
+
Concurrency note
|
|
286
|
+
----------------
|
|
287
|
+
``slm ingest`` is a SEPARATE OS process — the daemon's threading.RLock
|
|
288
|
+
write-lock cannot help here. PRAGMA busy_timeout is the only cross-process
|
|
289
|
+
lever: it makes SQLite WAIT for the single writer (daemon / hook) to finish
|
|
290
|
+
instead of immediately raising SQLITE_BUSY. The ``timeout`` kwarg to
|
|
291
|
+
sqlite3.connect is the Python-level retry budget (same duration).
|
|
284
292
|
"""
|
|
285
293
|
db_path = Path(MEMORY_DB)
|
|
286
294
|
if not db_path.exists():
|
|
287
295
|
return 0
|
|
288
296
|
|
|
289
|
-
|
|
297
|
+
_busy_ms = 10_000
|
|
298
|
+
conn = sqlite3.connect(str(db_path), timeout=_busy_ms / 1000.0)
|
|
290
299
|
count = 0
|
|
291
300
|
|
|
292
301
|
try:
|
|
302
|
+
conn.execute(f"PRAGMA busy_timeout={_busy_ms}")
|
|
293
303
|
for ev in events:
|
|
294
304
|
try:
|
|
295
305
|
conn.execute(
|
|
@@ -252,6 +252,36 @@ def main() -> None:
|
|
|
252
252
|
"--dry-run", action="store_true", dest="dry_run", default=False,
|
|
253
253
|
help="Show what would be written without making changes",
|
|
254
254
|
)
|
|
255
|
+
# Workstream C — transport flexibility (3.8.4)
|
|
256
|
+
connect_p.add_argument(
|
|
257
|
+
"--transport",
|
|
258
|
+
choices=["stdio", "http", "http-mcp-remote"],
|
|
259
|
+
default="stdio",
|
|
260
|
+
dest="transport",
|
|
261
|
+
help=(
|
|
262
|
+
"MCP transport to write into the IDE config. "
|
|
263
|
+
"stdio (default, zero regression) | "
|
|
264
|
+
"http (native Streamable-HTTP, requires SLM daemon) | "
|
|
265
|
+
"http-mcp-remote (stdio bridge via mcp-remote for stdio-only clients)"
|
|
266
|
+
),
|
|
267
|
+
)
|
|
268
|
+
connect_p.add_argument(
|
|
269
|
+
"--port",
|
|
270
|
+
type=int,
|
|
271
|
+
default=8765,
|
|
272
|
+
dest="daemon_port",
|
|
273
|
+
help="SLM daemon port for http/http-mcp-remote transport (default: 8765)",
|
|
274
|
+
)
|
|
275
|
+
connect_p.add_argument(
|
|
276
|
+
"--verify",
|
|
277
|
+
action="store_true",
|
|
278
|
+
default=False,
|
|
279
|
+
dest="verify",
|
|
280
|
+
help=(
|
|
281
|
+
"After writing the config, probe the daemon health endpoint to confirm "
|
|
282
|
+
"the transport is reachable (only meaningful for --transport http)"
|
|
283
|
+
),
|
|
284
|
+
)
|
|
255
285
|
|
|
256
286
|
migrate_p = sub.add_parser("migrate", help="Migrate data from V2 to V3 schema")
|
|
257
287
|
migrate_p.add_argument(
|
|
@@ -40,6 +40,13 @@ _MAX_RETRIES = 3
|
|
|
40
40
|
_STUCK_DAYS = 7
|
|
41
41
|
_MAX_RETRY_DELAY_SECONDS = 3600
|
|
42
42
|
|
|
43
|
+
# Fix F: hard cap on pending.db retries. After this many failed attempts the
|
|
44
|
+
# item transitions to status='dead_letter' (next_retry_at=NULL) so it is
|
|
45
|
+
# permanently excluded from the work queue. Value chosen to survive transient
|
|
46
|
+
# engine unavailability (12h at 30-min drain intervals) while still providing
|
|
47
|
+
# a definitive signal for genuinely poisoned content.
|
|
48
|
+
_MAX_RETRY_COUNT: int = 20
|
|
49
|
+
|
|
43
50
|
_SCHEMA = """
|
|
44
51
|
CREATE TABLE IF NOT EXISTS pending_memories (
|
|
45
52
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -180,9 +187,17 @@ def mark_done(row_id: int, base_dir: Path | None = None) -> None:
|
|
|
180
187
|
def mark_failed(row_id: int, error: str, base_dir: Path | None = None) -> None:
|
|
181
188
|
"""Mark a pending memory as failed with error message.
|
|
182
189
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
190
|
+
Fix F: when the retry counter reaches _MAX_RETRY_COUNT the item transitions
|
|
191
|
+
to ``status='dead_letter'`` with ``next_retry_at=NULL``. Dead-lettered
|
|
192
|
+
items are excluded from the work queue (``get_pending`` filters
|
|
193
|
+
``status='pending'``) and are not resurrected by the startup
|
|
194
|
+
``UPDATE…WHERE status='failed'`` sweep (which targets only 'failed', not
|
|
195
|
+
'dead_letter'). This gives operators a stable, inspectable record of
|
|
196
|
+
genuinely poisoned content instead of an infinite retry loop.
|
|
197
|
+
|
|
198
|
+
Below the cap, failures stay pending with bounded exponential backoff;
|
|
199
|
+
M018 idempotency makes repeated replay safe once the canonical operation
|
|
200
|
+
has been created.
|
|
186
201
|
"""
|
|
187
202
|
conn = _get_db(base_dir)
|
|
188
203
|
try:
|
|
@@ -193,17 +208,27 @@ def mark_failed(row_id: int, error: str, base_dir: Path | None = None) -> None:
|
|
|
193
208
|
if row is None:
|
|
194
209
|
return
|
|
195
210
|
next_count = int(row[0] or 0) + 1
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
211
|
+
if next_count >= _MAX_RETRY_COUNT:
|
|
212
|
+
# Fix F: permanently exclude from work queue — dead-letter.
|
|
213
|
+
conn.execute(
|
|
214
|
+
"UPDATE pending_memories SET error = ?, "
|
|
215
|
+
"retry_count = retry_count + 1, "
|
|
216
|
+
"status = 'dead_letter', next_retry_at = NULL "
|
|
217
|
+
"WHERE id = ?",
|
|
218
|
+
(error, row_id),
|
|
219
|
+
)
|
|
220
|
+
else:
|
|
221
|
+
delay = 0 if next_count == 1 else min(
|
|
222
|
+
2 ** min(next_count - 1, 12),
|
|
223
|
+
_MAX_RETRY_DELAY_SECONDS,
|
|
224
|
+
)
|
|
225
|
+
conn.execute(
|
|
226
|
+
"UPDATE pending_memories SET error = ?, "
|
|
227
|
+
"retry_count = retry_count + 1, "
|
|
228
|
+
"status = 'pending', next_retry_at = ? "
|
|
229
|
+
"WHERE id = ?",
|
|
230
|
+
(error, time.time() + delay, row_id),
|
|
231
|
+
)
|
|
207
232
|
conn.commit()
|
|
208
233
|
finally:
|
|
209
234
|
conn.close()
|
|
@@ -100,6 +100,11 @@ class BackendOrchestrator:
|
|
|
100
100
|
"Scale Engine remains on Local Core (state=%s)",
|
|
101
101
|
getattr(self._config, "scale_engine_state", "local_core"),
|
|
102
102
|
)
|
|
103
|
+
# v3.8.5: schedule a background check that auto-promotes to
|
|
104
|
+
# Cozo+LanceDB only once the DB is large enough that they beat the
|
|
105
|
+
# SQLite graph. A no-op (and never even starts the build) for the
|
|
106
|
+
# vast majority of installs, which sit far below the threshold.
|
|
107
|
+
self._maybe_schedule_auto_promote()
|
|
103
108
|
return
|
|
104
109
|
|
|
105
110
|
# 3. Initialize CozoDB if available
|
|
@@ -129,6 +134,94 @@ class BackendOrchestrator:
|
|
|
129
134
|
"active" if self._cozo and self._cozo_status() == "active" else "off",
|
|
130
135
|
"active" if self._lancedb and self._lancedb_status() == "active" else "off")
|
|
131
136
|
|
|
137
|
+
def _maybe_schedule_auto_promote(self) -> None:
|
|
138
|
+
"""Schedule a delayed, one-shot scale auto-promote check (v3.8.5).
|
|
139
|
+
|
|
140
|
+
Fires well after boot warmup so it never competes for CPU / the write
|
|
141
|
+
lock during the startup window — the daemon keeps serving canonical
|
|
142
|
+
SQLite throughout. A no-op (never even starts the build) unless
|
|
143
|
+
auto-promotion is enabled AND the DB has grown past the threshold where
|
|
144
|
+
a graph DB actually beats the well-indexed SQLite graph.
|
|
145
|
+
"""
|
|
146
|
+
import os
|
|
147
|
+
import threading
|
|
148
|
+
|
|
149
|
+
cfg = self._config
|
|
150
|
+
if not getattr(cfg, "scale_auto_promote_enabled", True):
|
|
151
|
+
return
|
|
152
|
+
if getattr(cfg, "scale_engine_state", "local_core") != "local_core":
|
|
153
|
+
return
|
|
154
|
+
try:
|
|
155
|
+
delay = float(os.environ.get("SLM_AUTO_PROMOTE_DELAY_S", "300"))
|
|
156
|
+
except (TypeError, ValueError):
|
|
157
|
+
delay = 300.0
|
|
158
|
+
timer = threading.Timer(delay, self._auto_promote_if_at_scale)
|
|
159
|
+
timer.daemon = True
|
|
160
|
+
timer.start()
|
|
161
|
+
|
|
162
|
+
def _count_default_edges(self) -> int:
|
|
163
|
+
"""graph_edges count for the default profile (fail-soft → 0)."""
|
|
164
|
+
try:
|
|
165
|
+
rows = self._db.execute(
|
|
166
|
+
"SELECT COUNT(*) AS c FROM graph_edges WHERE profile_id = 'default'"
|
|
167
|
+
)
|
|
168
|
+
return int(rows[0]["c"]) if rows else 0
|
|
169
|
+
except Exception:
|
|
170
|
+
return 0
|
|
171
|
+
|
|
172
|
+
def _auto_promote_if_at_scale(self) -> None:
|
|
173
|
+
"""Build + promote the Cozo/Lance projection iff the DB is at scale.
|
|
174
|
+
|
|
175
|
+
Uses the SAME staged parity gate as the manual CLI path
|
|
176
|
+
(prepare → verify → promote). Any failure leaves canonical SQLite
|
|
177
|
+
selected — the projection is derived data, never the source of truth.
|
|
178
|
+
The promoted backends only serve after the next daemon restart, so this
|
|
179
|
+
logs a clear, actionable message rather than swapping under a live
|
|
180
|
+
process.
|
|
181
|
+
"""
|
|
182
|
+
try:
|
|
183
|
+
import os
|
|
184
|
+
|
|
185
|
+
cfg = self._config
|
|
186
|
+
if getattr(cfg, "scale_engine_state", "local_core") != "local_core":
|
|
187
|
+
return
|
|
188
|
+
threshold = int(
|
|
189
|
+
os.environ.get("SLM_AUTO_PROMOTE_MIN_EDGES", "")
|
|
190
|
+
or getattr(cfg, "scale_auto_promote_min_edges", 1_000_000)
|
|
191
|
+
)
|
|
192
|
+
edges = self._count_default_edges()
|
|
193
|
+
if edges < threshold:
|
|
194
|
+
logger.info(
|
|
195
|
+
"Scale auto-promote: %d edges < threshold %d — Local Core "
|
|
196
|
+
"(SQLite) stays optimal; no projection built.",
|
|
197
|
+
edges, threshold,
|
|
198
|
+
)
|
|
199
|
+
return
|
|
200
|
+
logger.info(
|
|
201
|
+
"Scale auto-promote: %d edges >= threshold %d — building "
|
|
202
|
+
"Cozo+LanceDB projection in the background (SQLite keeps serving).",
|
|
203
|
+
edges, threshold,
|
|
204
|
+
)
|
|
205
|
+
from superlocalmemory.core.scale_engine import ScaleEngineManager
|
|
206
|
+
|
|
207
|
+
mgr = ScaleEngineManager(cfg, profile_id="default")
|
|
208
|
+
prepared = mgr.prepare()
|
|
209
|
+
stage_id = prepared.get("stage_id")
|
|
210
|
+
mgr.verify(stage_id)
|
|
211
|
+
mgr.promote(stage_id)
|
|
212
|
+
logger.warning(
|
|
213
|
+
"Scale Engine AUTO-PROMOTED to Cozo+LanceDB at %d edges. RESTART "
|
|
214
|
+
"the daemon (`slm restart`) to activate the backends; until then "
|
|
215
|
+
"it keeps serving canonical SQLite.",
|
|
216
|
+
edges,
|
|
217
|
+
)
|
|
218
|
+
except Exception as exc:
|
|
219
|
+
# Derived-data failure must never take down Local Core.
|
|
220
|
+
logger.warning(
|
|
221
|
+
"Scale auto-promote skipped — staying on Local Core / SQLite: %s",
|
|
222
|
+
exc,
|
|
223
|
+
)
|
|
224
|
+
|
|
132
225
|
def _recover_interrupted_scale_promotion(self) -> None:
|
|
133
226
|
"""Repair an interrupted promotion; never auto-mutate a legacy root."""
|
|
134
227
|
try:
|
|
@@ -265,8 +265,10 @@ def probe_reranker_model(config: Any = None) -> Component:
|
|
|
265
265
|
auto_fixable=enabled,
|
|
266
266
|
fix_cmd="slm doctor --fix",
|
|
267
267
|
)
|
|
268
|
-
if not enabled
|
|
269
|
-
#
|
|
268
|
+
if not enabled:
|
|
269
|
+
# A cached reranker is still inactive when the operator disabled the
|
|
270
|
+
# channel. The dashboard must describe configured runtime state, not
|
|
271
|
+
# machine-specific HuggingFace-cache state.
|
|
270
272
|
return replace(comp, status=STATUS_OK,
|
|
271
273
|
detail="disabled (retrieval.use_cross_encoder=false)",
|
|
272
274
|
fix_cmd="", auto_fixable=False)
|
|
@@ -960,6 +960,29 @@ class HealthConfig:
|
|
|
960
960
|
enable_structured_logging: bool = True
|
|
961
961
|
|
|
962
962
|
|
|
963
|
+
# ---------------------------------------------------------------------------
|
|
964
|
+
# Graph Pruning Config (Workstream G — #84)
|
|
965
|
+
# ---------------------------------------------------------------------------
|
|
966
|
+
|
|
967
|
+
@dataclass
|
|
968
|
+
class GraphPruningConfig:
|
|
969
|
+
"""Graph thinning parameters. Exposed via dashboard + CLI.
|
|
970
|
+
|
|
971
|
+
Defaults reproduce the previous hard-coded behaviour so existing
|
|
972
|
+
deployments see zero behavioural change after upgrade.
|
|
973
|
+
"""
|
|
974
|
+
|
|
975
|
+
#: Maximum in- or out-degree per node. Maps to the legacy module-level
|
|
976
|
+
#: ``_MAX_DEGREE_PER_NODE = 100`` in graph_pruner.py.
|
|
977
|
+
max_degree_per_node: int = 100
|
|
978
|
+
#: Discard edges with weight strictly below this floor.
|
|
979
|
+
#: 0.0 = no floor (current behaviour, always-safe default).
|
|
980
|
+
min_edge_weight: float = 0.0
|
|
981
|
+
#: Master kill-switch for all graph pruning. ``False`` skips the entire
|
|
982
|
+
#: prune step (useful for debugging graph bloat).
|
|
983
|
+
enabled: bool = True
|
|
984
|
+
|
|
985
|
+
|
|
963
986
|
# ---------------------------------------------------------------------------
|
|
964
987
|
# Master Config
|
|
965
988
|
# ---------------------------------------------------------------------------
|
|
@@ -1008,8 +1031,19 @@ class SLMConfig:
|
|
|
1008
1031
|
# Scale Engine is installed separately from activation. Existing roots
|
|
1009
1032
|
# stay on SQLite until a staged parity check promotes these projections.
|
|
1010
1033
|
scale_engine_state: str = "local_core" # local_core | prepared | verified | promoted
|
|
1034
|
+
# v3.8.5: auto-promote Cozo+LanceDB when a DB grows past the scale at which
|
|
1035
|
+
# they actually help. Below the threshold the well-indexed SQLite graph is
|
|
1036
|
+
# faster (measured ~1.7ms/traversal at 208K edges), so normal installs stay
|
|
1037
|
+
# on Local Core and never pay the projection/migration cost. The threshold
|
|
1038
|
+
# is deliberately high: the graph DB win appears at millions of edges, not
|
|
1039
|
+
# hundreds of thousands. Auto-promotion is background, uses the same staged
|
|
1040
|
+
# parity gate as the manual path, and falls back to SQLite on any failure.
|
|
1041
|
+
scale_auto_promote_enabled: bool = True
|
|
1042
|
+
scale_auto_promote_min_edges: int = 1_000_000
|
|
1011
1043
|
evolution: EvolutionConfig = field(default_factory=EvolutionConfig)
|
|
1012
1044
|
health: HealthConfig = field(default_factory=HealthConfig)
|
|
1045
|
+
# v3.8.4-G: Graph thinning parameters (#84)
|
|
1046
|
+
graph_pruning: GraphPruningConfig = field(default_factory=GraphPruningConfig)
|
|
1013
1047
|
|
|
1014
1048
|
# v3.4.3: Daemon configuration
|
|
1015
1049
|
daemon_idle_timeout: int = 0 # 0 = 24/7 (no auto-kill). >0 = seconds before auto-kill.
|
|
@@ -1122,6 +1156,15 @@ class SLMConfig:
|
|
|
1122
1156
|
state if state in {"local_core", "prepared", "verified", "promoted"}
|
|
1123
1157
|
else "local_core"
|
|
1124
1158
|
)
|
|
1159
|
+
config.scale_auto_promote_enabled = bool(
|
|
1160
|
+
data.get("scale_auto_promote_enabled", True)
|
|
1161
|
+
)
|
|
1162
|
+
try:
|
|
1163
|
+
config.scale_auto_promote_min_edges = int(
|
|
1164
|
+
data.get("scale_auto_promote_min_edges", 1_000_000)
|
|
1165
|
+
)
|
|
1166
|
+
except (TypeError, ValueError):
|
|
1167
|
+
config.scale_auto_promote_min_edges = 1_000_000
|
|
1125
1168
|
|
|
1126
1169
|
# V3.3 config fields (additive — defaults work if missing from JSON)
|
|
1127
1170
|
fg = data.get("forgetting", {})
|
|
@@ -1174,6 +1217,31 @@ class SLMConfig:
|
|
|
1174
1217
|
if k in HealthConfig.__dataclass_fields__
|
|
1175
1218
|
})
|
|
1176
1219
|
|
|
1220
|
+
# v3.8.4-G: Graph pruning config (#84) — additive, no migration needed.
|
|
1221
|
+
# Old configs without this section silently fall back to GraphPruningConfig()
|
|
1222
|
+
# defaults, which reproduce the existing _MAX_DEGREE_PER_NODE=100 behaviour.
|
|
1223
|
+
gp_raw = data.get("graph_pruning", {})
|
|
1224
|
+
if gp_raw:
|
|
1225
|
+
try:
|
|
1226
|
+
gp_kwargs = {
|
|
1227
|
+
k: v for k, v in gp_raw.items()
|
|
1228
|
+
if k in GraphPruningConfig.__dataclass_fields__
|
|
1229
|
+
}
|
|
1230
|
+
gp = GraphPruningConfig(**gp_kwargs)
|
|
1231
|
+
# Validate and clamp out-of-range values
|
|
1232
|
+
max_deg = gp.max_degree_per_node if gp.max_degree_per_node >= 1 else 100
|
|
1233
|
+
min_w = max(0.0, min(1.0, gp.min_edge_weight))
|
|
1234
|
+
config.graph_pruning = GraphPruningConfig(
|
|
1235
|
+
max_degree_per_node=max_deg,
|
|
1236
|
+
min_edge_weight=min_w,
|
|
1237
|
+
enabled=gp.enabled,
|
|
1238
|
+
)
|
|
1239
|
+
except (ValueError, TypeError) as exc:
|
|
1240
|
+
logger.warning(
|
|
1241
|
+
"graph_pruning config invalid (%s) — using defaults", exc
|
|
1242
|
+
)
|
|
1243
|
+
config.graph_pruning = GraphPruningConfig()
|
|
1244
|
+
|
|
1177
1245
|
# V3.4.65: Injection config (additive — defaults if missing from JSON)
|
|
1178
1246
|
inj = data.get("injection", {}) or {}
|
|
1179
1247
|
config.injection = InjectionConfig(
|
|
@@ -1271,6 +1339,8 @@ class SLMConfig:
|
|
|
1271
1339
|
"graph_backend": self.graph_backend,
|
|
1272
1340
|
"vector_backend": self.vector_backend,
|
|
1273
1341
|
"scale_engine_state": self.scale_engine_state,
|
|
1342
|
+
"scale_auto_promote_enabled": self.scale_auto_promote_enabled,
|
|
1343
|
+
"scale_auto_promote_min_edges": self.scale_auto_promote_min_edges,
|
|
1274
1344
|
"base_dir": str(self.base_dir), # V3.5.9: persist so load() can restore custom paths
|
|
1275
1345
|
"llm": {
|
|
1276
1346
|
"provider": self.llm.provider,
|
|
@@ -1329,6 +1399,14 @@ class SLMConfig:
|
|
|
1329
1399
|
# Multi-scope memory: behaviour defaults
|
|
1330
1400
|
data["scope"] = self.scope.as_dict()
|
|
1331
1401
|
|
|
1402
|
+
# v3.8.4-G: Persist graph pruning config (#84).
|
|
1403
|
+
# Always written — additive key, never overwrites unrelated sections.
|
|
1404
|
+
data["graph_pruning"] = {
|
|
1405
|
+
"max_degree_per_node": self.graph_pruning.max_degree_per_node,
|
|
1406
|
+
"min_edge_weight": self.graph_pruning.min_edge_weight,
|
|
1407
|
+
"enabled": self.graph_pruning.enabled,
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1332
1410
|
# Preserve existing V3.3 config sections that aren't in for_mode()
|
|
1333
1411
|
for key in ("forgetting", "quantization", "sagq", "embedding_signature", "auto_invoke"):
|
|
1334
1412
|
if key in existing:
|
|
@@ -708,93 +708,99 @@ class ConsolidationEngine:
|
|
|
708
708
|
Uses 'custom' category because the soft_prompt_templates CHECK
|
|
709
709
|
constraint does not include 'skill_evolution'. The content is
|
|
710
710
|
prefixed with [SKILL_EVOLUTION] for easy filtering.
|
|
711
|
-
"""
|
|
712
|
-
import sqlite3 as _sqlite3
|
|
713
711
|
|
|
714
|
-
|
|
712
|
+
Concurrency fix (v3.8.4): all writes routed through memory_write()
|
|
713
|
+
so the process write lock (get_write_lock) serialises in-process
|
|
714
|
+
writers and proper busy_timeout handles cross-process races.
|
|
715
|
+
No slow ops are held inside the write lock — this method has none.
|
|
716
|
+
"""
|
|
717
|
+
from superlocalmemory.storage.memory_write import memory_read, memory_write
|
|
715
718
|
|
|
716
|
-
|
|
717
|
-
conn.row_factory = _sqlite3.Row
|
|
719
|
+
db_path = self._db.db_path
|
|
718
720
|
|
|
719
|
-
#
|
|
721
|
+
# Phase 1: read-only fetch — does NOT hold the write lock.
|
|
720
722
|
try:
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
723
|
+
with memory_read(db_path) as rconn:
|
|
724
|
+
promoted_rows = [
|
|
725
|
+
dict(r)
|
|
726
|
+
for r in rconn.execute(
|
|
727
|
+
"SELECT id, skill_name, parent_skill_id, evolution_type, "
|
|
728
|
+
"mutation_summary, created_at "
|
|
729
|
+
"FROM skill_evolution_log "
|
|
730
|
+
"WHERE status = 'promoted' "
|
|
731
|
+
"ORDER BY created_at DESC LIMIT 20",
|
|
732
|
+
).fetchall()
|
|
733
|
+
]
|
|
734
|
+
except Exception:
|
|
729
735
|
# Table may not exist yet
|
|
730
|
-
conn.close()
|
|
731
736
|
return {"created": 0, "message": "skill_evolution_log table not found"}
|
|
732
737
|
|
|
733
|
-
|
|
738
|
+
if not promoted_rows:
|
|
739
|
+
return {"promoted_skills_found": 0, "soft_prompts_created": 0}
|
|
740
|
+
|
|
734
741
|
now = datetime.now(timezone.utc).isoformat()
|
|
742
|
+
created_count = 0
|
|
735
743
|
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
744
|
+
# Phase 2: short write transaction — hold the write lock for the
|
|
745
|
+
# INSERT/UPDATE loop only (pure SQL, no network / embed calls).
|
|
746
|
+
with memory_write(db_path) as conn:
|
|
747
|
+
for r in promoted_rows:
|
|
748
|
+
skill_name = r["skill_name"]
|
|
749
|
+
parent = r.get("parent_skill_id") or skill_name
|
|
750
|
+
evo_type = r["evolution_type"]
|
|
751
|
+
summary_txt = r.get("mutation_summary", "")
|
|
752
|
+
evo_id = r["id"]
|
|
753
|
+
|
|
754
|
+
content = (
|
|
755
|
+
f"[SKILL_EVOLUTION] Evolved skill: '{skill_name}' "
|
|
756
|
+
f"({'replaces' if evo_type == 'fix' else 'extends'} '{parent}' "
|
|
757
|
+
f"via {evo_type}). {summary_txt}. "
|
|
758
|
+
f"Use the evolved version for better results."
|
|
759
|
+
)
|
|
760
|
+
prompt_id = f"evo-{evo_id}"
|
|
751
761
|
|
|
752
|
-
|
|
753
|
-
|
|
762
|
+
existing = conn.execute(
|
|
763
|
+
"SELECT prompt_id FROM soft_prompt_templates WHERE prompt_id = ?",
|
|
764
|
+
(prompt_id,),
|
|
765
|
+
).fetchone()
|
|
754
766
|
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
767
|
+
if existing:
|
|
768
|
+
# M-REPLACE: Update existing record instead of INSERT OR REPLACE
|
|
769
|
+
# to avoid silently dropping columns with defaults.
|
|
770
|
+
try:
|
|
771
|
+
conn.execute(
|
|
772
|
+
"UPDATE soft_prompt_templates "
|
|
773
|
+
"SET content = ?, updated_at = ? "
|
|
774
|
+
"WHERE prompt_id = ?",
|
|
775
|
+
(content, now, prompt_id),
|
|
776
|
+
)
|
|
777
|
+
except Exception as upd_exc:
|
|
778
|
+
logger.debug(
|
|
779
|
+
"Failed to update soft prompt %s: %s", prompt_id, upd_exc
|
|
780
|
+
)
|
|
781
|
+
continue
|
|
760
782
|
|
|
761
|
-
if existing:
|
|
762
|
-
# M-REPLACE: Update existing record instead of INSERT OR REPLACE
|
|
763
|
-
# to avoid silently dropping columns with defaults
|
|
764
783
|
try:
|
|
765
|
-
conn.execute(
|
|
766
|
-
"
|
|
767
|
-
"
|
|
768
|
-
"
|
|
769
|
-
|
|
784
|
+
cur = conn.execute(
|
|
785
|
+
"INSERT OR IGNORE INTO soft_prompt_templates "
|
|
786
|
+
"(prompt_id, profile_id, category, content, source_pattern_ids, "
|
|
787
|
+
" confidence, effectiveness, token_count, retention_score, "
|
|
788
|
+
" active, version, created_at, updated_at) "
|
|
789
|
+
"VALUES (?, ?, 'custom', ?, ?, 0.8, 0.5, ?, 1.0, 1, 1, ?, ?)",
|
|
790
|
+
(
|
|
791
|
+
prompt_id, profile_id, content,
|
|
792
|
+
json.dumps([evo_id]),
|
|
793
|
+
len(content.split()),
|
|
794
|
+
now, now,
|
|
795
|
+
),
|
|
796
|
+
)
|
|
797
|
+
if cur.rowcount > 0:
|
|
798
|
+
created_count += 1
|
|
799
|
+
except Exception as ins_exc:
|
|
800
|
+
# Unique constraint on (profile_id, category) WHERE active=1
|
|
801
|
+
logger.debug(
|
|
802
|
+
"Skipping soft prompt for %s: %s", skill_name, ins_exc
|
|
770
803
|
)
|
|
771
|
-
except _sqlite3.OperationalError as upd_exc:
|
|
772
|
-
logger.debug("Failed to update soft prompt %s: %s", prompt_id, upd_exc)
|
|
773
|
-
continue
|
|
774
|
-
|
|
775
|
-
try:
|
|
776
|
-
conn.execute(
|
|
777
|
-
"INSERT OR IGNORE INTO soft_prompt_templates "
|
|
778
|
-
"(prompt_id, profile_id, category, content, source_pattern_ids, "
|
|
779
|
-
" confidence, effectiveness, token_count, retention_score, "
|
|
780
|
-
" active, version, created_at, updated_at) "
|
|
781
|
-
"VALUES (?, ?, 'custom', ?, ?, 0.8, 0.5, ?, 1.0, 1, 1, ?, ?)",
|
|
782
|
-
(prompt_id, profile_id, content,
|
|
783
|
-
json.dumps([evo_id]),
|
|
784
|
-
len(content.split()), # Rough token estimate
|
|
785
|
-
now, now),
|
|
786
|
-
)
|
|
787
|
-
if conn.total_changes:
|
|
788
|
-
created_count += 1
|
|
789
|
-
except _sqlite3.IntegrityError:
|
|
790
|
-
# Unique constraint on (profile_id, category) WHERE active=1
|
|
791
|
-
logger.debug(
|
|
792
|
-
"Skipping soft prompt for %s: unique constraint on active custom",
|
|
793
|
-
skill_name,
|
|
794
|
-
)
|
|
795
|
-
|
|
796
|
-
conn.commit()
|
|
797
|
-
conn.close()
|
|
798
804
|
|
|
799
805
|
return {
|
|
800
806
|
"promoted_skills_found": len(promoted_rows),
|
|
@@ -250,11 +250,38 @@ class EmbeddingService:
|
|
|
250
250
|
def dimension(self) -> int:
|
|
251
251
|
return self._config.dimension
|
|
252
252
|
|
|
253
|
-
def unload(self) ->
|
|
254
|
-
"""
|
|
255
|
-
|
|
253
|
+
def unload(self, timeout: float = 1.0) -> bool:
|
|
254
|
+
"""Release the worker without blocking daemon shutdown on an embed call.
|
|
255
|
+
|
|
256
|
+
An in-flight request owns ``_lock`` while it waits for the worker's
|
|
257
|
+
response. Shutdown must not wait behind a wedged response: callers
|
|
258
|
+
can continue teardown and the worker process will be handled by the
|
|
259
|
+
process supervisor if necessary.
|
|
260
|
+
"""
|
|
261
|
+
if not self._lock.acquire(timeout=max(0.0, timeout)):
|
|
262
|
+
logger.warning("EmbeddingService: unload skipped; embed worker is busy")
|
|
263
|
+
return False
|
|
264
|
+
try:
|
|
256
265
|
self._kill_worker()
|
|
257
266
|
logger.info("EmbeddingService: worker killed (idle timeout)")
|
|
267
|
+
return True
|
|
268
|
+
finally:
|
|
269
|
+
self._lock.release()
|
|
270
|
+
|
|
271
|
+
def shutdown(self, timeout: float = 1.0) -> None:
|
|
272
|
+
"""Force bounded process teardown even when an embed call owns the lock.
|
|
273
|
+
|
|
274
|
+
Shutdown is stronger than the idle-time ``unload`` operation. Once
|
|
275
|
+
the engine is closing, no new request may use this service, so it is
|
|
276
|
+
safe to detach and terminate a wedged child without waiting behind the
|
|
277
|
+
request lock.
|
|
278
|
+
"""
|
|
279
|
+
acquired = self._lock.acquire(timeout=max(0.0, timeout))
|
|
280
|
+
try:
|
|
281
|
+
self._kill_worker(timeout=min(max(0.0, timeout), 1.0))
|
|
282
|
+
finally:
|
|
283
|
+
if acquired:
|
|
284
|
+
self._lock.release()
|
|
258
285
|
|
|
259
286
|
# ------------------------------------------------------------------
|
|
260
287
|
# Public API
|
|
@@ -596,7 +623,7 @@ class EmbeddingService:
|
|
|
596
623
|
self._available = False
|
|
597
624
|
self._worker_proc = None
|
|
598
625
|
|
|
599
|
-
def _kill_worker(self) -> None:
|
|
626
|
+
def _kill_worker(self, timeout: float = 3.0) -> None:
|
|
600
627
|
"""Terminate the worker and close every owned pipe exactly once."""
|
|
601
628
|
if self._idle_timer is not None:
|
|
602
629
|
self._idle_timer.cancel()
|
|
@@ -610,7 +637,7 @@ class EmbeddingService:
|
|
|
610
637
|
try:
|
|
611
638
|
proc.stdin.write('{"cmd":"quit"}\n')
|
|
612
639
|
proc.stdin.flush()
|
|
613
|
-
proc.wait(timeout=
|
|
640
|
+
proc.wait(timeout=max(0.0, timeout))
|
|
614
641
|
except Exception:
|
|
615
642
|
try:
|
|
616
643
|
returncode = proc.poll()
|
|
@@ -621,7 +648,7 @@ class EmbeddingService:
|
|
|
621
648
|
if returncode is None or not isinstance(returncode, int):
|
|
622
649
|
try:
|
|
623
650
|
proc.kill()
|
|
624
|
-
proc.wait(timeout=
|
|
651
|
+
proc.wait(timeout=max(0.0, timeout))
|
|
625
652
|
except Exception:
|
|
626
653
|
pass
|
|
627
654
|
finally:
|