superlocalmemory 3.8.0 → 3.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/CHANGELOG.md +60 -0
  2. package/README.md +32 -120
  3. package/package.json +9 -2
  4. package/plugin/.claude-plugin/plugin.json +1 -2
  5. package/plugin/CLAUDE.md +3 -3
  6. package/plugin/agents/slm-governance-advisor.md +1 -1
  7. package/plugin/agents/slm-loop-runner.md +1 -1
  8. package/plugin/agents/slm-memory-advisor.md +1 -1
  9. package/plugin/agents/slm-optimize-advisor.md +1 -1
  10. package/plugin/requirements.txt +1 -1
  11. package/plugin/skills/slm-cache/SKILL.md +1 -1
  12. package/plugin/skills/slm-compress/SKILL.md +1 -1
  13. package/plugin/skills/slm-governance/SKILL.md +1 -1
  14. package/plugin/skills/slm-graph/SKILL.md +1 -1
  15. package/plugin/skills/slm-loop/SKILL.md +1 -1
  16. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  17. package/plugin/skills/slm-profile/SKILL.md +1 -1
  18. package/plugin/skills/slm-recall/SKILL.md +1 -1
  19. package/plugin/skills/slm-remember/SKILL.md +1 -1
  20. package/plugin/skills/slm-scope/SKILL.md +1 -1
  21. package/plugin/skills/slm-session/SKILL.md +1 -1
  22. package/plugin/skills/slm-status/SKILL.md +1 -1
  23. package/plugin-src/rules/AGENTS.md +1 -1
  24. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  25. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  26. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-recall/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-remember/SKILL.md +1 -1
  29. package/plugin-src/skills/slm-session/SKILL.md +1 -1
  30. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  31. package/pyproject.toml +2 -1
  32. package/src/superlocalmemory/__init__.py +1 -1
  33. package/src/superlocalmemory/cli/commands.py +134 -7
  34. package/src/superlocalmemory/cli/daemon.py +7 -0
  35. package/src/superlocalmemory/cli/loop_cmd.py +2 -7
  36. package/src/superlocalmemory/cli/main.py +10 -4
  37. package/src/superlocalmemory/cli/version_banner.py +17 -3
  38. package/src/superlocalmemory/core/backend_orchestrator.py +18 -16
  39. package/src/superlocalmemory/core/embedding_worker.py +4 -5
  40. package/src/superlocalmemory/core/embeddings.py +119 -45
  41. package/src/superlocalmemory/core/engine.py +24 -21
  42. package/src/superlocalmemory/core/engine_ingestion.py +332 -45
  43. package/src/superlocalmemory/core/ingestion_command.py +134 -25
  44. package/src/superlocalmemory/core/injection.py +12 -7
  45. package/src/superlocalmemory/core/maintenance_scheduler.py +17 -7
  46. package/src/superlocalmemory/core/recall_pipeline.py +5 -3
  47. package/src/superlocalmemory/core/store_pipeline.py +153 -20
  48. package/src/superlocalmemory/hooks/hook_handlers.py +6 -1
  49. package/src/superlocalmemory/hooks/portable_kit.py +34 -2
  50. package/src/superlocalmemory/learning/model_rollback.py +3 -0
  51. package/src/superlocalmemory/learning/ranker_retrain_online.py +2 -0
  52. package/src/superlocalmemory/learning/reward.py +50 -0
  53. package/src/superlocalmemory/learning/source_quality.py +523 -1
  54. package/src/superlocalmemory/loops/ledger.py +25 -5
  55. package/src/superlocalmemory/mcp/server.py +11 -30
  56. package/src/superlocalmemory/mcp/tools_core.py +4 -3
  57. package/src/superlocalmemory/mcp/tools_learning.py +2 -2
  58. package/src/superlocalmemory/retrieval/engine.py +53 -21
  59. package/src/superlocalmemory/retrieval/reranker.py +3 -4
  60. package/src/superlocalmemory/server/config_file.py +90 -0
  61. package/src/superlocalmemory/server/origin.py +50 -0
  62. package/src/superlocalmemory/server/routes/backup.py +293 -70
  63. package/src/superlocalmemory/server/routes/behavioral.py +336 -59
  64. package/src/superlocalmemory/server/routes/brain.py +57 -16
  65. package/src/superlocalmemory/server/routes/config_api.py +84 -82
  66. package/src/superlocalmemory/server/routes/entity.py +100 -23
  67. package/src/superlocalmemory/server/routes/evolution.py +103 -100
  68. package/src/superlocalmemory/server/routes/learning.py +276 -105
  69. package/src/superlocalmemory/server/routes/learning_telemetry.py +153 -0
  70. package/src/superlocalmemory/server/routes/mesh.py +121 -32
  71. package/src/superlocalmemory/server/routes/ratelimit.py +33 -25
  72. package/src/superlocalmemory/server/routes/stats.py +93 -155
  73. package/src/superlocalmemory/server/routes/token.py +3 -13
  74. package/src/superlocalmemory/server/routes/v3_api.py +64 -20
  75. package/src/superlocalmemory/server/unified_daemon.py +467 -40
  76. package/src/superlocalmemory/storage/migration_runner.py +79 -1
  77. package/src/superlocalmemory/storage/migrations/M010_evolution_config.py +5 -0
  78. package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +270 -0
  79. package/src/superlocalmemory/storage/migrations/M029_behavioral_history_indexes.py +137 -0
  80. package/src/superlocalmemory/storage/migrations/M030_entity_explorer_indexes.py +93 -0
  81. package/src/superlocalmemory/storage/migrations/__init__.py +4 -0
  82. package/src/superlocalmemory/storage/schema.py +49 -1
  83. package/src/superlocalmemory/storage/schema_v32.py +2 -0
  84. package/src/superlocalmemory/storage/schema_v347.py +4 -0
  85. package/src/superlocalmemory/ui/index.html +3 -6
  86. package/src/superlocalmemory/ui/js/core.js +52 -9
  87. package/src/superlocalmemory/ui/js/dashboard.js +169 -82
  88. package/src/superlocalmemory/ui/js/od-backup.js +156 -65
  89. package/src/superlocalmemory/ui/js/od-brain.js +88 -51
  90. package/src/superlocalmemory/ui/js/od-entities.js +22 -22
  91. package/src/superlocalmemory/ui/js/od-graph.js +11 -4
  92. package/src/superlocalmemory/ui/js/od-memories.js +47 -5
  93. package/src/superlocalmemory/ui/js/od-mesh.js +23 -9
  94. package/src/superlocalmemory/ui/js/od-settings.js +113 -59
  95. package/src/superlocalmemory/ui/js/od-shell.js +249 -33
  96. package/src/superlocalmemory/ui/js/od-skills.js +44 -17
  97. package/src/superlocalmemory/ui/js/settings.js +15 -1
  98. package/plugin-src/.mcp.json +0 -12
  99. package/plugin-src/agents/slm-governance-advisor.md +0 -80
  100. package/plugin-src/agents/slm-loop-runner.md +0 -71
  101. package/plugin-src/agents/slm-memory-advisor.md +0 -49
  102. package/plugin-src/agents/slm-optimize-advisor.md +0 -44
  103. package/plugin-src/commands/slm-loop.md +0 -31
  104. package/plugin-src/hooks/.gitkeep +0 -0
  105. package/plugin-src/hooks/hooks.json +0 -102
  106. package/plugin-src/manifest.json +0 -30
  107. package/plugin-src/requirements.txt +0 -1
  108. package/plugin-src/rules/CLAUDE.md.fragment +0 -44
  109. package/plugin-src/scripts/ensure-venv.bat +0 -122
  110. package/plugin-src/scripts/ensure-venv.sh +0 -105
  111. package/plugin-src/scripts/slm-launch +0 -62
  112. package/plugin-src/scripts/slm-launch.bat +0 -23
  113. package/plugin-src/settings.json +0 -25
  114. package/plugin-src/skills/slm-governance/SKILL.md +0 -248
  115. package/plugin-src/skills/slm-loop/SKILL.md +0 -99
  116. package/plugin-src/skills/slm-mesh/SKILL.md +0 -282
  117. package/plugin-src/skills/slm-profile/SKILL.md +0 -148
  118. package/plugin-src/skills/slm-scope/SKILL.md +0 -176
@@ -23,12 +23,14 @@ Storage:
23
23
 
24
24
  from __future__ import annotations
25
25
 
26
+ import json
26
27
  import logging
28
+ import math
27
29
  import sqlite3
28
30
  import threading
29
31
  from datetime import datetime, timezone
30
32
  from pathlib import Path
31
- from typing import Any, Dict, Optional
33
+ from typing import Any, Dict
32
34
 
33
35
  logger = logging.getLogger("superlocalmemory.learning.source_quality")
34
36
 
@@ -55,6 +57,35 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_sq_profile_source
55
57
  ON source_quality (profile_id, source_id)
56
58
  """
57
59
 
60
+ _CREATE_OBSERVATIONS = """
61
+ CREATE TABLE IF NOT EXISTS source_quality_observations (
62
+ profile_id TEXT NOT NULL,
63
+ outcome_id TEXT NOT NULL,
64
+ source_id TEXT NOT NULL,
65
+ reward REAL NOT NULL,
66
+ observed_at TEXT NOT NULL,
67
+ PRIMARY KEY (profile_id, outcome_id, source_id)
68
+ )
69
+ """
70
+
71
+ _CREATE_REPAIR_STATE = """
72
+ CREATE TABLE IF NOT EXISTS source_quality_repair_state (
73
+ profile_id TEXT PRIMARY KEY,
74
+ last_rowid INTEGER NOT NULL DEFAULT 0,
75
+ last_settled_at TEXT NOT NULL DEFAULT '',
76
+ last_outcome_id TEXT NOT NULL DEFAULT '',
77
+ updated_at TEXT NOT NULL
78
+ )
79
+ """
80
+
81
+ _MAX_FACTS_PER_OUTCOME = 100
82
+ _MAX_SOURCES_PER_OUTCOME = 100
83
+ _PROVENANCE_QUERY_CHUNK = 500
84
+
85
+
86
+ class SourceQualityRepairUnavailable(RuntimeError):
87
+ """A repair read failed transiently and must not be interpreted as EOF."""
88
+
58
89
 
59
90
  def _utcnow_iso() -> str:
60
91
  """Return current UTC time as ISO-8601 string."""
@@ -85,9 +116,36 @@ class SourceQualityScorer:
85
116
  def _ensure_schema(self) -> None:
86
117
  conn = self._connect()
87
118
  try:
119
+ # Separate scorer instances can be constructed concurrently during
120
+ # first startup (background history repair + outcome settlement).
121
+ # Serialize the read/ALTER sequence at SQLite's transaction
122
+ # boundary so two processes cannot both observe a legacy column as
123
+ # missing and race into ``duplicate column name``.
124
+ conn.execute("BEGIN IMMEDIATE")
88
125
  conn.execute(_CREATE_TABLE)
89
126
  conn.execute(_CREATE_UNIQUE)
127
+ conn.execute(_CREATE_OBSERVATIONS)
128
+ conn.execute(_CREATE_REPAIR_STATE)
129
+ repair_columns = {
130
+ str(row["name"])
131
+ for row in conn.execute(
132
+ "PRAGMA table_info(source_quality_repair_state)"
133
+ ).fetchall()
134
+ }
135
+ if "last_settled_at" not in repair_columns:
136
+ conn.execute(
137
+ "ALTER TABLE source_quality_repair_state "
138
+ "ADD COLUMN last_settled_at TEXT NOT NULL DEFAULT ''"
139
+ )
140
+ if "last_outcome_id" not in repair_columns:
141
+ conn.execute(
142
+ "ALTER TABLE source_quality_repair_state "
143
+ "ADD COLUMN last_outcome_id TEXT NOT NULL DEFAULT ''"
144
+ )
90
145
  conn.commit()
146
+ except Exception:
147
+ conn.rollback()
148
+ raise
91
149
  finally:
92
150
  conn.close()
93
151
 
@@ -159,6 +217,150 @@ class SourceQualityScorer:
159
217
  finally:
160
218
  conn.close()
161
219
 
220
+ def record_reward(
221
+ self,
222
+ profile_id: str,
223
+ outcome_id: str,
224
+ source_ids: list[str],
225
+ reward: float,
226
+ ) -> int:
227
+ """Apply one fractional Beta observation per unique source.
228
+
229
+ The observation ledger makes retries idempotent. A reward of 0.8
230
+ contributes ``+0.8`` to alpha and ``+0.2`` to beta rather than
231
+ inventing a binary success label.
232
+ """
233
+ return self.record_rewards([
234
+ (profile_id, outcome_id, source_ids, reward),
235
+ ])
236
+
237
+ def record_rewards(
238
+ self,
239
+ observations: list[tuple[str, str, list[str], float]],
240
+ ) -> int:
241
+ """Batch bounded reward observations in one learning-DB transaction."""
242
+ inserted = 0
243
+ now = _utcnow_iso()
244
+ with self._lock:
245
+ conn = self._connect()
246
+ try:
247
+ conn.execute("BEGIN IMMEDIATE")
248
+ for profile_id, outcome_id, sources, raw_reward in observations:
249
+ if not profile_id or not outcome_id:
250
+ continue
251
+ numeric_reward = float(raw_reward)
252
+ if not math.isfinite(numeric_reward):
253
+ continue
254
+ reward = max(0.0, min(1.0, numeric_reward))
255
+ for source_id in sorted(set(sources))[
256
+ :_MAX_SOURCES_PER_OUTCOME
257
+ ]:
258
+ if not source_id:
259
+ continue
260
+ cursor = conn.execute(
261
+ "INSERT OR IGNORE INTO source_quality_observations "
262
+ "(profile_id, outcome_id, source_id, reward, observed_at) "
263
+ "VALUES (?, ?, ?, ?, ?)",
264
+ (profile_id, outcome_id, source_id, reward, now),
265
+ )
266
+ if cursor.rowcount != 1:
267
+ continue
268
+ conn.execute(
269
+ "INSERT OR IGNORE INTO source_quality "
270
+ "(profile_id, source_id, alpha, beta, updated_at) "
271
+ "VALUES (?, ?, ?, ?, ?)",
272
+ (profile_id, source_id, _ALPHA, _BETA, now),
273
+ )
274
+ conn.execute(
275
+ "UPDATE source_quality SET "
276
+ "alpha = alpha + ?, beta = beta + ?, updated_at = ? "
277
+ "WHERE profile_id = ? AND source_id = ?",
278
+ (
279
+ reward, 1.0 - reward, now,
280
+ profile_id, source_id,
281
+ ),
282
+ )
283
+ inserted += 1
284
+ conn.commit()
285
+ except Exception:
286
+ conn.rollback()
287
+ raise
288
+ finally:
289
+ conn.close()
290
+ return inserted
291
+
292
+ def get_repair_cursor(self, profile_id: str) -> int:
293
+ conn = self._connect()
294
+ try:
295
+ row = conn.execute(
296
+ "SELECT last_rowid FROM source_quality_repair_state "
297
+ "WHERE profile_id = ?",
298
+ (profile_id,),
299
+ ).fetchone()
300
+ return int(row["last_rowid"] or 0) if row else 0
301
+ finally:
302
+ conn.close()
303
+
304
+ def set_repair_cursor(self, profile_id: str, rowid: int) -> None:
305
+ now = _utcnow_iso()
306
+ with self._lock:
307
+ conn = self._connect()
308
+ try:
309
+ conn.execute(
310
+ "INSERT INTO source_quality_repair_state "
311
+ "(profile_id, last_rowid, updated_at) VALUES (?, ?, ?) "
312
+ "ON CONFLICT(profile_id) DO UPDATE SET "
313
+ "last_rowid = excluded.last_rowid, "
314
+ "updated_at = excluded.updated_at",
315
+ (profile_id, int(rowid), now),
316
+ )
317
+ conn.commit()
318
+ finally:
319
+ conn.close()
320
+
321
+ def get_repair_position(self, profile_id: str) -> tuple[str, str]:
322
+ """Return the durable settlement-order cursor for historical repair."""
323
+ conn = self._connect()
324
+ try:
325
+ row = conn.execute(
326
+ "SELECT last_settled_at, last_outcome_id "
327
+ "FROM source_quality_repair_state WHERE profile_id = ?",
328
+ (profile_id,),
329
+ ).fetchone()
330
+ if row is None:
331
+ return ("", "")
332
+ return (
333
+ str(row["last_settled_at"] or ""),
334
+ str(row["last_outcome_id"] or ""),
335
+ )
336
+ finally:
337
+ conn.close()
338
+
339
+ def set_repair_position(
340
+ self,
341
+ profile_id: str,
342
+ settled_at: str,
343
+ outcome_id: str,
344
+ ) -> None:
345
+ """Advance repair by settlement order, not immutable insertion rowid."""
346
+ now = _utcnow_iso()
347
+ with self._lock:
348
+ conn = self._connect()
349
+ try:
350
+ conn.execute(
351
+ "INSERT INTO source_quality_repair_state "
352
+ "(profile_id,last_rowid,last_settled_at,last_outcome_id,"
353
+ "updated_at) VALUES (?,0,?,?,?) "
354
+ "ON CONFLICT(profile_id) DO UPDATE SET "
355
+ "last_settled_at=excluded.last_settled_at,"
356
+ "last_outcome_id=excluded.last_outcome_id,"
357
+ "updated_at=excluded.updated_at",
358
+ (profile_id, str(settled_at), str(outcome_id), now),
359
+ )
360
+ conn.commit()
361
+ finally:
362
+ conn.close()
363
+
162
364
  # ------------------------------------------------------------------
163
365
  # Public API: read quality
164
366
  # ------------------------------------------------------------------
@@ -301,3 +503,323 @@ class SourceQualityScorer:
301
503
  return result
302
504
  finally:
303
505
  conn.close()
506
+
507
+
508
+ def _source_key(row: sqlite3.Row) -> str:
509
+ source_type = str(row["source_type"] or "").strip()[:100]
510
+ actor = str(row["created_by"] or "").strip()
511
+ operation_or_legacy_source = str(row["source_id"] or "").strip()
512
+ # Canonical ingestion stores its unique operation UUID in source_id for
513
+ # lineage/idempotency and the stable trusted client in created_by. Quality
514
+ # must aggregate by the stable actor; operation IDs would create one
515
+ # single-observation "source" per remember call. Legacy provenance often
516
+ # has no actor, so retain its established source_id fallback.
517
+ identifier = (
518
+ actor
519
+ if actor and actor.lower() != "unknown"
520
+ else operation_or_legacy_source
521
+ )[:100]
522
+ if source_type and identifier:
523
+ return f"{source_type}:{identifier}"
524
+ if identifier:
525
+ return f"source:{identifier}"
526
+ return source_type
527
+
528
+
529
+ def _load_source_map(
530
+ memory_db_path: Path,
531
+ profile_id: str,
532
+ fact_ids: list[str],
533
+ *,
534
+ strict: bool = False,
535
+ ) -> dict[str, set[str]]:
536
+ """Read only the provenance needed by this bounded reward batch."""
537
+ unique_facts = list(dict.fromkeys(str(fid) for fid in fact_ids if fid))
538
+ if not unique_facts or not Path(memory_db_path).exists():
539
+ return {}
540
+ result: dict[str, set[str]] = {}
541
+ try:
542
+ conn = sqlite3.connect(
543
+ f"file:{Path(memory_db_path)}?mode=ro", uri=True, timeout=1.0,
544
+ )
545
+ conn.row_factory = sqlite3.Row
546
+ try:
547
+ columns = {
548
+ str(row["name"])
549
+ for row in conn.execute("PRAGMA table_info(provenance)")
550
+ }
551
+ required = {
552
+ "profile_id", "fact_id", "source_type",
553
+ "source_id", "created_by",
554
+ }
555
+ if not required.issubset(columns):
556
+ return {}
557
+ for start in range(0, len(unique_facts), _PROVENANCE_QUERY_CHUNK):
558
+ chunk = unique_facts[start:start + _PROVENANCE_QUERY_CHUNK]
559
+ placeholders = ",".join("?" for _ in chunk)
560
+ rows = conn.execute(
561
+ "SELECT DISTINCT fact_id, source_type, source_id, created_by "
562
+ "FROM provenance WHERE profile_id = ? "
563
+ f"AND fact_id IN ({placeholders})",
564
+ (profile_id, *chunk),
565
+ ).fetchall()
566
+ for row in rows:
567
+ key = _source_key(row)
568
+ if key:
569
+ result.setdefault(str(row["fact_id"]), set()).add(key)
570
+ finally:
571
+ conn.close()
572
+ except sqlite3.Error as exc:
573
+ logger.debug("source provenance unavailable: %s", exc)
574
+ if strict:
575
+ raise SourceQualityRepairUnavailable(
576
+ "source provenance temporarily unavailable",
577
+ ) from exc
578
+ return {}
579
+ return result
580
+
581
+
582
+ def update_source_quality_for_reward(
583
+ *,
584
+ memory_db_path: Path,
585
+ learning_db_path: Path,
586
+ profile_id: str,
587
+ outcome_id: str,
588
+ fact_ids: list[str],
589
+ reward: float,
590
+ ) -> int:
591
+ """Fail-soft online bridge from a finalized reward to real provenance."""
592
+ return update_source_quality_for_reward_batch(
593
+ memory_db_path=memory_db_path,
594
+ learning_db_path=learning_db_path,
595
+ rewards=[(profile_id, outcome_id, fact_ids, reward)],
596
+ )
597
+
598
+
599
+ def update_source_quality_for_reward_batch(
600
+ *,
601
+ memory_db_path: Path,
602
+ learning_db_path: Path,
603
+ rewards: list[tuple[str, str, list[str], float]],
604
+ ) -> int:
605
+ """Fail-soft bounded bridge for worker-finalized reward batches."""
606
+ try:
607
+ bounded_rewards = rewards[:1000]
608
+ normalized = []
609
+ for profile_id, outcome_id, fact_ids, reward in bounded_rewards:
610
+ bounded_facts = list(dict.fromkeys(fact_ids))[
611
+ :_MAX_FACTS_PER_OUTCOME
612
+ ]
613
+ normalized.append((
614
+ profile_id, outcome_id, bounded_facts, float(reward),
615
+ ))
616
+ if not normalized:
617
+ return 0
618
+ # Each reward batch is profile-homogeneous in current callers. Split
619
+ # defensively so provenance can never cross a profile boundary.
620
+ by_profile: dict[str, list[tuple[str, list[str], float]]] = {}
621
+ for profile_id, outcome_id, fact_ids, reward in normalized:
622
+ by_profile.setdefault(profile_id, []).append(
623
+ (outcome_id, fact_ids, reward),
624
+ )
625
+ scorer = SourceQualityScorer(Path(learning_db_path))
626
+ observations = []
627
+ for profile_id, profile_rewards in by_profile.items():
628
+ profile_facts = [
629
+ fact_id
630
+ for _, fact_ids, _ in profile_rewards
631
+ for fact_id in fact_ids
632
+ ]
633
+ source_map = _load_source_map(
634
+ Path(memory_db_path), profile_id, profile_facts,
635
+ )
636
+ for outcome_id, fact_ids, reward in profile_rewards:
637
+ source_ids = sorted({
638
+ source
639
+ for fact_id in fact_ids
640
+ for source in source_map.get(fact_id, set())
641
+ })[:_MAX_SOURCES_PER_OUTCOME]
642
+ observations.append((
643
+ profile_id, outcome_id, source_ids, reward,
644
+ ))
645
+ return scorer.record_rewards(observations)
646
+ except (OSError, sqlite3.Error, TypeError, ValueError) as exc:
647
+ logger.debug("source-quality reward update skipped: %s", exc)
648
+ return 0
649
+
650
+
651
+ def enumerate_source_quality_repair_profiles(
652
+ memory_db_path: Path,
653
+ ) -> list[str]:
654
+ """Return profiles with settled numeric outcomes eligible for repair."""
655
+ path = Path(memory_db_path)
656
+ if not path.exists():
657
+ return []
658
+ try:
659
+ conn = sqlite3.connect(
660
+ f"file:{path}?mode=ro", uri=True, timeout=1.0,
661
+ )
662
+ conn.row_factory = sqlite3.Row
663
+ try:
664
+ columns = {
665
+ str(row["name"])
666
+ for row in conn.execute("PRAGMA table_info(action_outcomes)")
667
+ }
668
+ required = {"profile_id", "reward", "settled"}
669
+ if not required.issubset(columns):
670
+ return []
671
+ rows = conn.execute(
672
+ "SELECT DISTINCT profile_id FROM action_outcomes "
673
+ "WHERE settled = 1 AND reward IS NOT NULL "
674
+ "AND typeof(reward) IN ('integer', 'real') "
675
+ "AND profile_id IS NOT NULL AND profile_id != '' "
676
+ "ORDER BY profile_id ASC",
677
+ ).fetchall()
678
+ return [str(row["profile_id"]) for row in rows]
679
+ finally:
680
+ conn.close()
681
+ except sqlite3.Error as exc:
682
+ logger.debug("source-quality profile enumeration unavailable: %s", exc)
683
+ raise SourceQualityRepairUnavailable(
684
+ "repair profile enumeration temporarily unavailable",
685
+ ) from exc
686
+
687
+
688
+ def _parse_repair_rows(
689
+ rows: list[sqlite3.Row],
690
+ ) -> tuple[list[tuple[sqlite3.Row, list[str]]], list[str]]:
691
+ parsed: list[tuple[sqlite3.Row, list[str]]] = []
692
+ all_facts: list[str] = []
693
+ for row in rows:
694
+ try:
695
+ value = json.loads(str(row["fact_ids_json"] or "[]"))
696
+ fact_ids = (
697
+ [str(item) for item in value if item][:_MAX_FACTS_PER_OUTCOME]
698
+ if isinstance(value, list) else []
699
+ )
700
+ except (TypeError, ValueError, json.JSONDecodeError):
701
+ fact_ids = []
702
+ parsed.append((row, fact_ids))
703
+ all_facts.extend(fact_ids)
704
+ return parsed, all_facts
705
+
706
+
707
+ def _repair_observations(
708
+ profile_id: str,
709
+ parsed: list[tuple[sqlite3.Row, list[str]]],
710
+ source_map: dict[str, set[str]],
711
+ ) -> list[tuple[str, str, list[str], float]]:
712
+ observations = []
713
+ for row, fact_ids in parsed:
714
+ sources = sorted({
715
+ source for fact_id in fact_ids
716
+ for source in source_map.get(fact_id, set())
717
+ })[:_MAX_SOURCES_PER_OUTCOME]
718
+ observations.append((
719
+ profile_id, str(row["outcome_id"]), sources, float(row["reward"]),
720
+ ))
721
+ return observations
722
+
723
+
724
+ def repair_historical_source_quality(
725
+ memory_db_path: Path,
726
+ learning_db_path: Path,
727
+ profile_id: str,
728
+ *,
729
+ batch_size: int = 250,
730
+ max_batches: int = 4,
731
+ ) -> dict[str, int | bool]:
732
+ """Explicit, resumable historical repair; never invoked by API handlers.
733
+
734
+ Work is capped to 1,000 outcomes per call by default. The cursor advances
735
+ only after the idempotent observation ledger commits, so interruption can
736
+ replay safely without double-counting.
737
+ """
738
+ safe_batch = max(1, min(1000, int(batch_size)))
739
+ safe_batches = max(1, min(100, int(max_batches)))
740
+ scorer = SourceQualityScorer(Path(learning_db_path))
741
+ scanned = 0
742
+ observations = 0
743
+ complete = False
744
+ for _ in range(safe_batches):
745
+ settled_cursor, outcome_cursor = scorer.get_repair_position(profile_id)
746
+ rows = _load_reward_repair_batch(
747
+ Path(memory_db_path),
748
+ profile_id,
749
+ settled_cursor,
750
+ outcome_cursor,
751
+ safe_batch,
752
+ )
753
+ if not rows:
754
+ complete = True
755
+ break
756
+ parsed, all_facts = _parse_repair_rows(rows)
757
+ source_map = _load_source_map(
758
+ Path(memory_db_path), profile_id, all_facts, strict=True,
759
+ )
760
+ batch_observations = _repair_observations(
761
+ profile_id, parsed, source_map,
762
+ )
763
+ observations += scorer.record_rewards(batch_observations)
764
+ scanned += len(rows)
765
+ scorer.set_repair_position(
766
+ profile_id,
767
+ str(rows[-1]["settled_key"] or ""),
768
+ str(rows[-1]["outcome_id"]),
769
+ )
770
+ return {
771
+ "scanned": scanned,
772
+ "observations": observations,
773
+ "complete": complete,
774
+ }
775
+
776
+
777
+ def _load_reward_repair_batch(
778
+ memory_db_path: Path,
779
+ profile_id: str,
780
+ after_settled_at: str,
781
+ after_outcome_id: str,
782
+ limit: int,
783
+ ) -> list[sqlite3.Row]:
784
+ if not memory_db_path.exists():
785
+ return []
786
+ try:
787
+ conn = sqlite3.connect(
788
+ f"file:{memory_db_path}?mode=ro", uri=True, timeout=1.0,
789
+ )
790
+ conn.row_factory = sqlite3.Row
791
+ try:
792
+ columns = {
793
+ str(row["name"])
794
+ for row in conn.execute("PRAGMA table_info(action_outcomes)")
795
+ }
796
+ required = {
797
+ "outcome_id", "profile_id", "fact_ids_json",
798
+ "reward", "settled",
799
+ }
800
+ if not required.issubset(columns):
801
+ return []
802
+ return conn.execute(
803
+ "SELECT outcome_id, fact_ids_json, reward, "
804
+ "COALESCE(settled_at, '') AS settled_key "
805
+ "FROM action_outcomes WHERE profile_id = ? "
806
+ "AND settled = 1 AND reward IS NOT NULL "
807
+ "AND typeof(reward) IN ('integer', 'real') "
808
+ "AND (COALESCE(settled_at, '') > ? OR "
809
+ "(COALESCE(settled_at, '') = ? AND outcome_id > ?)) "
810
+ "ORDER BY COALESCE(settled_at, '') ASC, outcome_id ASC LIMIT ?",
811
+ (
812
+ profile_id,
813
+ str(after_settled_at),
814
+ str(after_settled_at),
815
+ str(after_outcome_id),
816
+ int(limit),
817
+ ),
818
+ ).fetchall()
819
+ finally:
820
+ conn.close()
821
+ except sqlite3.Error as exc:
822
+ logger.debug("source-quality repair unavailable: %s", exc)
823
+ raise SourceQualityRepairUnavailable(
824
+ "repair batch temporarily unavailable",
825
+ ) from exc
@@ -156,8 +156,10 @@ class SLMMemoryLedger:
156
156
  class _EngineLedgerStore:
157
157
  """Minimal profile-scoped store over a SuperLocalMemory engine.
158
158
 
159
- Uses ``MemoryEngine.store`` for writes and direct, escaped, profile-scoped
160
- reads the same contract the LangChain/LlamaIndex adapters depend on.
159
+ Uses the engine's non-blocking write-through path when available and
160
+ direct, escaped, profile-scoped reads. The ``store`` fallback preserves
161
+ compatibility with lightweight adapter/test engines that predate
162
+ ``store_fast``.
161
163
  """
162
164
 
163
165
  def __init__(self, engine: Any, *, owns_engine: bool = True) -> None:
@@ -169,9 +171,27 @@ class _EngineLedgerStore:
169
171
  self._owns_engine = owns_engine
170
172
 
171
173
  def add(self, content: str, *, session_id: str, metadata: dict) -> None:
172
- # The parent memory row is what the ledger needs; fact extraction is a
173
- # bonus, so (unlike chat history) an empty fact set is not an error.
174
- self._engine.store(content, session_id=session_id, metadata=metadata)
174
+ # A loop ledger needs the durable parent row and immediate lexical
175
+ # recall, not synchronous embeddings/entity/graph enrichment. Loading
176
+ # the heavyweight embedding worker for every bounded-loop lap can stall
177
+ # the loop for the full worker timeout and consume ~1 GB for metadata.
178
+ # The write-through path persists the same session-scoped content in
179
+ # milliseconds; ordinary background enrichment can still promote it.
180
+ fast_metadata = {**metadata, "session_id": session_id}
181
+ store_fast = getattr(self._engine, "store_fast", None)
182
+ if callable(store_fast):
183
+ store_fast(
184
+ content,
185
+ metadata=fast_metadata,
186
+ index_external=False,
187
+ )
188
+ return
189
+
190
+ self._engine.store(
191
+ content,
192
+ session_id=session_id,
193
+ metadata=metadata,
194
+ )
175
195
 
176
196
  def list_session(self, session_id: str) -> list[dict]:
177
197
  # Cap the read: a bounded-loop run is capped at max_iterations laps, so
@@ -203,7 +203,7 @@ class _FilteredServer:
203
203
  """
204
204
  __slots__ = ("_server", "_allowed")
205
205
 
206
- def __init__(self, real_server: FastMCP, allowed: frozenset[str]) -> None:
206
+ def __init__(self, real_server: SLMFastMCP, allowed: frozenset[str]) -> None:
207
207
  self._server = real_server
208
208
  self._allowed = allowed
209
209
 
@@ -265,29 +265,20 @@ from superlocalmemory.mcp.tools_loops import register_loop_tools
265
265
  register_loop_tools(_target, get_engine) # v3.8.0: bounded-loop tools (CLI+command+MCP)
266
266
 
267
267
 
268
- # V3.3.21: Eager engine warmup start initializing BEFORE first tool call.
269
- # The MCP server process starts when the IDE launches. Previously, the engine
270
- # was lazy-loaded on first tool call 23s cold start for the user.
271
- # Now: engine starts warming in a background thread immediately. By the time
272
- # the first tool call arrives (1-2s later), the engine is already warm.
273
- # This applies to ALL IDEs: Claude Code, Cursor, Antigravity, Gemini CLI, etc.
268
+ # Keep stdio MCP processes thin until a tool truly needs a local LIGHT engine.
269
+ # Every open IDE/task owns a stdio process; eagerly opening memory.db in all of
270
+ # them multiplied RAM and SQLite writers on machines with many long-lived
271
+ # sessions. The shared daemon owns model warmup and common remember/recall work.
274
272
  def _eager_warmup() -> None:
275
- """Pre-warm LIGHT engine + ensure daemon is running + auto-register mesh.
273
+ """Ensure the shared daemon is running without opening a per-client engine.
276
274
 
277
- LIGHT engine init is cheap (DB only, ~100 ms). The real reason this
278
- stays in a background thread is the follow-on side effects
279
- (``ensure_daemon``, ``auto_register_mesh``) which do I/O.
275
+ Mesh registration is intentionally lazy: a local stdio session is not a
276
+ remote peer, and heartbeat writes should begin only after a mesh tool is
277
+ actually used.
280
278
  """
281
- import logging
282
279
  _logger = logging.getLogger(__name__)
283
- try:
284
- get_engine()
285
- _logger.info("MCP engine pre-warmed successfully")
286
- except Exception as exc:
287
- _logger.warning("MCP engine pre-warmup failed: %s", exc)
288
280
 
289
- # Measurement / test harnesses set this to skip daemon-start and
290
- # mesh-register. The LIGHT engine init above still runs.
281
+ # Measurement / test harnesses set this to skip daemon-start.
291
282
  if _os.environ.get("SLM_DISABLE_WARMUP_SIDE_EFFECTS") == "1":
292
283
  return
293
284
 
@@ -300,22 +291,12 @@ def _eager_warmup() -> None:
300
291
  except Exception as exc:
301
292
  _logger.warning("Daemon auto-start failed: %s", exc)
302
293
 
303
- # V3.4.6: Auto-register this MCP session as a mesh peer immediately.
304
- # Previously, registration was lazy (only on first mesh tool call).
305
- # Now every Claude session appears on the mesh from startup.
306
- try:
307
- from superlocalmemory.mcp.tools_mesh import auto_register_mesh
308
- auto_register_mesh()
309
- _logger.info("Mesh peer auto-registered at startup")
310
- except Exception as exc:
311
- _logger.warning("Mesh auto-register failed: %s", exc)
312
-
313
294
  import threading
314
295
 
315
296
  # v3.6.7: Suppress standalone-process behaviours when the MCP server is
316
297
  # imported inside the daemon (SLM_MCP_EMBEDDED=1). Three threads are safe
317
298
  # to run in a dedicated `slm mcp` subprocess but harmful inside the daemon:
318
- # mcp-warmup — creates a LIGHT engine duplicate; daemon has a FULL one.
299
+ # mcp-warmup — ensures the shared daemon only; never creates an engine.
319
300
  # parent-watchdog — calls os._exit(0) if its parent IDE quits, which would
320
301
  # kill the daemon along with it.
321
302
  # stdin-eof-monitor — monitors stdin pipe; meaningless inside the daemon.