superlocalmemory 3.8.0 → 3.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/README.md +32 -120
- package/package.json +9 -2
- package/plugin/.claude-plugin/plugin.json +1 -2
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +2 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +134 -7
- package/src/superlocalmemory/cli/daemon.py +7 -0
- package/src/superlocalmemory/cli/loop_cmd.py +2 -7
- package/src/superlocalmemory/cli/main.py +10 -4
- package/src/superlocalmemory/cli/version_banner.py +17 -3
- package/src/superlocalmemory/core/backend_orchestrator.py +18 -16
- package/src/superlocalmemory/core/embedding_worker.py +4 -5
- package/src/superlocalmemory/core/embeddings.py +119 -45
- package/src/superlocalmemory/core/engine.py +24 -21
- package/src/superlocalmemory/core/engine_ingestion.py +332 -45
- package/src/superlocalmemory/core/ingestion_command.py +134 -25
- package/src/superlocalmemory/core/injection.py +12 -7
- package/src/superlocalmemory/core/maintenance_scheduler.py +17 -7
- package/src/superlocalmemory/core/recall_pipeline.py +5 -3
- package/src/superlocalmemory/core/store_pipeline.py +153 -20
- package/src/superlocalmemory/hooks/hook_handlers.py +6 -1
- package/src/superlocalmemory/hooks/portable_kit.py +34 -2
- package/src/superlocalmemory/learning/model_rollback.py +3 -0
- package/src/superlocalmemory/learning/ranker_retrain_online.py +2 -0
- package/src/superlocalmemory/learning/reward.py +50 -0
- package/src/superlocalmemory/learning/source_quality.py +523 -1
- package/src/superlocalmemory/loops/ledger.py +25 -5
- package/src/superlocalmemory/mcp/server.py +11 -30
- package/src/superlocalmemory/mcp/tools_core.py +4 -3
- package/src/superlocalmemory/mcp/tools_learning.py +2 -2
- package/src/superlocalmemory/retrieval/engine.py +53 -21
- package/src/superlocalmemory/retrieval/reranker.py +3 -4
- package/src/superlocalmemory/server/config_file.py +90 -0
- package/src/superlocalmemory/server/origin.py +50 -0
- package/src/superlocalmemory/server/routes/backup.py +293 -70
- package/src/superlocalmemory/server/routes/behavioral.py +336 -59
- package/src/superlocalmemory/server/routes/brain.py +57 -16
- package/src/superlocalmemory/server/routes/config_api.py +84 -82
- package/src/superlocalmemory/server/routes/entity.py +100 -23
- package/src/superlocalmemory/server/routes/evolution.py +103 -100
- package/src/superlocalmemory/server/routes/learning.py +276 -105
- package/src/superlocalmemory/server/routes/learning_telemetry.py +153 -0
- package/src/superlocalmemory/server/routes/mesh.py +121 -32
- package/src/superlocalmemory/server/routes/ratelimit.py +33 -25
- package/src/superlocalmemory/server/routes/stats.py +93 -155
- package/src/superlocalmemory/server/routes/token.py +3 -13
- package/src/superlocalmemory/server/routes/v3_api.py +64 -20
- package/src/superlocalmemory/server/unified_daemon.py +467 -40
- package/src/superlocalmemory/storage/migration_runner.py +79 -1
- package/src/superlocalmemory/storage/migrations/M010_evolution_config.py +5 -0
- package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +270 -0
- package/src/superlocalmemory/storage/migrations/M029_behavioral_history_indexes.py +137 -0
- package/src/superlocalmemory/storage/migrations/M030_entity_explorer_indexes.py +93 -0
- package/src/superlocalmemory/storage/migrations/__init__.py +4 -0
- package/src/superlocalmemory/storage/schema.py +49 -1
- package/src/superlocalmemory/storage/schema_v32.py +2 -0
- package/src/superlocalmemory/storage/schema_v347.py +4 -0
- package/src/superlocalmemory/ui/index.html +3 -6
- package/src/superlocalmemory/ui/js/core.js +52 -9
- package/src/superlocalmemory/ui/js/dashboard.js +169 -82
- package/src/superlocalmemory/ui/js/od-backup.js +156 -65
- package/src/superlocalmemory/ui/js/od-brain.js +88 -51
- package/src/superlocalmemory/ui/js/od-entities.js +22 -22
- package/src/superlocalmemory/ui/js/od-graph.js +11 -4
- package/src/superlocalmemory/ui/js/od-memories.js +47 -5
- package/src/superlocalmemory/ui/js/od-mesh.js +23 -9
- package/src/superlocalmemory/ui/js/od-settings.js +113 -59
- package/src/superlocalmemory/ui/js/od-shell.js +249 -33
- package/src/superlocalmemory/ui/js/od-skills.js +44 -17
- package/src/superlocalmemory/ui/js/settings.js +15 -1
- package/plugin-src/.mcp.json +0 -12
- package/plugin-src/agents/slm-governance-advisor.md +0 -80
- package/plugin-src/agents/slm-loop-runner.md +0 -71
- package/plugin-src/agents/slm-memory-advisor.md +0 -49
- package/plugin-src/agents/slm-optimize-advisor.md +0 -44
- package/plugin-src/commands/slm-loop.md +0 -31
- package/plugin-src/hooks/.gitkeep +0 -0
- package/plugin-src/hooks/hooks.json +0 -102
- package/plugin-src/manifest.json +0 -30
- package/plugin-src/requirements.txt +0 -1
- package/plugin-src/rules/CLAUDE.md.fragment +0 -44
- package/plugin-src/scripts/ensure-venv.bat +0 -122
- package/plugin-src/scripts/ensure-venv.sh +0 -105
- package/plugin-src/scripts/slm-launch +0 -62
- package/plugin-src/scripts/slm-launch.bat +0 -23
- package/plugin-src/settings.json +0 -25
- package/plugin-src/skills/slm-governance/SKILL.md +0 -248
- package/plugin-src/skills/slm-loop/SKILL.md +0 -99
- package/plugin-src/skills/slm-mesh/SKILL.md +0 -282
- package/plugin-src/skills/slm-profile/SKILL.md +0 -148
- package/plugin-src/skills/slm-scope/SKILL.md +0 -176
|
@@ -23,12 +23,14 @@ Storage:
|
|
|
23
23
|
|
|
24
24
|
from __future__ import annotations
|
|
25
25
|
|
|
26
|
+
import json
|
|
26
27
|
import logging
|
|
28
|
+
import math
|
|
27
29
|
import sqlite3
|
|
28
30
|
import threading
|
|
29
31
|
from datetime import datetime, timezone
|
|
30
32
|
from pathlib import Path
|
|
31
|
-
from typing import Any, Dict
|
|
33
|
+
from typing import Any, Dict
|
|
32
34
|
|
|
33
35
|
logger = logging.getLogger("superlocalmemory.learning.source_quality")
|
|
34
36
|
|
|
@@ -55,6 +57,35 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_sq_profile_source
|
|
|
55
57
|
ON source_quality (profile_id, source_id)
|
|
56
58
|
"""
|
|
57
59
|
|
|
60
|
+
_CREATE_OBSERVATIONS = """
|
|
61
|
+
CREATE TABLE IF NOT EXISTS source_quality_observations (
|
|
62
|
+
profile_id TEXT NOT NULL,
|
|
63
|
+
outcome_id TEXT NOT NULL,
|
|
64
|
+
source_id TEXT NOT NULL,
|
|
65
|
+
reward REAL NOT NULL,
|
|
66
|
+
observed_at TEXT NOT NULL,
|
|
67
|
+
PRIMARY KEY (profile_id, outcome_id, source_id)
|
|
68
|
+
)
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
_CREATE_REPAIR_STATE = """
|
|
72
|
+
CREATE TABLE IF NOT EXISTS source_quality_repair_state (
|
|
73
|
+
profile_id TEXT PRIMARY KEY,
|
|
74
|
+
last_rowid INTEGER NOT NULL DEFAULT 0,
|
|
75
|
+
last_settled_at TEXT NOT NULL DEFAULT '',
|
|
76
|
+
last_outcome_id TEXT NOT NULL DEFAULT '',
|
|
77
|
+
updated_at TEXT NOT NULL
|
|
78
|
+
)
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
_MAX_FACTS_PER_OUTCOME = 100
|
|
82
|
+
_MAX_SOURCES_PER_OUTCOME = 100
|
|
83
|
+
_PROVENANCE_QUERY_CHUNK = 500
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class SourceQualityRepairUnavailable(RuntimeError):
|
|
87
|
+
"""A repair read failed transiently and must not be interpreted as EOF."""
|
|
88
|
+
|
|
58
89
|
|
|
59
90
|
def _utcnow_iso() -> str:
|
|
60
91
|
"""Return current UTC time as ISO-8601 string."""
|
|
@@ -85,9 +116,36 @@ class SourceQualityScorer:
|
|
|
85
116
|
def _ensure_schema(self) -> None:
|
|
86
117
|
conn = self._connect()
|
|
87
118
|
try:
|
|
119
|
+
# Separate scorer instances can be constructed concurrently during
|
|
120
|
+
# first startup (background history repair + outcome settlement).
|
|
121
|
+
# Serialize the read/ALTER sequence at SQLite's transaction
|
|
122
|
+
# boundary so two processes cannot both observe a legacy column as
|
|
123
|
+
# missing and race into ``duplicate column name``.
|
|
124
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
88
125
|
conn.execute(_CREATE_TABLE)
|
|
89
126
|
conn.execute(_CREATE_UNIQUE)
|
|
127
|
+
conn.execute(_CREATE_OBSERVATIONS)
|
|
128
|
+
conn.execute(_CREATE_REPAIR_STATE)
|
|
129
|
+
repair_columns = {
|
|
130
|
+
str(row["name"])
|
|
131
|
+
for row in conn.execute(
|
|
132
|
+
"PRAGMA table_info(source_quality_repair_state)"
|
|
133
|
+
).fetchall()
|
|
134
|
+
}
|
|
135
|
+
if "last_settled_at" not in repair_columns:
|
|
136
|
+
conn.execute(
|
|
137
|
+
"ALTER TABLE source_quality_repair_state "
|
|
138
|
+
"ADD COLUMN last_settled_at TEXT NOT NULL DEFAULT ''"
|
|
139
|
+
)
|
|
140
|
+
if "last_outcome_id" not in repair_columns:
|
|
141
|
+
conn.execute(
|
|
142
|
+
"ALTER TABLE source_quality_repair_state "
|
|
143
|
+
"ADD COLUMN last_outcome_id TEXT NOT NULL DEFAULT ''"
|
|
144
|
+
)
|
|
90
145
|
conn.commit()
|
|
146
|
+
except Exception:
|
|
147
|
+
conn.rollback()
|
|
148
|
+
raise
|
|
91
149
|
finally:
|
|
92
150
|
conn.close()
|
|
93
151
|
|
|
@@ -159,6 +217,150 @@ class SourceQualityScorer:
|
|
|
159
217
|
finally:
|
|
160
218
|
conn.close()
|
|
161
219
|
|
|
220
|
+
def record_reward(
|
|
221
|
+
self,
|
|
222
|
+
profile_id: str,
|
|
223
|
+
outcome_id: str,
|
|
224
|
+
source_ids: list[str],
|
|
225
|
+
reward: float,
|
|
226
|
+
) -> int:
|
|
227
|
+
"""Apply one fractional Beta observation per unique source.
|
|
228
|
+
|
|
229
|
+
The observation ledger makes retries idempotent. A reward of 0.8
|
|
230
|
+
contributes ``+0.8`` to alpha and ``+0.2`` to beta rather than
|
|
231
|
+
inventing a binary success label.
|
|
232
|
+
"""
|
|
233
|
+
return self.record_rewards([
|
|
234
|
+
(profile_id, outcome_id, source_ids, reward),
|
|
235
|
+
])
|
|
236
|
+
|
|
237
|
+
def record_rewards(
|
|
238
|
+
self,
|
|
239
|
+
observations: list[tuple[str, str, list[str], float]],
|
|
240
|
+
) -> int:
|
|
241
|
+
"""Batch bounded reward observations in one learning-DB transaction."""
|
|
242
|
+
inserted = 0
|
|
243
|
+
now = _utcnow_iso()
|
|
244
|
+
with self._lock:
|
|
245
|
+
conn = self._connect()
|
|
246
|
+
try:
|
|
247
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
248
|
+
for profile_id, outcome_id, sources, raw_reward in observations:
|
|
249
|
+
if not profile_id or not outcome_id:
|
|
250
|
+
continue
|
|
251
|
+
numeric_reward = float(raw_reward)
|
|
252
|
+
if not math.isfinite(numeric_reward):
|
|
253
|
+
continue
|
|
254
|
+
reward = max(0.0, min(1.0, numeric_reward))
|
|
255
|
+
for source_id in sorted(set(sources))[
|
|
256
|
+
:_MAX_SOURCES_PER_OUTCOME
|
|
257
|
+
]:
|
|
258
|
+
if not source_id:
|
|
259
|
+
continue
|
|
260
|
+
cursor = conn.execute(
|
|
261
|
+
"INSERT OR IGNORE INTO source_quality_observations "
|
|
262
|
+
"(profile_id, outcome_id, source_id, reward, observed_at) "
|
|
263
|
+
"VALUES (?, ?, ?, ?, ?)",
|
|
264
|
+
(profile_id, outcome_id, source_id, reward, now),
|
|
265
|
+
)
|
|
266
|
+
if cursor.rowcount != 1:
|
|
267
|
+
continue
|
|
268
|
+
conn.execute(
|
|
269
|
+
"INSERT OR IGNORE INTO source_quality "
|
|
270
|
+
"(profile_id, source_id, alpha, beta, updated_at) "
|
|
271
|
+
"VALUES (?, ?, ?, ?, ?)",
|
|
272
|
+
(profile_id, source_id, _ALPHA, _BETA, now),
|
|
273
|
+
)
|
|
274
|
+
conn.execute(
|
|
275
|
+
"UPDATE source_quality SET "
|
|
276
|
+
"alpha = alpha + ?, beta = beta + ?, updated_at = ? "
|
|
277
|
+
"WHERE profile_id = ? AND source_id = ?",
|
|
278
|
+
(
|
|
279
|
+
reward, 1.0 - reward, now,
|
|
280
|
+
profile_id, source_id,
|
|
281
|
+
),
|
|
282
|
+
)
|
|
283
|
+
inserted += 1
|
|
284
|
+
conn.commit()
|
|
285
|
+
except Exception:
|
|
286
|
+
conn.rollback()
|
|
287
|
+
raise
|
|
288
|
+
finally:
|
|
289
|
+
conn.close()
|
|
290
|
+
return inserted
|
|
291
|
+
|
|
292
|
+
def get_repair_cursor(self, profile_id: str) -> int:
|
|
293
|
+
conn = self._connect()
|
|
294
|
+
try:
|
|
295
|
+
row = conn.execute(
|
|
296
|
+
"SELECT last_rowid FROM source_quality_repair_state "
|
|
297
|
+
"WHERE profile_id = ?",
|
|
298
|
+
(profile_id,),
|
|
299
|
+
).fetchone()
|
|
300
|
+
return int(row["last_rowid"] or 0) if row else 0
|
|
301
|
+
finally:
|
|
302
|
+
conn.close()
|
|
303
|
+
|
|
304
|
+
def set_repair_cursor(self, profile_id: str, rowid: int) -> None:
|
|
305
|
+
now = _utcnow_iso()
|
|
306
|
+
with self._lock:
|
|
307
|
+
conn = self._connect()
|
|
308
|
+
try:
|
|
309
|
+
conn.execute(
|
|
310
|
+
"INSERT INTO source_quality_repair_state "
|
|
311
|
+
"(profile_id, last_rowid, updated_at) VALUES (?, ?, ?) "
|
|
312
|
+
"ON CONFLICT(profile_id) DO UPDATE SET "
|
|
313
|
+
"last_rowid = excluded.last_rowid, "
|
|
314
|
+
"updated_at = excluded.updated_at",
|
|
315
|
+
(profile_id, int(rowid), now),
|
|
316
|
+
)
|
|
317
|
+
conn.commit()
|
|
318
|
+
finally:
|
|
319
|
+
conn.close()
|
|
320
|
+
|
|
321
|
+
def get_repair_position(self, profile_id: str) -> tuple[str, str]:
|
|
322
|
+
"""Return the durable settlement-order cursor for historical repair."""
|
|
323
|
+
conn = self._connect()
|
|
324
|
+
try:
|
|
325
|
+
row = conn.execute(
|
|
326
|
+
"SELECT last_settled_at, last_outcome_id "
|
|
327
|
+
"FROM source_quality_repair_state WHERE profile_id = ?",
|
|
328
|
+
(profile_id,),
|
|
329
|
+
).fetchone()
|
|
330
|
+
if row is None:
|
|
331
|
+
return ("", "")
|
|
332
|
+
return (
|
|
333
|
+
str(row["last_settled_at"] or ""),
|
|
334
|
+
str(row["last_outcome_id"] or ""),
|
|
335
|
+
)
|
|
336
|
+
finally:
|
|
337
|
+
conn.close()
|
|
338
|
+
|
|
339
|
+
def set_repair_position(
|
|
340
|
+
self,
|
|
341
|
+
profile_id: str,
|
|
342
|
+
settled_at: str,
|
|
343
|
+
outcome_id: str,
|
|
344
|
+
) -> None:
|
|
345
|
+
"""Advance repair by settlement order, not immutable insertion rowid."""
|
|
346
|
+
now = _utcnow_iso()
|
|
347
|
+
with self._lock:
|
|
348
|
+
conn = self._connect()
|
|
349
|
+
try:
|
|
350
|
+
conn.execute(
|
|
351
|
+
"INSERT INTO source_quality_repair_state "
|
|
352
|
+
"(profile_id,last_rowid,last_settled_at,last_outcome_id,"
|
|
353
|
+
"updated_at) VALUES (?,0,?,?,?) "
|
|
354
|
+
"ON CONFLICT(profile_id) DO UPDATE SET "
|
|
355
|
+
"last_settled_at=excluded.last_settled_at,"
|
|
356
|
+
"last_outcome_id=excluded.last_outcome_id,"
|
|
357
|
+
"updated_at=excluded.updated_at",
|
|
358
|
+
(profile_id, str(settled_at), str(outcome_id), now),
|
|
359
|
+
)
|
|
360
|
+
conn.commit()
|
|
361
|
+
finally:
|
|
362
|
+
conn.close()
|
|
363
|
+
|
|
162
364
|
# ------------------------------------------------------------------
|
|
163
365
|
# Public API: read quality
|
|
164
366
|
# ------------------------------------------------------------------
|
|
@@ -301,3 +503,323 @@ class SourceQualityScorer:
|
|
|
301
503
|
return result
|
|
302
504
|
finally:
|
|
303
505
|
conn.close()
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def _source_key(row: sqlite3.Row) -> str:
|
|
509
|
+
source_type = str(row["source_type"] or "").strip()[:100]
|
|
510
|
+
actor = str(row["created_by"] or "").strip()
|
|
511
|
+
operation_or_legacy_source = str(row["source_id"] or "").strip()
|
|
512
|
+
# Canonical ingestion stores its unique operation UUID in source_id for
|
|
513
|
+
# lineage/idempotency and the stable trusted client in created_by. Quality
|
|
514
|
+
# must aggregate by the stable actor; operation IDs would create one
|
|
515
|
+
# single-observation "source" per remember call. Legacy provenance often
|
|
516
|
+
# has no actor, so retain its established source_id fallback.
|
|
517
|
+
identifier = (
|
|
518
|
+
actor
|
|
519
|
+
if actor and actor.lower() != "unknown"
|
|
520
|
+
else operation_or_legacy_source
|
|
521
|
+
)[:100]
|
|
522
|
+
if source_type and identifier:
|
|
523
|
+
return f"{source_type}:{identifier}"
|
|
524
|
+
if identifier:
|
|
525
|
+
return f"source:{identifier}"
|
|
526
|
+
return source_type
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def _load_source_map(
|
|
530
|
+
memory_db_path: Path,
|
|
531
|
+
profile_id: str,
|
|
532
|
+
fact_ids: list[str],
|
|
533
|
+
*,
|
|
534
|
+
strict: bool = False,
|
|
535
|
+
) -> dict[str, set[str]]:
|
|
536
|
+
"""Read only the provenance needed by this bounded reward batch."""
|
|
537
|
+
unique_facts = list(dict.fromkeys(str(fid) for fid in fact_ids if fid))
|
|
538
|
+
if not unique_facts or not Path(memory_db_path).exists():
|
|
539
|
+
return {}
|
|
540
|
+
result: dict[str, set[str]] = {}
|
|
541
|
+
try:
|
|
542
|
+
conn = sqlite3.connect(
|
|
543
|
+
f"file:{Path(memory_db_path)}?mode=ro", uri=True, timeout=1.0,
|
|
544
|
+
)
|
|
545
|
+
conn.row_factory = sqlite3.Row
|
|
546
|
+
try:
|
|
547
|
+
columns = {
|
|
548
|
+
str(row["name"])
|
|
549
|
+
for row in conn.execute("PRAGMA table_info(provenance)")
|
|
550
|
+
}
|
|
551
|
+
required = {
|
|
552
|
+
"profile_id", "fact_id", "source_type",
|
|
553
|
+
"source_id", "created_by",
|
|
554
|
+
}
|
|
555
|
+
if not required.issubset(columns):
|
|
556
|
+
return {}
|
|
557
|
+
for start in range(0, len(unique_facts), _PROVENANCE_QUERY_CHUNK):
|
|
558
|
+
chunk = unique_facts[start:start + _PROVENANCE_QUERY_CHUNK]
|
|
559
|
+
placeholders = ",".join("?" for _ in chunk)
|
|
560
|
+
rows = conn.execute(
|
|
561
|
+
"SELECT DISTINCT fact_id, source_type, source_id, created_by "
|
|
562
|
+
"FROM provenance WHERE profile_id = ? "
|
|
563
|
+
f"AND fact_id IN ({placeholders})",
|
|
564
|
+
(profile_id, *chunk),
|
|
565
|
+
).fetchall()
|
|
566
|
+
for row in rows:
|
|
567
|
+
key = _source_key(row)
|
|
568
|
+
if key:
|
|
569
|
+
result.setdefault(str(row["fact_id"]), set()).add(key)
|
|
570
|
+
finally:
|
|
571
|
+
conn.close()
|
|
572
|
+
except sqlite3.Error as exc:
|
|
573
|
+
logger.debug("source provenance unavailable: %s", exc)
|
|
574
|
+
if strict:
|
|
575
|
+
raise SourceQualityRepairUnavailable(
|
|
576
|
+
"source provenance temporarily unavailable",
|
|
577
|
+
) from exc
|
|
578
|
+
return {}
|
|
579
|
+
return result
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def update_source_quality_for_reward(
|
|
583
|
+
*,
|
|
584
|
+
memory_db_path: Path,
|
|
585
|
+
learning_db_path: Path,
|
|
586
|
+
profile_id: str,
|
|
587
|
+
outcome_id: str,
|
|
588
|
+
fact_ids: list[str],
|
|
589
|
+
reward: float,
|
|
590
|
+
) -> int:
|
|
591
|
+
"""Fail-soft online bridge from a finalized reward to real provenance."""
|
|
592
|
+
return update_source_quality_for_reward_batch(
|
|
593
|
+
memory_db_path=memory_db_path,
|
|
594
|
+
learning_db_path=learning_db_path,
|
|
595
|
+
rewards=[(profile_id, outcome_id, fact_ids, reward)],
|
|
596
|
+
)
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
def update_source_quality_for_reward_batch(
|
|
600
|
+
*,
|
|
601
|
+
memory_db_path: Path,
|
|
602
|
+
learning_db_path: Path,
|
|
603
|
+
rewards: list[tuple[str, str, list[str], float]],
|
|
604
|
+
) -> int:
|
|
605
|
+
"""Fail-soft bounded bridge for worker-finalized reward batches."""
|
|
606
|
+
try:
|
|
607
|
+
bounded_rewards = rewards[:1000]
|
|
608
|
+
normalized = []
|
|
609
|
+
for profile_id, outcome_id, fact_ids, reward in bounded_rewards:
|
|
610
|
+
bounded_facts = list(dict.fromkeys(fact_ids))[
|
|
611
|
+
:_MAX_FACTS_PER_OUTCOME
|
|
612
|
+
]
|
|
613
|
+
normalized.append((
|
|
614
|
+
profile_id, outcome_id, bounded_facts, float(reward),
|
|
615
|
+
))
|
|
616
|
+
if not normalized:
|
|
617
|
+
return 0
|
|
618
|
+
# Each reward batch is profile-homogeneous in current callers. Split
|
|
619
|
+
# defensively so provenance can never cross a profile boundary.
|
|
620
|
+
by_profile: dict[str, list[tuple[str, list[str], float]]] = {}
|
|
621
|
+
for profile_id, outcome_id, fact_ids, reward in normalized:
|
|
622
|
+
by_profile.setdefault(profile_id, []).append(
|
|
623
|
+
(outcome_id, fact_ids, reward),
|
|
624
|
+
)
|
|
625
|
+
scorer = SourceQualityScorer(Path(learning_db_path))
|
|
626
|
+
observations = []
|
|
627
|
+
for profile_id, profile_rewards in by_profile.items():
|
|
628
|
+
profile_facts = [
|
|
629
|
+
fact_id
|
|
630
|
+
for _, fact_ids, _ in profile_rewards
|
|
631
|
+
for fact_id in fact_ids
|
|
632
|
+
]
|
|
633
|
+
source_map = _load_source_map(
|
|
634
|
+
Path(memory_db_path), profile_id, profile_facts,
|
|
635
|
+
)
|
|
636
|
+
for outcome_id, fact_ids, reward in profile_rewards:
|
|
637
|
+
source_ids = sorted({
|
|
638
|
+
source
|
|
639
|
+
for fact_id in fact_ids
|
|
640
|
+
for source in source_map.get(fact_id, set())
|
|
641
|
+
})[:_MAX_SOURCES_PER_OUTCOME]
|
|
642
|
+
observations.append((
|
|
643
|
+
profile_id, outcome_id, source_ids, reward,
|
|
644
|
+
))
|
|
645
|
+
return scorer.record_rewards(observations)
|
|
646
|
+
except (OSError, sqlite3.Error, TypeError, ValueError) as exc:
|
|
647
|
+
logger.debug("source-quality reward update skipped: %s", exc)
|
|
648
|
+
return 0
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def enumerate_source_quality_repair_profiles(
|
|
652
|
+
memory_db_path: Path,
|
|
653
|
+
) -> list[str]:
|
|
654
|
+
"""Return profiles with settled numeric outcomes eligible for repair."""
|
|
655
|
+
path = Path(memory_db_path)
|
|
656
|
+
if not path.exists():
|
|
657
|
+
return []
|
|
658
|
+
try:
|
|
659
|
+
conn = sqlite3.connect(
|
|
660
|
+
f"file:{path}?mode=ro", uri=True, timeout=1.0,
|
|
661
|
+
)
|
|
662
|
+
conn.row_factory = sqlite3.Row
|
|
663
|
+
try:
|
|
664
|
+
columns = {
|
|
665
|
+
str(row["name"])
|
|
666
|
+
for row in conn.execute("PRAGMA table_info(action_outcomes)")
|
|
667
|
+
}
|
|
668
|
+
required = {"profile_id", "reward", "settled"}
|
|
669
|
+
if not required.issubset(columns):
|
|
670
|
+
return []
|
|
671
|
+
rows = conn.execute(
|
|
672
|
+
"SELECT DISTINCT profile_id FROM action_outcomes "
|
|
673
|
+
"WHERE settled = 1 AND reward IS NOT NULL "
|
|
674
|
+
"AND typeof(reward) IN ('integer', 'real') "
|
|
675
|
+
"AND profile_id IS NOT NULL AND profile_id != '' "
|
|
676
|
+
"ORDER BY profile_id ASC",
|
|
677
|
+
).fetchall()
|
|
678
|
+
return [str(row["profile_id"]) for row in rows]
|
|
679
|
+
finally:
|
|
680
|
+
conn.close()
|
|
681
|
+
except sqlite3.Error as exc:
|
|
682
|
+
logger.debug("source-quality profile enumeration unavailable: %s", exc)
|
|
683
|
+
raise SourceQualityRepairUnavailable(
|
|
684
|
+
"repair profile enumeration temporarily unavailable",
|
|
685
|
+
) from exc
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def _parse_repair_rows(
|
|
689
|
+
rows: list[sqlite3.Row],
|
|
690
|
+
) -> tuple[list[tuple[sqlite3.Row, list[str]]], list[str]]:
|
|
691
|
+
parsed: list[tuple[sqlite3.Row, list[str]]] = []
|
|
692
|
+
all_facts: list[str] = []
|
|
693
|
+
for row in rows:
|
|
694
|
+
try:
|
|
695
|
+
value = json.loads(str(row["fact_ids_json"] or "[]"))
|
|
696
|
+
fact_ids = (
|
|
697
|
+
[str(item) for item in value if item][:_MAX_FACTS_PER_OUTCOME]
|
|
698
|
+
if isinstance(value, list) else []
|
|
699
|
+
)
|
|
700
|
+
except (TypeError, ValueError, json.JSONDecodeError):
|
|
701
|
+
fact_ids = []
|
|
702
|
+
parsed.append((row, fact_ids))
|
|
703
|
+
all_facts.extend(fact_ids)
|
|
704
|
+
return parsed, all_facts
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def _repair_observations(
|
|
708
|
+
profile_id: str,
|
|
709
|
+
parsed: list[tuple[sqlite3.Row, list[str]]],
|
|
710
|
+
source_map: dict[str, set[str]],
|
|
711
|
+
) -> list[tuple[str, str, list[str], float]]:
|
|
712
|
+
observations = []
|
|
713
|
+
for row, fact_ids in parsed:
|
|
714
|
+
sources = sorted({
|
|
715
|
+
source for fact_id in fact_ids
|
|
716
|
+
for source in source_map.get(fact_id, set())
|
|
717
|
+
})[:_MAX_SOURCES_PER_OUTCOME]
|
|
718
|
+
observations.append((
|
|
719
|
+
profile_id, str(row["outcome_id"]), sources, float(row["reward"]),
|
|
720
|
+
))
|
|
721
|
+
return observations
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
def repair_historical_source_quality(
|
|
725
|
+
memory_db_path: Path,
|
|
726
|
+
learning_db_path: Path,
|
|
727
|
+
profile_id: str,
|
|
728
|
+
*,
|
|
729
|
+
batch_size: int = 250,
|
|
730
|
+
max_batches: int = 4,
|
|
731
|
+
) -> dict[str, int | bool]:
|
|
732
|
+
"""Explicit, resumable historical repair; never invoked by API handlers.
|
|
733
|
+
|
|
734
|
+
Work is capped to 1,000 outcomes per call by default. The cursor advances
|
|
735
|
+
only after the idempotent observation ledger commits, so interruption can
|
|
736
|
+
replay safely without double-counting.
|
|
737
|
+
"""
|
|
738
|
+
safe_batch = max(1, min(1000, int(batch_size)))
|
|
739
|
+
safe_batches = max(1, min(100, int(max_batches)))
|
|
740
|
+
scorer = SourceQualityScorer(Path(learning_db_path))
|
|
741
|
+
scanned = 0
|
|
742
|
+
observations = 0
|
|
743
|
+
complete = False
|
|
744
|
+
for _ in range(safe_batches):
|
|
745
|
+
settled_cursor, outcome_cursor = scorer.get_repair_position(profile_id)
|
|
746
|
+
rows = _load_reward_repair_batch(
|
|
747
|
+
Path(memory_db_path),
|
|
748
|
+
profile_id,
|
|
749
|
+
settled_cursor,
|
|
750
|
+
outcome_cursor,
|
|
751
|
+
safe_batch,
|
|
752
|
+
)
|
|
753
|
+
if not rows:
|
|
754
|
+
complete = True
|
|
755
|
+
break
|
|
756
|
+
parsed, all_facts = _parse_repair_rows(rows)
|
|
757
|
+
source_map = _load_source_map(
|
|
758
|
+
Path(memory_db_path), profile_id, all_facts, strict=True,
|
|
759
|
+
)
|
|
760
|
+
batch_observations = _repair_observations(
|
|
761
|
+
profile_id, parsed, source_map,
|
|
762
|
+
)
|
|
763
|
+
observations += scorer.record_rewards(batch_observations)
|
|
764
|
+
scanned += len(rows)
|
|
765
|
+
scorer.set_repair_position(
|
|
766
|
+
profile_id,
|
|
767
|
+
str(rows[-1]["settled_key"] or ""),
|
|
768
|
+
str(rows[-1]["outcome_id"]),
|
|
769
|
+
)
|
|
770
|
+
return {
|
|
771
|
+
"scanned": scanned,
|
|
772
|
+
"observations": observations,
|
|
773
|
+
"complete": complete,
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def _load_reward_repair_batch(
|
|
778
|
+
memory_db_path: Path,
|
|
779
|
+
profile_id: str,
|
|
780
|
+
after_settled_at: str,
|
|
781
|
+
after_outcome_id: str,
|
|
782
|
+
limit: int,
|
|
783
|
+
) -> list[sqlite3.Row]:
|
|
784
|
+
if not memory_db_path.exists():
|
|
785
|
+
return []
|
|
786
|
+
try:
|
|
787
|
+
conn = sqlite3.connect(
|
|
788
|
+
f"file:{memory_db_path}?mode=ro", uri=True, timeout=1.0,
|
|
789
|
+
)
|
|
790
|
+
conn.row_factory = sqlite3.Row
|
|
791
|
+
try:
|
|
792
|
+
columns = {
|
|
793
|
+
str(row["name"])
|
|
794
|
+
for row in conn.execute("PRAGMA table_info(action_outcomes)")
|
|
795
|
+
}
|
|
796
|
+
required = {
|
|
797
|
+
"outcome_id", "profile_id", "fact_ids_json",
|
|
798
|
+
"reward", "settled",
|
|
799
|
+
}
|
|
800
|
+
if not required.issubset(columns):
|
|
801
|
+
return []
|
|
802
|
+
return conn.execute(
|
|
803
|
+
"SELECT outcome_id, fact_ids_json, reward, "
|
|
804
|
+
"COALESCE(settled_at, '') AS settled_key "
|
|
805
|
+
"FROM action_outcomes WHERE profile_id = ? "
|
|
806
|
+
"AND settled = 1 AND reward IS NOT NULL "
|
|
807
|
+
"AND typeof(reward) IN ('integer', 'real') "
|
|
808
|
+
"AND (COALESCE(settled_at, '') > ? OR "
|
|
809
|
+
"(COALESCE(settled_at, '') = ? AND outcome_id > ?)) "
|
|
810
|
+
"ORDER BY COALESCE(settled_at, '') ASC, outcome_id ASC LIMIT ?",
|
|
811
|
+
(
|
|
812
|
+
profile_id,
|
|
813
|
+
str(after_settled_at),
|
|
814
|
+
str(after_settled_at),
|
|
815
|
+
str(after_outcome_id),
|
|
816
|
+
int(limit),
|
|
817
|
+
),
|
|
818
|
+
).fetchall()
|
|
819
|
+
finally:
|
|
820
|
+
conn.close()
|
|
821
|
+
except sqlite3.Error as exc:
|
|
822
|
+
logger.debug("source-quality repair unavailable: %s", exc)
|
|
823
|
+
raise SourceQualityRepairUnavailable(
|
|
824
|
+
"repair batch temporarily unavailable",
|
|
825
|
+
) from exc
|
|
@@ -156,8 +156,10 @@ class SLMMemoryLedger:
|
|
|
156
156
|
class _EngineLedgerStore:
|
|
157
157
|
"""Minimal profile-scoped store over a SuperLocalMemory engine.
|
|
158
158
|
|
|
159
|
-
Uses
|
|
160
|
-
|
|
159
|
+
Uses the engine's non-blocking write-through path when available and
|
|
160
|
+
direct, escaped, profile-scoped reads. The ``store`` fallback preserves
|
|
161
|
+
compatibility with lightweight adapter/test engines that predate
|
|
162
|
+
``store_fast``.
|
|
161
163
|
"""
|
|
162
164
|
|
|
163
165
|
def __init__(self, engine: Any, *, owns_engine: bool = True) -> None:
|
|
@@ -169,9 +171,27 @@ class _EngineLedgerStore:
|
|
|
169
171
|
self._owns_engine = owns_engine
|
|
170
172
|
|
|
171
173
|
def add(self, content: str, *, session_id: str, metadata: dict) -> None:
|
|
172
|
-
#
|
|
173
|
-
#
|
|
174
|
-
|
|
174
|
+
# A loop ledger needs the durable parent row and immediate lexical
|
|
175
|
+
# recall, not synchronous embeddings/entity/graph enrichment. Loading
|
|
176
|
+
# the heavyweight embedding worker for every bounded-loop lap can stall
|
|
177
|
+
# the loop for the full worker timeout and consume ~1 GB for metadata.
|
|
178
|
+
# The write-through path persists the same session-scoped content in
|
|
179
|
+
# milliseconds; ordinary background enrichment can still promote it.
|
|
180
|
+
fast_metadata = {**metadata, "session_id": session_id}
|
|
181
|
+
store_fast = getattr(self._engine, "store_fast", None)
|
|
182
|
+
if callable(store_fast):
|
|
183
|
+
store_fast(
|
|
184
|
+
content,
|
|
185
|
+
metadata=fast_metadata,
|
|
186
|
+
index_external=False,
|
|
187
|
+
)
|
|
188
|
+
return
|
|
189
|
+
|
|
190
|
+
self._engine.store(
|
|
191
|
+
content,
|
|
192
|
+
session_id=session_id,
|
|
193
|
+
metadata=metadata,
|
|
194
|
+
)
|
|
175
195
|
|
|
176
196
|
def list_session(self, session_id: str) -> list[dict]:
|
|
177
197
|
# Cap the read: a bounded-loop run is capped at max_iterations laps, so
|
|
@@ -203,7 +203,7 @@ class _FilteredServer:
|
|
|
203
203
|
"""
|
|
204
204
|
__slots__ = ("_server", "_allowed")
|
|
205
205
|
|
|
206
|
-
def __init__(self, real_server:
|
|
206
|
+
def __init__(self, real_server: SLMFastMCP, allowed: frozenset[str]) -> None:
|
|
207
207
|
self._server = real_server
|
|
208
208
|
self._allowed = allowed
|
|
209
209
|
|
|
@@ -265,29 +265,20 @@ from superlocalmemory.mcp.tools_loops import register_loop_tools
|
|
|
265
265
|
register_loop_tools(_target, get_engine) # v3.8.0: bounded-loop tools (CLI+command+MCP)
|
|
266
266
|
|
|
267
267
|
|
|
268
|
-
#
|
|
269
|
-
#
|
|
270
|
-
#
|
|
271
|
-
#
|
|
272
|
-
# the first tool call arrives (1-2s later), the engine is already warm.
|
|
273
|
-
# This applies to ALL IDEs: Claude Code, Cursor, Antigravity, Gemini CLI, etc.
|
|
268
|
+
# Keep stdio MCP processes thin until a tool truly needs a local LIGHT engine.
|
|
269
|
+
# Every open IDE/task owns a stdio process; eagerly opening memory.db in all of
|
|
270
|
+
# them multiplied RAM and SQLite writers on machines with many long-lived
|
|
271
|
+
# sessions. The shared daemon owns model warmup and common remember/recall work.
|
|
274
272
|
def _eager_warmup() -> None:
|
|
275
|
-
"""
|
|
273
|
+
"""Ensure the shared daemon is running without opening a per-client engine.
|
|
276
274
|
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
275
|
+
Mesh registration is intentionally lazy: a local stdio session is not a
|
|
276
|
+
remote peer, and heartbeat writes should begin only after a mesh tool is
|
|
277
|
+
actually used.
|
|
280
278
|
"""
|
|
281
|
-
import logging
|
|
282
279
|
_logger = logging.getLogger(__name__)
|
|
283
|
-
try:
|
|
284
|
-
get_engine()
|
|
285
|
-
_logger.info("MCP engine pre-warmed successfully")
|
|
286
|
-
except Exception as exc:
|
|
287
|
-
_logger.warning("MCP engine pre-warmup failed: %s", exc)
|
|
288
280
|
|
|
289
|
-
# Measurement / test harnesses set this to skip daemon-start
|
|
290
|
-
# mesh-register. The LIGHT engine init above still runs.
|
|
281
|
+
# Measurement / test harnesses set this to skip daemon-start.
|
|
291
282
|
if _os.environ.get("SLM_DISABLE_WARMUP_SIDE_EFFECTS") == "1":
|
|
292
283
|
return
|
|
293
284
|
|
|
@@ -300,22 +291,12 @@ def _eager_warmup() -> None:
|
|
|
300
291
|
except Exception as exc:
|
|
301
292
|
_logger.warning("Daemon auto-start failed: %s", exc)
|
|
302
293
|
|
|
303
|
-
# V3.4.6: Auto-register this MCP session as a mesh peer immediately.
|
|
304
|
-
# Previously, registration was lazy (only on first mesh tool call).
|
|
305
|
-
# Now every Claude session appears on the mesh from startup.
|
|
306
|
-
try:
|
|
307
|
-
from superlocalmemory.mcp.tools_mesh import auto_register_mesh
|
|
308
|
-
auto_register_mesh()
|
|
309
|
-
_logger.info("Mesh peer auto-registered at startup")
|
|
310
|
-
except Exception as exc:
|
|
311
|
-
_logger.warning("Mesh auto-register failed: %s", exc)
|
|
312
|
-
|
|
313
294
|
import threading
|
|
314
295
|
|
|
315
296
|
# v3.6.7: Suppress standalone-process behaviours when the MCP server is
|
|
316
297
|
# imported inside the daemon (SLM_MCP_EMBEDDED=1). Three threads are safe
|
|
317
298
|
# to run in a dedicated `slm mcp` subprocess but harmful inside the daemon:
|
|
318
|
-
# mcp-warmup —
|
|
299
|
+
# mcp-warmup — ensures the shared daemon only; never creates an engine.
|
|
319
300
|
# parent-watchdog — calls os._exit(0) if its parent IDE quits, which would
|
|
320
301
|
# kill the daemon along with it.
|
|
321
302
|
# stdin-eof-monitor — monitors stdin pipe; meaningless inside the daemon.
|