superlocalmemory 3.8.10 → 3.8.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +91 -0
  2. package/README.md +7 -3
  3. package/package.json +1 -1
  4. package/plugin/.claude-plugin/plugin.json +1 -1
  5. package/plugin/CLAUDE.md +3 -3
  6. package/plugin/agents/slm-governance-advisor.md +1 -1
  7. package/plugin/agents/slm-loop-runner.md +1 -1
  8. package/plugin/agents/slm-memory-advisor.md +1 -1
  9. package/plugin/agents/slm-optimize-advisor.md +1 -1
  10. package/plugin/requirements.txt +1 -1
  11. package/plugin/skills/slm-cache/SKILL.md +1 -1
  12. package/plugin/skills/slm-compress/SKILL.md +1 -1
  13. package/plugin/skills/slm-governance/SKILL.md +1 -1
  14. package/plugin/skills/slm-graph/SKILL.md +1 -1
  15. package/plugin/skills/slm-loop/SKILL.md +1 -1
  16. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  17. package/plugin/skills/slm-profile/SKILL.md +1 -1
  18. package/plugin/skills/slm-recall/SKILL.md +1 -1
  19. package/plugin/skills/slm-remember/SKILL.md +1 -1
  20. package/plugin/skills/slm-scope/SKILL.md +1 -1
  21. package/plugin/skills/slm-session/SKILL.md +1 -1
  22. package/plugin/skills/slm-status/SKILL.md +1 -1
  23. package/plugin-src/rules/AGENTS.md +1 -1
  24. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  25. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  26. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-recall/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-remember/SKILL.md +1 -1
  29. package/plugin-src/skills/slm-session/SKILL.md +1 -1
  30. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  31. package/pyproject.toml +1 -1
  32. package/src/superlocalmemory/__init__.py +1 -1
  33. package/src/superlocalmemory/cli/commands.py +28 -6
  34. package/src/superlocalmemory/cli/daemon.py +219 -10
  35. package/src/superlocalmemory/cli/setup_wizard.py +45 -1
  36. package/src/superlocalmemory/core/component_registry.py +25 -0
  37. package/src/superlocalmemory/core/config.py +35 -1
  38. package/src/superlocalmemory/core/engine_wiring.py +81 -5
  39. package/src/superlocalmemory/core/recall_pipeline.py +25 -4
  40. package/src/superlocalmemory/core/reranker_worker.py +78 -17
  41. package/src/superlocalmemory/infra/daemon_identity.py +16 -0
  42. package/src/superlocalmemory/infra/process_identity.py +180 -0
  43. package/src/superlocalmemory/learning/feedback.py +328 -27
  44. package/src/superlocalmemory/learning/legacy_migration.py +45 -4
  45. package/src/superlocalmemory/learning/pattern_miner.py +31 -11
  46. package/src/superlocalmemory/mcp/_daemon_proxy.py +23 -1
  47. package/src/superlocalmemory/mcp/tools_active.py +179 -17
  48. package/src/superlocalmemory/mcp/tools_core.py +6 -5
  49. package/src/superlocalmemory/retrieval/remote_reranker.py +636 -0
  50. package/src/superlocalmemory/retrieval/reranker.py +52 -5
  51. package/src/superlocalmemory/server/unified_daemon.py +4 -0
  52. package/src/superlocalmemory/storage/migration_runner.py +9 -0
  53. package/src/superlocalmemory/storage/migrations/M033_learning_feedback_channel.py +77 -0
  54. package/src/superlocalmemory/storage/migrations/__init__.py +2 -0
@@ -15,7 +15,25 @@ Privacy:
15
15
  - Queries are hashed to SHA-256[:16] for grouping.
16
16
 
17
17
  Storage:
18
- Uses direct sqlite3 with a self-contained ``learning_feedback`` table.
18
+ Every explicit-feedback event is written to the CANONICAL learning store
19
+ -- a ``learning_signals`` row paired 1:1 with a ``learning_features`` row
20
+ -- in the same transaction as the historic ``learning_feedback`` row.
21
+
22
+ ``learning_signals`` is canonical because every live consumer already
23
+ reads it: the dashboard's Living Brain panel and ranker-phase card
24
+ (``server/routes/brain.py``, ``server/routes/learning.py``), the LightGBM
25
+ retrainer, and -- since issue #106 -- the recall phase gate.
26
+ ``learning_feedback`` is the pre-v3.4.22 table: ``legacy_migration``
27
+ copies it forward into ``learning_signals``, the dashboard reports it as
28
+ ``legacy_feedback_rows`` with a "pending migration" card, and the phase
29
+ gate's own docstring calls it legacy. Writing feedback only there (the
30
+ v3.8.11 attempt at issue #102) put the durable write in a table no phase
31
+ counter consumes, which is why reported feedback still changed nothing.
32
+
33
+ It is kept written for one more release (LLD-07 D5) so ``pattern_miner``
34
+ channel mining and GDPR erasure keep working; the shared identity from
35
+ ``legacy_migration.legacy_query_id`` stops the two writers double-counting.
36
+
19
37
  NOT coupled to V3 DatabaseManager -- this is a standalone data collector.
20
38
  """
21
39
 
@@ -25,6 +43,7 @@ import hashlib
25
43
  import logging
26
44
  import sqlite3
27
45
  import threading
46
+ from dataclasses import dataclass
28
47
  from datetime import datetime, timezone
29
48
  from pathlib import Path
30
49
  from typing import Any, Dict, List, Optional
@@ -53,6 +72,13 @@ _DASHBOARD_SIGNAL_MAP: Dict[str, tuple[str, float]] = {
53
72
  "dwell_negative": ("dwell_negative", 0.2),
54
73
  }
55
74
 
75
+ # ``channel`` records WHICH retrieval channel surfaced the fact (semantic,
76
+ # bm25, entity_graph, temporal, ...). ``pattern_miner._mine_channel_and_
77
+ # coretrieval`` groups on it to mine ``channel_performance`` patterns. It was
78
+ # read by the miner but never defined here, so every fresh database raised
79
+ # "no such column: channel" — swallowed at debug level, which silently killed
80
+ # BOTH channel mining and the co-retrieval mining that followed it in the same
81
+ # try block. Defined here for new databases; M033 back-fills existing ones.
56
82
  _CREATE_TABLE = """
57
83
  CREATE TABLE IF NOT EXISTS learning_feedback (
58
84
  id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -62,7 +88,8 @@ CREATE TABLE IF NOT EXISTS learning_feedback (
62
88
  signal_value REAL NOT NULL,
63
89
  query_hash TEXT,
64
90
  created_at TEXT NOT NULL,
65
- metadata TEXT
91
+ metadata TEXT,
92
+ channel TEXT DEFAULT 'unknown'
66
93
  )
67
94
  """
68
95
 
@@ -71,12 +98,78 @@ CREATE INDEX IF NOT EXISTS idx_feedback_profile
71
98
  ON learning_feedback (profile_id, created_at DESC)
72
99
  """
73
100
 
101
+ _CREATE_CHANNEL_INDEX = """
102
+ CREATE INDEX IF NOT EXISTS idx_feedback_channel
103
+ ON learning_feedback (profile_id, channel)
104
+ """
105
+
106
+
107
+ # Signal type stamped on the canonical ``learning_signals`` row for an
108
+ # explicit-feedback event. Identical to what ``legacy_migration`` writes when
109
+ # it carries a ``learning_feedback`` row forward, so a row recorded eagerly
110
+ # and a row migrated in batch are indistinguishable to every consumer.
111
+ CANONICAL_SIGNAL_TYPE = "legacy_feedback"
112
+
113
+ # ``learning_features.features_json`` for a feedback event. Feedback arrives
114
+ # out of band -- there is no ranked candidate list to extract a real feature
115
+ # vector from -- so the row is empty and flagged ``is_synthetic=1``. The
116
+ # LightGBM retrainer selects ``WHERE is_synthetic=0``, so these rows move the
117
+ # phase counters and the bandit without ever polluting model training.
118
+ _SYNTHETIC_FEATURES_JSON = "{}"
119
+
120
+
121
+ @dataclass(frozen=True)
122
+ class FeedbackWrite:
123
+ """Outcome of one explicit-feedback write.
124
+
125
+ ``canonical`` is the only field callers should gate user-facing success
126
+ on: it is True when the ``learning_signals`` row that every phase counter
127
+ reads actually landed. ``feedback_row_id`` alone means the legacy row was
128
+ written, which on its own influences nothing.
129
+ """
130
+
131
+ feedback_row_id: Optional[int]
132
+ signal_row_id: Optional[int]
133
+ canonical: bool
134
+
74
135
 
75
136
  def _utcnow_iso() -> str:
76
137
  """Return current UTC time as ISO-8601 string."""
77
138
  return datetime.now(timezone.utc).isoformat()
78
139
 
79
140
 
141
+ def _canonical_schema_ready(conn: sqlite3.Connection) -> bool:
142
+ """Return True when learning.db can accept a canonical feedback event.
143
+
144
+ Both tables must exist AND carry the LLD-02 columns the event needs
145
+ (``learning_signals.query_id`` for the shared identity that keeps the
146
+ batch migration from double-counting, ``learning_features.is_synthetic``
147
+ for the flag that keeps these rows out of LightGBM training). A table
148
+ that exists without them cannot hold the event correctly, so treating
149
+ mere existence as readiness would write a row that silently breaks both
150
+ invariants.
151
+ """
152
+ try:
153
+ signal_cols = {
154
+ row[1] for row in conn.execute(
155
+ "PRAGMA table_info(learning_signals)",
156
+ )
157
+ }
158
+ feature_cols = {
159
+ row[1] for row in conn.execute(
160
+ "PRAGMA table_info(learning_features)",
161
+ )
162
+ }
163
+ except sqlite3.Error:
164
+ return False
165
+ return (
166
+ "query_id" in signal_cols
167
+ and "query_text_hash" in signal_cols
168
+ and "signal_id" in feature_cols
169
+ and "is_synthetic" in feature_cols
170
+ )
171
+
172
+
80
173
  def _hash_query(query: str) -> str:
81
174
  """Privacy-preserving SHA-256[:16] query hash."""
82
175
  return hashlib.sha256(query.encode("utf-8")).hexdigest()[:16]
@@ -96,7 +189,12 @@ class FeedbackCollector:
96
189
  def __init__(self, db_path: Path) -> None:
97
190
  self._db_path = Path(db_path)
98
191
  self._lock = threading.Lock()
192
+ # Latched once the canonical LLD-02 tables are confirmed present, so
193
+ # the sqlite_master probe runs at most once per collector instead of
194
+ # on every feedback write.
195
+ self._canonical_ready = False
99
196
  self._ensure_schema()
197
+ self._bootstrap_canonical_schema()
100
198
 
101
199
  # ------------------------------------------------------------------
102
200
  # Schema
@@ -108,6 +206,20 @@ class FeedbackCollector:
108
206
  try:
109
207
  conn.execute(_CREATE_TABLE)
110
208
  conn.execute(_CREATE_INDEX)
209
+ # Pre-3.8.11 databases created ``learning_feedback`` without the
210
+ # ``channel`` column. M033 covers migrated installs; this ADD keeps
211
+ # a collector pointed at a legacy file self-healing rather than
212
+ # failing every channel query for the life of the process.
213
+ existing = {
214
+ row[1] for row in
215
+ conn.execute("PRAGMA table_info(learning_feedback)")
216
+ }
217
+ if "channel" not in existing:
218
+ conn.execute(
219
+ "ALTER TABLE learning_feedback "
220
+ "ADD COLUMN channel TEXT DEFAULT 'unknown'"
221
+ )
222
+ conn.execute(_CREATE_CHANNEL_INDEX)
111
223
  conn.commit()
112
224
  finally:
113
225
  conn.close()
@@ -196,9 +308,48 @@ class FeedbackCollector:
196
308
  fact_id: str,
197
309
  signal_type: str,
198
310
  value: float,
311
+ query: str = "",
312
+ channel: str = "unknown",
199
313
  ) -> Optional[int]:
314
+ """Record explicit user feedback on a specific fact.
315
+
316
+ Back-compatible wrapper: returns the ``learning_feedback`` row id.
317
+ Callers that must tell a user whether the feedback actually reached
318
+ the store the phase counters read should use
319
+ :meth:`record_explicit_event` and check ``FeedbackWrite.canonical`` —
320
+ a legacy row id on its own influences no consumer.
321
+ """
322
+ return self.record_explicit_event(
323
+ profile_id=profile_id, fact_id=fact_id, signal_type=signal_type,
324
+ value=value, query=query, channel=channel,
325
+ ).feedback_row_id
326
+
327
+ def record_explicit_event(
328
+ self,
329
+ profile_id: str,
330
+ fact_id: str,
331
+ signal_type: str,
332
+ value: float,
333
+ query: str = "",
334
+ channel: str = "unknown",
335
+ ) -> FeedbackWrite:
200
336
  """
201
- Record explicit user feedback on a specific fact.
337
+ Record explicit user feedback as ONE atomic learning event.
338
+
339
+ Writes three rows in a single transaction: the historic
340
+ ``learning_feedback`` row (kept one more release for ``pattern_miner``
341
+ channel mining and GDPR erasure) plus the canonical
342
+ ``learning_signals`` + ``learning_features`` pair that the dashboard,
343
+ the recall phase gate, and the retrainer all read. Either the whole
344
+ event is durable or none of it is — a partial write would leave the
345
+ legacy table and the phase counters permanently disagreeing, which is
346
+ the shape of issue #106.
347
+
348
+ Recall itself is deliberately read-only (it must never open a writer —
349
+ see
350
+ ``test_readonly_bandit_uses_uri_read_connection_and_never_records_play``),
351
+ so explicit feedback is the only path that grows these tables outside
352
+ the signal worker.
202
353
 
203
354
  Args:
204
355
  profile_id: Profile providing feedback.
@@ -206,15 +357,19 @@ class FeedbackCollector:
206
357
  signal_type: One of ``user_positive``, ``user_negative``,
207
358
  ``user_correction``, or any custom type.
208
359
  value: Numeric signal value (0.0 to 1.0).
360
+ query: Originating query. Stored only as a SHA-256[:16]
361
+ hash — full text is never persisted.
362
+ channel: Retrieval channel that surfaced the fact.
209
363
 
210
364
  Returns:
211
- Row ID of the inserted record, or None on error.
365
+ A :class:`FeedbackWrite` describing exactly which rows landed.
212
366
  """
213
367
  if not profile_id or not fact_id:
214
- return None
368
+ return FeedbackWrite(None, None, False)
215
369
 
216
370
  clamped = max(0.0, min(1.0, float(value)))
217
371
  now = _utcnow_iso()
372
+ query_hash = _hash_query(query) if query else None
218
373
 
219
374
  with self._lock:
220
375
  conn = self._connect()
@@ -222,15 +377,165 @@ class FeedbackCollector:
222
377
  cursor = conn.execute(
223
378
  "INSERT INTO learning_feedback "
224
379
  "(profile_id, fact_id, signal_type, signal_value, "
225
- "query_hash, created_at, metadata) "
226
- "VALUES (?, ?, ?, ?, ?, ?, ?)",
227
- (profile_id, fact_id, signal_type, clamped, None, now, None),
380
+ "query_hash, created_at, metadata, channel) "
381
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
382
+ (profile_id, fact_id, signal_type, clamped, query_hash,
383
+ now, None, channel or "unknown"),
384
+ )
385
+ feedback_row_id = cursor.lastrowid
386
+ signal_row_id = self._insert_canonical_pair(
387
+ conn,
388
+ profile_id=profile_id,
389
+ fact_id=fact_id,
390
+ value=clamped,
391
+ query_hash=query_hash,
392
+ created_at=now,
393
+ feedback_row_id=feedback_row_id,
228
394
  )
229
395
  conn.commit()
230
- return cursor.lastrowid
396
+ return FeedbackWrite(
397
+ feedback_row_id, signal_row_id, signal_row_id is not None,
398
+ )
399
+ except sqlite3.Error:
400
+ conn.rollback()
401
+ raise
231
402
  finally:
232
403
  conn.close()
233
404
 
405
+ # ------------------------------------------------------------------
406
+ # Canonical store
407
+ # ------------------------------------------------------------------
408
+
409
+ def _insert_canonical_pair(
410
+ self,
411
+ conn: sqlite3.Connection,
412
+ *,
413
+ profile_id: str,
414
+ fact_id: str,
415
+ value: float,
416
+ query_hash: Optional[str],
417
+ created_at: str,
418
+ feedback_row_id: Optional[int],
419
+ ) -> Optional[int]:
420
+ """Insert the ``learning_signals`` + ``learning_features`` pair.
421
+
422
+ Runs inside the caller's open transaction so the canonical rows commit
423
+ with the legacy row or not at all. Returns the new signal row id, or
424
+ None when the canonical tables are absent — they are owned by the
425
+ migration runner (LLD-06 H15 forbids DDL here), so on a database that
426
+ predates them the caller is told the truth rather than handed a
427
+ fabricated success.
428
+ """
429
+ if feedback_row_id is None:
430
+ return None
431
+ if not self._canonical_tables_present(conn):
432
+ return None
433
+
434
+ from superlocalmemory.learning.legacy_migration import legacy_query_id
435
+
436
+ query_id = legacy_query_id(feedback_row_id)
437
+ # Pad to 32 hex chars so an eagerly-written row has the same shape as
438
+ # both a migrated row and a fresh signal-worker row.
439
+ padded_hash = ((query_hash or "") + ("0" * 32))[:32]
440
+
441
+ cursor = conn.execute(
442
+ "INSERT INTO learning_signals "
443
+ "(profile_id, query, fact_id, signal_type, value, created_at, "
444
+ " query_id, query_text_hash, position, channel_scores, "
445
+ " cross_encoder) "
446
+ "VALUES (?, '', ?, ?, ?, ?, ?, ?, 0, '{}', NULL)",
447
+ (profile_id, fact_id, CANONICAL_SIGNAL_TYPE, value, created_at,
448
+ query_id, padded_hash),
449
+ )
450
+ signal_row_id = cursor.lastrowid
451
+ conn.execute(
452
+ "INSERT INTO learning_features "
453
+ "(profile_id, query_id, fact_id, features_json, label, "
454
+ " created_at, signal_id, is_synthetic) "
455
+ "VALUES (?, ?, ?, ?, ?, ?, ?, 1)",
456
+ (profile_id, query_id, fact_id, _SYNTHETIC_FEATURES_JSON,
457
+ value, created_at, signal_row_id),
458
+ )
459
+ return signal_row_id
460
+
461
+ def _bootstrap_canonical_schema(self) -> None:
462
+ """Make sure the canonical store exists before any feedback arrives.
463
+
464
+ Explicit feedback can be the very first learning write on a machine —
465
+ a user can rate a recall before the daemon has ever run the migration
466
+ runner. Without this the write would honestly, but uselessly, report
467
+ that it never reached the store that gates ranking.
468
+
469
+ No DDL is authored here (LLD-06 H15). The base tables come from
470
+ ``LearningDatabase``, which the migration runner itself calls as its
471
+ first-boot bootstrap, and the LLD-02 columns come from M001's own DDL.
472
+ Applying M001's DDL without a ``migration_log`` row is safe: when the
473
+ real runner reaches M001 its ALTERs fail, ``M001.verify`` passes, and
474
+ the runner records it as "already applied (verified via schema
475
+ inspection)".
476
+
477
+ Never fatal — a read-only or unwritable learning.db must not stop a
478
+ collector from being constructed.
479
+ """
480
+ try:
481
+ from superlocalmemory.learning.database import LearningDatabase
482
+ from superlocalmemory.storage.migrations import (
483
+ M001_add_signal_features_columns as _m001,
484
+ )
485
+
486
+ LearningDatabase(self._db_path)
487
+ conn = self._connect()
488
+ try:
489
+ if not _m001.verify(conn):
490
+ conn.executescript(_m001.DDL)
491
+ conn.commit()
492
+ self._canonical_ready = _canonical_schema_ready(conn)
493
+ finally:
494
+ conn.close()
495
+ except Exception as exc: # noqa: BLE001 — construction must not fail
496
+ logger.warning(
497
+ "canonical learning schema bootstrap failed for %s: %s",
498
+ self._db_path, exc,
499
+ )
500
+
501
+ def _canonical_tables_present(self, conn: sqlite3.Connection) -> bool:
502
+ """Return True when both canonical LLD-02 tables exist.
503
+
504
+ Re-probes while unready so a collector constructed before the
505
+ migration runner ran starts writing canonically as soon as the tables
506
+ appear, instead of degrading for the life of the process.
507
+ """
508
+ if self._canonical_ready:
509
+ return True
510
+ self._canonical_ready = _canonical_schema_ready(conn)
511
+ if not self._canonical_ready:
512
+ logger.warning(
513
+ "learning.db at %s has no usable learning_signals/"
514
+ "learning_features schema; explicit feedback cannot reach the "
515
+ "store that gates adaptive ranking.",
516
+ self._db_path,
517
+ )
518
+ return self._canonical_ready
519
+
520
+ def get_signal_count(self, profile_id: str) -> int:
521
+ """Return the canonical signal count that gates the ranking phase.
522
+
523
+ This is the single number the recall phase gate, the dashboard's
524
+ Living Brain panel, and the ranker-phase card all resolve their phase
525
+ from. Reporting anything else to a user — as ``report_feedback`` did
526
+ with ``feedback_records`` before issue #106 — shows progress toward a
527
+ threshold nothing is actually measuring.
528
+ """
529
+ conn = self._connect()
530
+ try:
531
+ row = conn.execute(
532
+ "SELECT COUNT(*) FROM learning_signals WHERE profile_id = ?",
533
+ (profile_id,),
534
+ ).fetchone()
535
+ return row[0] if row else 0
536
+ finally:
537
+ conn.close()
538
+
234
539
  # ------------------------------------------------------------------
235
540
  # Public API: record dashboard feedback
236
541
  # ------------------------------------------------------------------
@@ -253,30 +558,26 @@ class FeedbackCollector:
253
558
  This method restores the dashboard feedback path: the HTTP routes in
254
559
  ``server/routes/learning.py`` called it before it existed, so every
255
560
  thumbs/pin/dwell write raised ``AttributeError`` (issues #53/#59).
561
+
562
+ Routed through :meth:`record_explicit_event` so a thumbs-up from the
563
+ dashboard lands in exactly the same canonical store as a thumbs-up
564
+ from MCP. Before issue #106 this path wrote only ``learning_feedback``,
565
+ so the dashboard's own Living Brain counter — which reads
566
+ ``learning_signals`` — never moved in response to its own buttons.
256
567
  """
257
568
  if not memory_id:
258
569
  return None
259
570
  signal_type, value = _DASHBOARD_SIGNAL_MAP.get(
260
571
  feedback_type, ("user_correction", 0.5),
261
572
  )
262
- qhash = _hash_query(query) if query else None
263
- now = _utcnow_iso()
264
-
265
- with self._lock:
266
- conn = self._connect()
267
- try:
268
- cursor = conn.execute(
269
- "INSERT INTO learning_feedback "
270
- "(profile_id, fact_id, signal_type, signal_value, "
271
- "query_hash, created_at, metadata) "
272
- "VALUES (?, ?, ?, ?, ?, ?, ?)",
273
- (profile_id or "default", str(memory_id), signal_type,
274
- value, qhash, now, None),
275
- )
276
- conn.commit()
277
- return cursor.lastrowid
278
- finally:
279
- conn.close()
573
+ return self.record_explicit_event(
574
+ profile_id=profile_id or "default",
575
+ fact_id=str(memory_id),
576
+ signal_type=signal_type,
577
+ value=value,
578
+ query=query,
579
+ channel="dashboard",
580
+ ).feedback_row_id
280
581
 
281
582
  # ------------------------------------------------------------------
282
583
  # Public API: read feedback
@@ -41,6 +41,20 @@ MIGRATION_NAME = "LEG001_feedback_to_signals"
41
41
  _COPY_BATCH_SIZE = 500
42
42
 
43
43
 
44
+ def legacy_query_id(feedback_row_id: int | str) -> str:
45
+ """Return the canonical ``learning_signals.query_id`` for a feedback row.
46
+
47
+ One explicit-feedback event has exactly ONE canonical identity, whichever
48
+ path writes it: this batch migration, or ``FeedbackCollector`` writing the
49
+ canonical pair eagerly at feedback time. Both derive the id from the
50
+ ``learning_feedback`` row id through this function, which is what lets the
51
+ migration recognise — and skip — rows already carried forward. Without a
52
+ shared identity the two writers would double-count the same event into the
53
+ store that gates the ranking phase.
54
+ """
55
+ return f"legacy:{feedback_row_id}"
56
+
57
+
44
58
  def migrate_legacy_feedback(
45
59
  learning_db: Path,
46
60
  *,
@@ -156,10 +170,18 @@ def _copy_rows(conn: sqlite3.Connection) -> tuple[int, int]:
156
170
 
157
171
  Returns ``(copied, failed)``. Does not raise. Commits per batch so
158
172
  a later failure still leaves the earlier batches durable.
173
+
174
+ Rows whose canonical ``query_id`` is already present in
175
+ ``learning_signals`` are skipped rather than copied a second time.
176
+ ``FeedbackCollector`` writes the canonical pair at feedback time, so on
177
+ any install where this migration has not yet been sentinel-marked the
178
+ newest rows are already carried forward; copying them again would
179
+ inflate the very counter that gates the ranking phase.
159
180
  """
160
181
  copied = 0
161
182
  failed = 0
162
183
  offset = 0
184
+ already_present = _existing_legacy_query_ids(conn)
163
185
  while True:
164
186
  try:
165
187
  batch = conn.execute(
@@ -179,6 +201,8 @@ def _copy_rows(conn: sqlite3.Connection) -> tuple[int, int]:
179
201
  try:
180
202
  conn.execute("BEGIN IMMEDIATE")
181
203
  for row in batch:
204
+ if legacy_query_id(row["id"]) in already_present:
205
+ continue
182
206
  try:
183
207
  _copy_single_row(conn, row)
184
208
  copied += 1
@@ -199,6 +223,23 @@ def _copy_rows(conn: sqlite3.Connection) -> tuple[int, int]:
199
223
  return copied, failed
200
224
 
201
225
 
226
+ def _existing_legacy_query_ids(conn: sqlite3.Connection) -> set[str]:
227
+ """Return every ``legacy:`` query_id already present in learning_signals.
228
+
229
+ Read once up front: a per-row EXISTS probe over a signals table that grows
230
+ to tens of thousands of rows turns an O(n) copy into O(n*m).
231
+ """
232
+ try:
233
+ rows = conn.execute(
234
+ "SELECT DISTINCT query_id FROM learning_signals "
235
+ "WHERE query_id LIKE 'legacy:%'",
236
+ ).fetchall()
237
+ except sqlite3.Error as exc:
238
+ logger.warning("legacy migration: dedupe probe failed: %s", exc)
239
+ return set()
240
+ return {str(row[0]) for row in rows}
241
+
242
+
202
243
  def _copy_single_row(conn: sqlite3.Connection, row: sqlite3.Row) -> None:
203
244
  """Insert one legacy row into learning_signals + learning_features.
204
245
 
@@ -218,7 +259,7 @@ def _copy_single_row(conn: sqlite3.Connection, row: sqlite3.Row) -> None:
218
259
  datetime.now(timezone.utc).isoformat(timespec="seconds"))
219
260
  profile_id = str(row["profile_id"] or "default")
220
261
  fact_id = str(row["fact_id"] or "")
221
- legacy_query_id = f"legacy:{row['id']}"
262
+ query_id = legacy_query_id(row["id"])
222
263
 
223
264
  # Insert the signal row. ``signal_type='legacy_feedback'`` marks it
224
265
  # clearly so consumers (dashboard, labeler) can treat it correctly.
@@ -229,7 +270,7 @@ def _copy_single_row(conn: sqlite3.Connection, row: sqlite3.Row) -> None:
229
270
  "VALUES (?, '', ?, 'legacy_feedback', ?, ?, ?, ?, 0, '{}', NULL)",
230
271
  (profile_id, fact_id,
231
272
  float(row["signal_value"] or 1.0),
232
- created_at, legacy_query_id, query_hash),
273
+ created_at, query_id, query_hash),
233
274
  )
234
275
  sid = cur.lastrowid
235
276
 
@@ -241,7 +282,7 @@ def _copy_single_row(conn: sqlite3.Connection, row: sqlite3.Row) -> None:
241
282
  "(profile_id, query_id, fact_id, features_json, label, created_at, "
242
283
  " signal_id, is_synthetic) "
243
284
  "VALUES (?, ?, ?, '{}', 0.0, ?, ?, 1)",
244
- (profile_id, legacy_query_id, fact_id, created_at, sid),
285
+ (profile_id, query_id, fact_id, created_at, sid),
245
286
  )
246
287
 
247
288
 
@@ -274,4 +315,4 @@ def _record_migration(
274
315
  logger.warning("legacy migration: log record failed: %s", exc)
275
316
 
276
317
 
277
- __all__ = ("migrate_legacy_feedback", "MIGRATION_NAME")
318
+ __all__ = ("migrate_legacy_feedback", "MIGRATION_NAME", "legacy_query_id")
@@ -325,14 +325,28 @@ def _mine_channel_and_coretrieval(
325
325
  learn_conn = sqlite3.connect(learning_db, timeout=10)
326
326
  learn_conn.row_factory = sqlite3.Row
327
327
 
328
- channel_rows = learn_conn.execute(
329
- "SELECT channel, COUNT(*) AS cnt, "
330
- "AVG(signal_value) AS avg_signal "
331
- "FROM learning_feedback "
332
- "WHERE profile_id = ? "
333
- "GROUP BY channel ORDER BY cnt DESC",
334
- (profile_id,),
335
- ).fetchall()
328
+ # Isolated from the co-retrieval block below. Until 3.8.11
329
+ # ``learning_feedback`` had no ``channel`` column, so this query
330
+ # raised and — sharing one try with co-retrieval — took that mining
331
+ # down with it. Two pattern types died from one missing column, and
332
+ # the only trace was a DEBUG line. Each miner now fails alone, loudly.
333
+ channel_rows = []
334
+ try:
335
+ channel_rows = learn_conn.execute(
336
+ "SELECT channel, COUNT(*) AS cnt, "
337
+ "AVG(signal_value) AS avg_signal "
338
+ "FROM learning_feedback "
339
+ "WHERE profile_id = ? "
340
+ "GROUP BY channel ORDER BY cnt DESC",
341
+ (profile_id,),
342
+ ).fetchall()
343
+ except sqlite3.Error as exc:
344
+ logger.warning(
345
+ "Channel pattern mining skipped — learning_feedback query "
346
+ "failed (%s). Run 'slm db migrate' to apply M033 if this "
347
+ "reports a missing 'channel' column. Co-retrieval mining "
348
+ "continues.", exc,
349
+ )
336
350
 
337
351
  for row in channel_rows:
338
352
  d = dict(row)
@@ -376,12 +390,18 @@ def _mine_channel_and_coretrieval(
376
390
  confidence=min(1.0, len(coret_rows) / 10),
377
391
  )
378
392
  gen += 1
379
- except Exception:
380
- pass
393
+ except sqlite3.Error as exc:
394
+ logger.warning(
395
+ "Co-retrieval pattern mining skipped — co_retrieval_edges "
396
+ "query failed: %s", exc,
397
+ )
381
398
 
382
399
  learn_conn.close()
383
400
  except Exception as exc:
384
- logger.debug("Signal pattern mining failed: %s", exc)
401
+ # Was DEBUG. A learning subsystem that mines nothing must say so at a
402
+ # level operators actually see; issue #102 went undiagnosed for weeks
403
+ # because the only evidence was invisible by default.
404
+ logger.warning("Signal pattern mining failed: %s", exc)
385
405
  return gen
386
406
 
387
407
 
@@ -23,6 +23,28 @@ from typing import Any
23
23
 
24
24
  logger = logging.getLogger(__name__)
25
25
 
26
+ _OPAQUE_UNAVAILABLE = "DAEMON_UNAVAILABLE: owned daemon is unavailable; retry later."
27
+
28
+
29
+ def daemon_unavailable_error() -> str:
30
+ """Return a one-line, *diagnosed* daemon-unavailable message.
31
+
32
+ The opaque wording this replaces described a stopped daemon, a recycled
33
+ PID, an unreachable port and an identity mismatch identically (issue #104).
34
+ Diagnosis is best effort: if it fails for any reason the caller still gets
35
+ the original, retryable message rather than an exception.
36
+ """
37
+ try:
38
+ from superlocalmemory.cli.daemon import describe_daemon_unavailability
39
+
40
+ diagnosis = describe_daemon_unavailability()
41
+ return (
42
+ f"DAEMON_UNAVAILABLE ({diagnosis['reason']}): "
43
+ f"{diagnosis['message']} {diagnosis['hint']}"
44
+ )
45
+ except Exception: # noqa: BLE001 - diagnosis must never mask the failure
46
+ return _OPAQUE_UNAVAILABLE
47
+
26
48
 
27
49
  class DaemonPoolProxy:
28
50
  """:class:`WorkerPool`-shaped facade that talks to the daemon over HTTP.
@@ -52,7 +74,7 @@ class DaemonPoolProxy:
52
74
  "ok": False,
53
75
  "code": "DAEMON_UNAVAILABLE",
54
76
  "retryable": True,
55
- "error": "DAEMON_UNAVAILABLE: owned daemon is unavailable; retry later.",
77
+ "error": daemon_unavailable_error(),
56
78
  }
57
79
 
58
80
  def recall(