superlocalmemory 3.8.10 → 3.8.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +91 -0
- package/README.md +7 -3
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +28 -6
- package/src/superlocalmemory/cli/daemon.py +219 -10
- package/src/superlocalmemory/cli/setup_wizard.py +45 -1
- package/src/superlocalmemory/core/component_registry.py +25 -0
- package/src/superlocalmemory/core/config.py +35 -1
- package/src/superlocalmemory/core/engine_wiring.py +81 -5
- package/src/superlocalmemory/core/recall_pipeline.py +25 -4
- package/src/superlocalmemory/core/reranker_worker.py +78 -17
- package/src/superlocalmemory/infra/daemon_identity.py +16 -0
- package/src/superlocalmemory/infra/process_identity.py +180 -0
- package/src/superlocalmemory/learning/feedback.py +328 -27
- package/src/superlocalmemory/learning/legacy_migration.py +45 -4
- package/src/superlocalmemory/learning/pattern_miner.py +31 -11
- package/src/superlocalmemory/mcp/_daemon_proxy.py +23 -1
- package/src/superlocalmemory/mcp/tools_active.py +179 -17
- package/src/superlocalmemory/mcp/tools_core.py +6 -5
- package/src/superlocalmemory/retrieval/remote_reranker.py +636 -0
- package/src/superlocalmemory/retrieval/reranker.py +52 -5
- package/src/superlocalmemory/server/unified_daemon.py +4 -0
- package/src/superlocalmemory/storage/migration_runner.py +9 -0
- package/src/superlocalmemory/storage/migrations/M033_learning_feedback_channel.py +77 -0
- package/src/superlocalmemory/storage/migrations/__init__.py +2 -0
|
@@ -15,7 +15,25 @@ Privacy:
|
|
|
15
15
|
- Queries are hashed to SHA-256[:16] for grouping.
|
|
16
16
|
|
|
17
17
|
Storage:
|
|
18
|
-
|
|
18
|
+
Every explicit-feedback event is written to the CANONICAL learning store
|
|
19
|
+
-- a ``learning_signals`` row paired 1:1 with a ``learning_features`` row
|
|
20
|
+
-- in the same transaction as the historic ``learning_feedback`` row.
|
|
21
|
+
|
|
22
|
+
``learning_signals`` is canonical because every live consumer already
|
|
23
|
+
reads it: the dashboard's Living Brain panel and ranker-phase card
|
|
24
|
+
(``server/routes/brain.py``, ``server/routes/learning.py``), the LightGBM
|
|
25
|
+
retrainer, and -- since issue #106 -- the recall phase gate.
|
|
26
|
+
``learning_feedback`` is the pre-v3.4.22 table: ``legacy_migration``
|
|
27
|
+
copies it forward into ``learning_signals``, the dashboard reports it as
|
|
28
|
+
``legacy_feedback_rows`` with a "pending migration" card, and the phase
|
|
29
|
+
gate's own docstring calls it legacy. Writing feedback only there (the
|
|
30
|
+
v3.8.11 attempt at issue #102) put the durable write in a table no phase
|
|
31
|
+
counter consumes, which is why reported feedback still changed nothing.
|
|
32
|
+
|
|
33
|
+
It is kept written for one more release (LLD-07 D5) so ``pattern_miner``
|
|
34
|
+
channel mining and GDPR erasure keep working; the shared identity from
|
|
35
|
+
``legacy_migration.legacy_query_id`` stops the two writers double-counting.
|
|
36
|
+
|
|
19
37
|
NOT coupled to V3 DatabaseManager -- this is a standalone data collector.
|
|
20
38
|
"""
|
|
21
39
|
|
|
@@ -25,6 +43,7 @@ import hashlib
|
|
|
25
43
|
import logging
|
|
26
44
|
import sqlite3
|
|
27
45
|
import threading
|
|
46
|
+
from dataclasses import dataclass
|
|
28
47
|
from datetime import datetime, timezone
|
|
29
48
|
from pathlib import Path
|
|
30
49
|
from typing import Any, Dict, List, Optional
|
|
@@ -53,6 +72,13 @@ _DASHBOARD_SIGNAL_MAP: Dict[str, tuple[str, float]] = {
|
|
|
53
72
|
"dwell_negative": ("dwell_negative", 0.2),
|
|
54
73
|
}
|
|
55
74
|
|
|
75
|
+
# ``channel`` records WHICH retrieval channel surfaced the fact (semantic,
|
|
76
|
+
# bm25, entity_graph, temporal, ...). ``pattern_miner._mine_channel_and_
|
|
77
|
+
# coretrieval`` groups on it to mine ``channel_performance`` patterns. It was
|
|
78
|
+
# read by the miner but never defined here, so every fresh database raised
|
|
79
|
+
# "no such column: channel" — swallowed at debug level, which silently killed
|
|
80
|
+
# BOTH channel mining and the co-retrieval mining that followed it in the same
|
|
81
|
+
# try block. Defined here for new databases; M033 back-fills existing ones.
|
|
56
82
|
_CREATE_TABLE = """
|
|
57
83
|
CREATE TABLE IF NOT EXISTS learning_feedback (
|
|
58
84
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -62,7 +88,8 @@ CREATE TABLE IF NOT EXISTS learning_feedback (
|
|
|
62
88
|
signal_value REAL NOT NULL,
|
|
63
89
|
query_hash TEXT,
|
|
64
90
|
created_at TEXT NOT NULL,
|
|
65
|
-
metadata TEXT
|
|
91
|
+
metadata TEXT,
|
|
92
|
+
channel TEXT DEFAULT 'unknown'
|
|
66
93
|
)
|
|
67
94
|
"""
|
|
68
95
|
|
|
@@ -71,12 +98,78 @@ CREATE INDEX IF NOT EXISTS idx_feedback_profile
|
|
|
71
98
|
ON learning_feedback (profile_id, created_at DESC)
|
|
72
99
|
"""
|
|
73
100
|
|
|
101
|
+
_CREATE_CHANNEL_INDEX = """
|
|
102
|
+
CREATE INDEX IF NOT EXISTS idx_feedback_channel
|
|
103
|
+
ON learning_feedback (profile_id, channel)
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
# Signal type stamped on the canonical ``learning_signals`` row for an
|
|
108
|
+
# explicit-feedback event. Identical to what ``legacy_migration`` writes when
|
|
109
|
+
# it carries a ``learning_feedback`` row forward, so a row recorded eagerly
|
|
110
|
+
# and a row migrated in batch are indistinguishable to every consumer.
|
|
111
|
+
CANONICAL_SIGNAL_TYPE = "legacy_feedback"
|
|
112
|
+
|
|
113
|
+
# ``learning_features.features_json`` for a feedback event. Feedback arrives
|
|
114
|
+
# out of band -- there is no ranked candidate list to extract a real feature
|
|
115
|
+
# vector from -- so the row is empty and flagged ``is_synthetic=1``. The
|
|
116
|
+
# LightGBM retrainer selects ``WHERE is_synthetic=0``, so these rows move the
|
|
117
|
+
# phase counters and the bandit without ever polluting model training.
|
|
118
|
+
_SYNTHETIC_FEATURES_JSON = "{}"
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@dataclass(frozen=True)
|
|
122
|
+
class FeedbackWrite:
|
|
123
|
+
"""Outcome of one explicit-feedback write.
|
|
124
|
+
|
|
125
|
+
``canonical`` is the only field callers should gate user-facing success
|
|
126
|
+
on: it is True when the ``learning_signals`` row that every phase counter
|
|
127
|
+
reads actually landed. ``feedback_row_id`` alone means the legacy row was
|
|
128
|
+
written, which on its own influences nothing.
|
|
129
|
+
"""
|
|
130
|
+
|
|
131
|
+
feedback_row_id: Optional[int]
|
|
132
|
+
signal_row_id: Optional[int]
|
|
133
|
+
canonical: bool
|
|
134
|
+
|
|
74
135
|
|
|
75
136
|
def _utcnow_iso() -> str:
|
|
76
137
|
"""Return current UTC time as ISO-8601 string."""
|
|
77
138
|
return datetime.now(timezone.utc).isoformat()
|
|
78
139
|
|
|
79
140
|
|
|
141
|
+
def _canonical_schema_ready(conn: sqlite3.Connection) -> bool:
|
|
142
|
+
"""Return True when learning.db can accept a canonical feedback event.
|
|
143
|
+
|
|
144
|
+
Both tables must exist AND carry the LLD-02 columns the event needs
|
|
145
|
+
(``learning_signals.query_id`` for the shared identity that keeps the
|
|
146
|
+
batch migration from double-counting, ``learning_features.is_synthetic``
|
|
147
|
+
for the flag that keeps these rows out of LightGBM training). A table
|
|
148
|
+
that exists without them cannot hold the event correctly, so treating
|
|
149
|
+
mere existence as readiness would write a row that silently breaks both
|
|
150
|
+
invariants.
|
|
151
|
+
"""
|
|
152
|
+
try:
|
|
153
|
+
signal_cols = {
|
|
154
|
+
row[1] for row in conn.execute(
|
|
155
|
+
"PRAGMA table_info(learning_signals)",
|
|
156
|
+
)
|
|
157
|
+
}
|
|
158
|
+
feature_cols = {
|
|
159
|
+
row[1] for row in conn.execute(
|
|
160
|
+
"PRAGMA table_info(learning_features)",
|
|
161
|
+
)
|
|
162
|
+
}
|
|
163
|
+
except sqlite3.Error:
|
|
164
|
+
return False
|
|
165
|
+
return (
|
|
166
|
+
"query_id" in signal_cols
|
|
167
|
+
and "query_text_hash" in signal_cols
|
|
168
|
+
and "signal_id" in feature_cols
|
|
169
|
+
and "is_synthetic" in feature_cols
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
80
173
|
def _hash_query(query: str) -> str:
|
|
81
174
|
"""Privacy-preserving SHA-256[:16] query hash."""
|
|
82
175
|
return hashlib.sha256(query.encode("utf-8")).hexdigest()[:16]
|
|
@@ -96,7 +189,12 @@ class FeedbackCollector:
|
|
|
96
189
|
def __init__(self, db_path: Path) -> None:
|
|
97
190
|
self._db_path = Path(db_path)
|
|
98
191
|
self._lock = threading.Lock()
|
|
192
|
+
# Latched once the canonical LLD-02 tables are confirmed present, so
|
|
193
|
+
# the sqlite_master probe runs at most once per collector instead of
|
|
194
|
+
# on every feedback write.
|
|
195
|
+
self._canonical_ready = False
|
|
99
196
|
self._ensure_schema()
|
|
197
|
+
self._bootstrap_canonical_schema()
|
|
100
198
|
|
|
101
199
|
# ------------------------------------------------------------------
|
|
102
200
|
# Schema
|
|
@@ -108,6 +206,20 @@ class FeedbackCollector:
|
|
|
108
206
|
try:
|
|
109
207
|
conn.execute(_CREATE_TABLE)
|
|
110
208
|
conn.execute(_CREATE_INDEX)
|
|
209
|
+
# Pre-3.8.11 databases created ``learning_feedback`` without the
|
|
210
|
+
# ``channel`` column. M033 covers migrated installs; this ADD keeps
|
|
211
|
+
# a collector pointed at a legacy file self-healing rather than
|
|
212
|
+
# failing every channel query for the life of the process.
|
|
213
|
+
existing = {
|
|
214
|
+
row[1] for row in
|
|
215
|
+
conn.execute("PRAGMA table_info(learning_feedback)")
|
|
216
|
+
}
|
|
217
|
+
if "channel" not in existing:
|
|
218
|
+
conn.execute(
|
|
219
|
+
"ALTER TABLE learning_feedback "
|
|
220
|
+
"ADD COLUMN channel TEXT DEFAULT 'unknown'"
|
|
221
|
+
)
|
|
222
|
+
conn.execute(_CREATE_CHANNEL_INDEX)
|
|
111
223
|
conn.commit()
|
|
112
224
|
finally:
|
|
113
225
|
conn.close()
|
|
@@ -196,9 +308,48 @@ class FeedbackCollector:
|
|
|
196
308
|
fact_id: str,
|
|
197
309
|
signal_type: str,
|
|
198
310
|
value: float,
|
|
311
|
+
query: str = "",
|
|
312
|
+
channel: str = "unknown",
|
|
199
313
|
) -> Optional[int]:
|
|
314
|
+
"""Record explicit user feedback on a specific fact.
|
|
315
|
+
|
|
316
|
+
Back-compatible wrapper: returns the ``learning_feedback`` row id.
|
|
317
|
+
Callers that must tell a user whether the feedback actually reached
|
|
318
|
+
the store the phase counters read should use
|
|
319
|
+
:meth:`record_explicit_event` and check ``FeedbackWrite.canonical`` —
|
|
320
|
+
a legacy row id on its own influences no consumer.
|
|
321
|
+
"""
|
|
322
|
+
return self.record_explicit_event(
|
|
323
|
+
profile_id=profile_id, fact_id=fact_id, signal_type=signal_type,
|
|
324
|
+
value=value, query=query, channel=channel,
|
|
325
|
+
).feedback_row_id
|
|
326
|
+
|
|
327
|
+
def record_explicit_event(
|
|
328
|
+
self,
|
|
329
|
+
profile_id: str,
|
|
330
|
+
fact_id: str,
|
|
331
|
+
signal_type: str,
|
|
332
|
+
value: float,
|
|
333
|
+
query: str = "",
|
|
334
|
+
channel: str = "unknown",
|
|
335
|
+
) -> FeedbackWrite:
|
|
200
336
|
"""
|
|
201
|
-
Record explicit user feedback
|
|
337
|
+
Record explicit user feedback as ONE atomic learning event.
|
|
338
|
+
|
|
339
|
+
Writes three rows in a single transaction: the historic
|
|
340
|
+
``learning_feedback`` row (kept one more release for ``pattern_miner``
|
|
341
|
+
channel mining and GDPR erasure) plus the canonical
|
|
342
|
+
``learning_signals`` + ``learning_features`` pair that the dashboard,
|
|
343
|
+
the recall phase gate, and the retrainer all read. Either the whole
|
|
344
|
+
event is durable or none of it is — a partial write would leave the
|
|
345
|
+
legacy table and the phase counters permanently disagreeing, which is
|
|
346
|
+
the shape of issue #106.
|
|
347
|
+
|
|
348
|
+
Recall itself is deliberately read-only (it must never open a writer —
|
|
349
|
+
see
|
|
350
|
+
``test_readonly_bandit_uses_uri_read_connection_and_never_records_play``),
|
|
351
|
+
so explicit feedback is the only path that grows these tables outside
|
|
352
|
+
the signal worker.
|
|
202
353
|
|
|
203
354
|
Args:
|
|
204
355
|
profile_id: Profile providing feedback.
|
|
@@ -206,15 +357,19 @@ class FeedbackCollector:
|
|
|
206
357
|
signal_type: One of ``user_positive``, ``user_negative``,
|
|
207
358
|
``user_correction``, or any custom type.
|
|
208
359
|
value: Numeric signal value (0.0 to 1.0).
|
|
360
|
+
query: Originating query. Stored only as a SHA-256[:16]
|
|
361
|
+
hash — full text is never persisted.
|
|
362
|
+
channel: Retrieval channel that surfaced the fact.
|
|
209
363
|
|
|
210
364
|
Returns:
|
|
211
|
-
|
|
365
|
+
A :class:`FeedbackWrite` describing exactly which rows landed.
|
|
212
366
|
"""
|
|
213
367
|
if not profile_id or not fact_id:
|
|
214
|
-
return None
|
|
368
|
+
return FeedbackWrite(None, None, False)
|
|
215
369
|
|
|
216
370
|
clamped = max(0.0, min(1.0, float(value)))
|
|
217
371
|
now = _utcnow_iso()
|
|
372
|
+
query_hash = _hash_query(query) if query else None
|
|
218
373
|
|
|
219
374
|
with self._lock:
|
|
220
375
|
conn = self._connect()
|
|
@@ -222,15 +377,165 @@ class FeedbackCollector:
|
|
|
222
377
|
cursor = conn.execute(
|
|
223
378
|
"INSERT INTO learning_feedback "
|
|
224
379
|
"(profile_id, fact_id, signal_type, signal_value, "
|
|
225
|
-
"query_hash, created_at, metadata) "
|
|
226
|
-
"VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
227
|
-
(profile_id, fact_id, signal_type, clamped,
|
|
380
|
+
"query_hash, created_at, metadata, channel) "
|
|
381
|
+
"VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
|
382
|
+
(profile_id, fact_id, signal_type, clamped, query_hash,
|
|
383
|
+
now, None, channel or "unknown"),
|
|
384
|
+
)
|
|
385
|
+
feedback_row_id = cursor.lastrowid
|
|
386
|
+
signal_row_id = self._insert_canonical_pair(
|
|
387
|
+
conn,
|
|
388
|
+
profile_id=profile_id,
|
|
389
|
+
fact_id=fact_id,
|
|
390
|
+
value=clamped,
|
|
391
|
+
query_hash=query_hash,
|
|
392
|
+
created_at=now,
|
|
393
|
+
feedback_row_id=feedback_row_id,
|
|
228
394
|
)
|
|
229
395
|
conn.commit()
|
|
230
|
-
return
|
|
396
|
+
return FeedbackWrite(
|
|
397
|
+
feedback_row_id, signal_row_id, signal_row_id is not None,
|
|
398
|
+
)
|
|
399
|
+
except sqlite3.Error:
|
|
400
|
+
conn.rollback()
|
|
401
|
+
raise
|
|
231
402
|
finally:
|
|
232
403
|
conn.close()
|
|
233
404
|
|
|
405
|
+
# ------------------------------------------------------------------
|
|
406
|
+
# Canonical store
|
|
407
|
+
# ------------------------------------------------------------------
|
|
408
|
+
|
|
409
|
+
def _insert_canonical_pair(
|
|
410
|
+
self,
|
|
411
|
+
conn: sqlite3.Connection,
|
|
412
|
+
*,
|
|
413
|
+
profile_id: str,
|
|
414
|
+
fact_id: str,
|
|
415
|
+
value: float,
|
|
416
|
+
query_hash: Optional[str],
|
|
417
|
+
created_at: str,
|
|
418
|
+
feedback_row_id: Optional[int],
|
|
419
|
+
) -> Optional[int]:
|
|
420
|
+
"""Insert the ``learning_signals`` + ``learning_features`` pair.
|
|
421
|
+
|
|
422
|
+
Runs inside the caller's open transaction so the canonical rows commit
|
|
423
|
+
with the legacy row or not at all. Returns the new signal row id, or
|
|
424
|
+
None when the canonical tables are absent — they are owned by the
|
|
425
|
+
migration runner (LLD-06 H15 forbids DDL here), so on a database that
|
|
426
|
+
predates them the caller is told the truth rather than handed a
|
|
427
|
+
fabricated success.
|
|
428
|
+
"""
|
|
429
|
+
if feedback_row_id is None:
|
|
430
|
+
return None
|
|
431
|
+
if not self._canonical_tables_present(conn):
|
|
432
|
+
return None
|
|
433
|
+
|
|
434
|
+
from superlocalmemory.learning.legacy_migration import legacy_query_id
|
|
435
|
+
|
|
436
|
+
query_id = legacy_query_id(feedback_row_id)
|
|
437
|
+
# Pad to 32 hex chars so an eagerly-written row has the same shape as
|
|
438
|
+
# both a migrated row and a fresh signal-worker row.
|
|
439
|
+
padded_hash = ((query_hash or "") + ("0" * 32))[:32]
|
|
440
|
+
|
|
441
|
+
cursor = conn.execute(
|
|
442
|
+
"INSERT INTO learning_signals "
|
|
443
|
+
"(profile_id, query, fact_id, signal_type, value, created_at, "
|
|
444
|
+
" query_id, query_text_hash, position, channel_scores, "
|
|
445
|
+
" cross_encoder) "
|
|
446
|
+
"VALUES (?, '', ?, ?, ?, ?, ?, ?, 0, '{}', NULL)",
|
|
447
|
+
(profile_id, fact_id, CANONICAL_SIGNAL_TYPE, value, created_at,
|
|
448
|
+
query_id, padded_hash),
|
|
449
|
+
)
|
|
450
|
+
signal_row_id = cursor.lastrowid
|
|
451
|
+
conn.execute(
|
|
452
|
+
"INSERT INTO learning_features "
|
|
453
|
+
"(profile_id, query_id, fact_id, features_json, label, "
|
|
454
|
+
" created_at, signal_id, is_synthetic) "
|
|
455
|
+
"VALUES (?, ?, ?, ?, ?, ?, ?, 1)",
|
|
456
|
+
(profile_id, query_id, fact_id, _SYNTHETIC_FEATURES_JSON,
|
|
457
|
+
value, created_at, signal_row_id),
|
|
458
|
+
)
|
|
459
|
+
return signal_row_id
|
|
460
|
+
|
|
461
|
+
def _bootstrap_canonical_schema(self) -> None:
|
|
462
|
+
"""Make sure the canonical store exists before any feedback arrives.
|
|
463
|
+
|
|
464
|
+
Explicit feedback can be the very first learning write on a machine —
|
|
465
|
+
a user can rate a recall before the daemon has ever run the migration
|
|
466
|
+
runner. Without this the write would honestly, but uselessly, report
|
|
467
|
+
that it never reached the store that gates ranking.
|
|
468
|
+
|
|
469
|
+
No DDL is authored here (LLD-06 H15). The base tables come from
|
|
470
|
+
``LearningDatabase``, which the migration runner itself calls as its
|
|
471
|
+
first-boot bootstrap, and the LLD-02 columns come from M001's own DDL.
|
|
472
|
+
Applying M001's DDL without a ``migration_log`` row is safe: when the
|
|
473
|
+
real runner reaches M001 its ALTERs fail, ``M001.verify`` passes, and
|
|
474
|
+
the runner records it as "already applied (verified via schema
|
|
475
|
+
inspection)".
|
|
476
|
+
|
|
477
|
+
Never fatal — a read-only or unwritable learning.db must not stop a
|
|
478
|
+
collector from being constructed.
|
|
479
|
+
"""
|
|
480
|
+
try:
|
|
481
|
+
from superlocalmemory.learning.database import LearningDatabase
|
|
482
|
+
from superlocalmemory.storage.migrations import (
|
|
483
|
+
M001_add_signal_features_columns as _m001,
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
LearningDatabase(self._db_path)
|
|
487
|
+
conn = self._connect()
|
|
488
|
+
try:
|
|
489
|
+
if not _m001.verify(conn):
|
|
490
|
+
conn.executescript(_m001.DDL)
|
|
491
|
+
conn.commit()
|
|
492
|
+
self._canonical_ready = _canonical_schema_ready(conn)
|
|
493
|
+
finally:
|
|
494
|
+
conn.close()
|
|
495
|
+
except Exception as exc: # noqa: BLE001 — construction must not fail
|
|
496
|
+
logger.warning(
|
|
497
|
+
"canonical learning schema bootstrap failed for %s: %s",
|
|
498
|
+
self._db_path, exc,
|
|
499
|
+
)
|
|
500
|
+
|
|
501
|
+
def _canonical_tables_present(self, conn: sqlite3.Connection) -> bool:
|
|
502
|
+
"""Return True when both canonical LLD-02 tables exist.
|
|
503
|
+
|
|
504
|
+
Re-probes while unready so a collector constructed before the
|
|
505
|
+
migration runner ran starts writing canonically as soon as the tables
|
|
506
|
+
appear, instead of degrading for the life of the process.
|
|
507
|
+
"""
|
|
508
|
+
if self._canonical_ready:
|
|
509
|
+
return True
|
|
510
|
+
self._canonical_ready = _canonical_schema_ready(conn)
|
|
511
|
+
if not self._canonical_ready:
|
|
512
|
+
logger.warning(
|
|
513
|
+
"learning.db at %s has no usable learning_signals/"
|
|
514
|
+
"learning_features schema; explicit feedback cannot reach the "
|
|
515
|
+
"store that gates adaptive ranking.",
|
|
516
|
+
self._db_path,
|
|
517
|
+
)
|
|
518
|
+
return self._canonical_ready
|
|
519
|
+
|
|
520
|
+
def get_signal_count(self, profile_id: str) -> int:
|
|
521
|
+
"""Return the canonical signal count that gates the ranking phase.
|
|
522
|
+
|
|
523
|
+
This is the single number the recall phase gate, the dashboard's
|
|
524
|
+
Living Brain panel, and the ranker-phase card all resolve their phase
|
|
525
|
+
from. Reporting anything else to a user — as ``report_feedback`` did
|
|
526
|
+
with ``feedback_records`` before issue #106 — shows progress toward a
|
|
527
|
+
threshold nothing is actually measuring.
|
|
528
|
+
"""
|
|
529
|
+
conn = self._connect()
|
|
530
|
+
try:
|
|
531
|
+
row = conn.execute(
|
|
532
|
+
"SELECT COUNT(*) FROM learning_signals WHERE profile_id = ?",
|
|
533
|
+
(profile_id,),
|
|
534
|
+
).fetchone()
|
|
535
|
+
return row[0] if row else 0
|
|
536
|
+
finally:
|
|
537
|
+
conn.close()
|
|
538
|
+
|
|
234
539
|
# ------------------------------------------------------------------
|
|
235
540
|
# Public API: record dashboard feedback
|
|
236
541
|
# ------------------------------------------------------------------
|
|
@@ -253,30 +558,26 @@ class FeedbackCollector:
|
|
|
253
558
|
This method restores the dashboard feedback path: the HTTP routes in
|
|
254
559
|
``server/routes/learning.py`` called it before it existed, so every
|
|
255
560
|
thumbs/pin/dwell write raised ``AttributeError`` (issues #53/#59).
|
|
561
|
+
|
|
562
|
+
Routed through :meth:`record_explicit_event` so a thumbs-up from the
|
|
563
|
+
dashboard lands in exactly the same canonical store as a thumbs-up
|
|
564
|
+
from MCP. Before issue #106 this path wrote only ``learning_feedback``,
|
|
565
|
+
so the dashboard's own Living Brain counter — which reads
|
|
566
|
+
``learning_signals`` — never moved in response to its own buttons.
|
|
256
567
|
"""
|
|
257
568
|
if not memory_id:
|
|
258
569
|
return None
|
|
259
570
|
signal_type, value = _DASHBOARD_SIGNAL_MAP.get(
|
|
260
571
|
feedback_type, ("user_correction", 0.5),
|
|
261
572
|
)
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
"(profile_id, fact_id, signal_type, signal_value, "
|
|
271
|
-
"query_hash, created_at, metadata) "
|
|
272
|
-
"VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
273
|
-
(profile_id or "default", str(memory_id), signal_type,
|
|
274
|
-
value, qhash, now, None),
|
|
275
|
-
)
|
|
276
|
-
conn.commit()
|
|
277
|
-
return cursor.lastrowid
|
|
278
|
-
finally:
|
|
279
|
-
conn.close()
|
|
573
|
+
return self.record_explicit_event(
|
|
574
|
+
profile_id=profile_id or "default",
|
|
575
|
+
fact_id=str(memory_id),
|
|
576
|
+
signal_type=signal_type,
|
|
577
|
+
value=value,
|
|
578
|
+
query=query,
|
|
579
|
+
channel="dashboard",
|
|
580
|
+
).feedback_row_id
|
|
280
581
|
|
|
281
582
|
# ------------------------------------------------------------------
|
|
282
583
|
# Public API: read feedback
|
|
@@ -41,6 +41,20 @@ MIGRATION_NAME = "LEG001_feedback_to_signals"
|
|
|
41
41
|
_COPY_BATCH_SIZE = 500
|
|
42
42
|
|
|
43
43
|
|
|
44
|
+
def legacy_query_id(feedback_row_id: int | str) -> str:
|
|
45
|
+
"""Return the canonical ``learning_signals.query_id`` for a feedback row.
|
|
46
|
+
|
|
47
|
+
One explicit-feedback event has exactly ONE canonical identity, whichever
|
|
48
|
+
path writes it: this batch migration, or ``FeedbackCollector`` writing the
|
|
49
|
+
canonical pair eagerly at feedback time. Both derive the id from the
|
|
50
|
+
``learning_feedback`` row id through this function, which is what lets the
|
|
51
|
+
migration recognise — and skip — rows already carried forward. Without a
|
|
52
|
+
shared identity the two writers would double-count the same event into the
|
|
53
|
+
store that gates the ranking phase.
|
|
54
|
+
"""
|
|
55
|
+
return f"legacy:{feedback_row_id}"
|
|
56
|
+
|
|
57
|
+
|
|
44
58
|
def migrate_legacy_feedback(
|
|
45
59
|
learning_db: Path,
|
|
46
60
|
*,
|
|
@@ -156,10 +170,18 @@ def _copy_rows(conn: sqlite3.Connection) -> tuple[int, int]:
|
|
|
156
170
|
|
|
157
171
|
Returns ``(copied, failed)``. Does not raise. Commits per batch so
|
|
158
172
|
a later failure still leaves the earlier batches durable.
|
|
173
|
+
|
|
174
|
+
Rows whose canonical ``query_id`` is already present in
|
|
175
|
+
``learning_signals`` are skipped rather than copied a second time.
|
|
176
|
+
``FeedbackCollector`` writes the canonical pair at feedback time, so on
|
|
177
|
+
any install where this migration has not yet been sentinel-marked the
|
|
178
|
+
newest rows are already carried forward; copying them again would
|
|
179
|
+
inflate the very counter that gates the ranking phase.
|
|
159
180
|
"""
|
|
160
181
|
copied = 0
|
|
161
182
|
failed = 0
|
|
162
183
|
offset = 0
|
|
184
|
+
already_present = _existing_legacy_query_ids(conn)
|
|
163
185
|
while True:
|
|
164
186
|
try:
|
|
165
187
|
batch = conn.execute(
|
|
@@ -179,6 +201,8 @@ def _copy_rows(conn: sqlite3.Connection) -> tuple[int, int]:
|
|
|
179
201
|
try:
|
|
180
202
|
conn.execute("BEGIN IMMEDIATE")
|
|
181
203
|
for row in batch:
|
|
204
|
+
if legacy_query_id(row["id"]) in already_present:
|
|
205
|
+
continue
|
|
182
206
|
try:
|
|
183
207
|
_copy_single_row(conn, row)
|
|
184
208
|
copied += 1
|
|
@@ -199,6 +223,23 @@ def _copy_rows(conn: sqlite3.Connection) -> tuple[int, int]:
|
|
|
199
223
|
return copied, failed
|
|
200
224
|
|
|
201
225
|
|
|
226
|
+
def _existing_legacy_query_ids(conn: sqlite3.Connection) -> set[str]:
|
|
227
|
+
"""Return every ``legacy:`` query_id already present in learning_signals.
|
|
228
|
+
|
|
229
|
+
Read once up front: a per-row EXISTS probe over a signals table that grows
|
|
230
|
+
to tens of thousands of rows turns an O(n) copy into O(n*m).
|
|
231
|
+
"""
|
|
232
|
+
try:
|
|
233
|
+
rows = conn.execute(
|
|
234
|
+
"SELECT DISTINCT query_id FROM learning_signals "
|
|
235
|
+
"WHERE query_id LIKE 'legacy:%'",
|
|
236
|
+
).fetchall()
|
|
237
|
+
except sqlite3.Error as exc:
|
|
238
|
+
logger.warning("legacy migration: dedupe probe failed: %s", exc)
|
|
239
|
+
return set()
|
|
240
|
+
return {str(row[0]) for row in rows}
|
|
241
|
+
|
|
242
|
+
|
|
202
243
|
def _copy_single_row(conn: sqlite3.Connection, row: sqlite3.Row) -> None:
|
|
203
244
|
"""Insert one legacy row into learning_signals + learning_features.
|
|
204
245
|
|
|
@@ -218,7 +259,7 @@ def _copy_single_row(conn: sqlite3.Connection, row: sqlite3.Row) -> None:
|
|
|
218
259
|
datetime.now(timezone.utc).isoformat(timespec="seconds"))
|
|
219
260
|
profile_id = str(row["profile_id"] or "default")
|
|
220
261
|
fact_id = str(row["fact_id"] or "")
|
|
221
|
-
|
|
262
|
+
query_id = legacy_query_id(row["id"])
|
|
222
263
|
|
|
223
264
|
# Insert the signal row. ``signal_type='legacy_feedback'`` marks it
|
|
224
265
|
# clearly so consumers (dashboard, labeler) can treat it correctly.
|
|
@@ -229,7 +270,7 @@ def _copy_single_row(conn: sqlite3.Connection, row: sqlite3.Row) -> None:
|
|
|
229
270
|
"VALUES (?, '', ?, 'legacy_feedback', ?, ?, ?, ?, 0, '{}', NULL)",
|
|
230
271
|
(profile_id, fact_id,
|
|
231
272
|
float(row["signal_value"] or 1.0),
|
|
232
|
-
created_at,
|
|
273
|
+
created_at, query_id, query_hash),
|
|
233
274
|
)
|
|
234
275
|
sid = cur.lastrowid
|
|
235
276
|
|
|
@@ -241,7 +282,7 @@ def _copy_single_row(conn: sqlite3.Connection, row: sqlite3.Row) -> None:
|
|
|
241
282
|
"(profile_id, query_id, fact_id, features_json, label, created_at, "
|
|
242
283
|
" signal_id, is_synthetic) "
|
|
243
284
|
"VALUES (?, ?, ?, '{}', 0.0, ?, ?, 1)",
|
|
244
|
-
(profile_id,
|
|
285
|
+
(profile_id, query_id, fact_id, created_at, sid),
|
|
245
286
|
)
|
|
246
287
|
|
|
247
288
|
|
|
@@ -274,4 +315,4 @@ def _record_migration(
|
|
|
274
315
|
logger.warning("legacy migration: log record failed: %s", exc)
|
|
275
316
|
|
|
276
317
|
|
|
277
|
-
__all__ = ("migrate_legacy_feedback", "MIGRATION_NAME")
|
|
318
|
+
__all__ = ("migrate_legacy_feedback", "MIGRATION_NAME", "legacy_query_id")
|
|
@@ -325,14 +325,28 @@ def _mine_channel_and_coretrieval(
|
|
|
325
325
|
learn_conn = sqlite3.connect(learning_db, timeout=10)
|
|
326
326
|
learn_conn.row_factory = sqlite3.Row
|
|
327
327
|
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
328
|
+
# Isolated from the co-retrieval block below. Until 3.8.11
|
|
329
|
+
# ``learning_feedback`` had no ``channel`` column, so this query
|
|
330
|
+
# raised and — sharing one try with co-retrieval — took that mining
|
|
331
|
+
# down with it. Two pattern types died from one missing column, and
|
|
332
|
+
# the only trace was a DEBUG line. Each miner now fails alone, loudly.
|
|
333
|
+
channel_rows = []
|
|
334
|
+
try:
|
|
335
|
+
channel_rows = learn_conn.execute(
|
|
336
|
+
"SELECT channel, COUNT(*) AS cnt, "
|
|
337
|
+
"AVG(signal_value) AS avg_signal "
|
|
338
|
+
"FROM learning_feedback "
|
|
339
|
+
"WHERE profile_id = ? "
|
|
340
|
+
"GROUP BY channel ORDER BY cnt DESC",
|
|
341
|
+
(profile_id,),
|
|
342
|
+
).fetchall()
|
|
343
|
+
except sqlite3.Error as exc:
|
|
344
|
+
logger.warning(
|
|
345
|
+
"Channel pattern mining skipped — learning_feedback query "
|
|
346
|
+
"failed (%s). Run 'slm db migrate' to apply M033 if this "
|
|
347
|
+
"reports a missing 'channel' column. Co-retrieval mining "
|
|
348
|
+
"continues.", exc,
|
|
349
|
+
)
|
|
336
350
|
|
|
337
351
|
for row in channel_rows:
|
|
338
352
|
d = dict(row)
|
|
@@ -376,12 +390,18 @@ def _mine_channel_and_coretrieval(
|
|
|
376
390
|
confidence=min(1.0, len(coret_rows) / 10),
|
|
377
391
|
)
|
|
378
392
|
gen += 1
|
|
379
|
-
except
|
|
380
|
-
|
|
393
|
+
except sqlite3.Error as exc:
|
|
394
|
+
logger.warning(
|
|
395
|
+
"Co-retrieval pattern mining skipped — co_retrieval_edges "
|
|
396
|
+
"query failed: %s", exc,
|
|
397
|
+
)
|
|
381
398
|
|
|
382
399
|
learn_conn.close()
|
|
383
400
|
except Exception as exc:
|
|
384
|
-
|
|
401
|
+
# Was DEBUG. A learning subsystem that mines nothing must say so at a
|
|
402
|
+
# level operators actually see; issue #102 went undiagnosed for weeks
|
|
403
|
+
# because the only evidence was invisible by default.
|
|
404
|
+
logger.warning("Signal pattern mining failed: %s", exc)
|
|
385
405
|
return gen
|
|
386
406
|
|
|
387
407
|
|
|
@@ -23,6 +23,28 @@ from typing import Any
|
|
|
23
23
|
|
|
24
24
|
logger = logging.getLogger(__name__)
|
|
25
25
|
|
|
26
|
+
_OPAQUE_UNAVAILABLE = "DAEMON_UNAVAILABLE: owned daemon is unavailable; retry later."
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def daemon_unavailable_error() -> str:
|
|
30
|
+
"""Return a one-line, *diagnosed* daemon-unavailable message.
|
|
31
|
+
|
|
32
|
+
The opaque wording this replaces described a stopped daemon, a recycled
|
|
33
|
+
PID, an unreachable port and an identity mismatch identically (issue #104).
|
|
34
|
+
Diagnosis is best effort: if it fails for any reason the caller still gets
|
|
35
|
+
the original, retryable message rather than an exception.
|
|
36
|
+
"""
|
|
37
|
+
try:
|
|
38
|
+
from superlocalmemory.cli.daemon import describe_daemon_unavailability
|
|
39
|
+
|
|
40
|
+
diagnosis = describe_daemon_unavailability()
|
|
41
|
+
return (
|
|
42
|
+
f"DAEMON_UNAVAILABLE ({diagnosis['reason']}): "
|
|
43
|
+
f"{diagnosis['message']} {diagnosis['hint']}"
|
|
44
|
+
)
|
|
45
|
+
except Exception: # noqa: BLE001 - diagnosis must never mask the failure
|
|
46
|
+
return _OPAQUE_UNAVAILABLE
|
|
47
|
+
|
|
26
48
|
|
|
27
49
|
class DaemonPoolProxy:
|
|
28
50
|
""":class:`WorkerPool`-shaped facade that talks to the daemon over HTTP.
|
|
@@ -52,7 +74,7 @@ class DaemonPoolProxy:
|
|
|
52
74
|
"ok": False,
|
|
53
75
|
"code": "DAEMON_UNAVAILABLE",
|
|
54
76
|
"retryable": True,
|
|
55
|
-
"error":
|
|
77
|
+
"error": daemon_unavailable_error(),
|
|
56
78
|
}
|
|
57
79
|
|
|
58
80
|
def recall(
|