superlocalmemory 3.8.11 → 3.8.13

package/src/superlocalmemory/learning/feedback.py

@@ -15,7 +15,25 @@ Privacy:
     - Queries are hashed to SHA-256[:16] for grouping.
 
 Storage:
-    Uses direct sqlite3 with a self-contained ``learning_feedback`` table.
+    Every explicit-feedback event is written to the CANONICAL learning store
+    -- a ``learning_signals`` row paired 1:1 with a ``learning_features`` row
+    -- in the same transaction as the historic ``learning_feedback`` row.
+
+    ``learning_signals`` is canonical because every live consumer already
+    reads it: the dashboard's Living Brain panel and ranker-phase card
+    (``server/routes/brain.py``, ``server/routes/learning.py``), the LightGBM
+    retrainer, and -- since issue #106 -- the recall phase gate.
+    ``learning_feedback`` is the pre-v3.4.22 table: ``legacy_migration``
+    copies it forward into ``learning_signals``, the dashboard reports it as
+    ``legacy_feedback_rows`` with a "pending migration" card, and the phase
+    gate's own docstring calls it legacy. Writing feedback only there (the
+    v3.8.11 attempt at issue #102) put the durable write in a table no phase
+    counter consumes, which is why reported feedback still changed nothing.
+
+    It is kept written for one more release (LLD-07 D5) so ``pattern_miner``
+    channel mining and GDPR erasure keep working; the shared identity from
+    ``legacy_migration.legacy_query_id`` stops the two writers double-counting.
+
     NOT coupled to V3 DatabaseManager -- this is a standalone data collector.
 """
 
@@ -25,6 +43,7 @@ import hashlib
 import logging
 import sqlite3
 import threading
+from dataclasses import dataclass
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, Dict, List, Optional
@@ -85,11 +104,72 @@ CREATE INDEX IF NOT EXISTS idx_feedback_channel
 """
 
 
+# Signal type stamped on the canonical ``learning_signals`` row for an
+# explicit-feedback event. Identical to what ``legacy_migration`` writes when
+# it carries a ``learning_feedback`` row forward, so a row recorded eagerly
+# and a row migrated in batch are indistinguishable to every consumer.
+CANONICAL_SIGNAL_TYPE = "legacy_feedback"
+
+# ``learning_features.features_json`` for a feedback event. Feedback arrives
+# out of band -- there is no ranked candidate list to extract a real feature
+# vector from -- so the row is empty and flagged ``is_synthetic=1``. The
+# LightGBM retrainer selects ``WHERE is_synthetic=0``, so these rows move the
+# phase counters and the bandit without ever polluting model training.
+_SYNTHETIC_FEATURES_JSON = "{}"
+
+
+@dataclass(frozen=True)
+class FeedbackWrite:
+    """Outcome of one explicit-feedback write.
+
+    ``canonical`` is the only field callers should gate user-facing success
+    on: it is True when the ``learning_signals`` row that every phase counter
+    reads actually landed. ``feedback_row_id`` alone means the legacy row was
+    written, which on its own influences nothing.
+    """
+
+    feedback_row_id: Optional[int]
+    signal_row_id: Optional[int]
+    canonical: bool
+
+
 def _utcnow_iso() -> str:
     """Return current UTC time as ISO-8601 string."""
     return datetime.now(timezone.utc).isoformat()
 
 
+def _canonical_schema_ready(conn: sqlite3.Connection) -> bool:
+    """Return True when learning.db can accept a canonical feedback event.
+
+    Both tables must exist AND carry the LLD-02 columns the event needs
+    (``learning_signals.query_id`` for the shared identity that keeps the
+    batch migration from double-counting, ``learning_features.is_synthetic``
+    for the flag that keeps these rows out of LightGBM training). A table
+    that exists without them cannot hold the event correctly, so treating
+    mere existence as readiness would write a row that silently breaks both
+    invariants.
+    """
+    try:
+        signal_cols = {
+            row[1] for row in conn.execute(
+                "PRAGMA table_info(learning_signals)",
+            )
+        }
+        feature_cols = {
+            row[1] for row in conn.execute(
+                "PRAGMA table_info(learning_features)",
+            )
+        }
+    except sqlite3.Error:
+        return False
+    return (
+        "query_id" in signal_cols
+        and "query_text_hash" in signal_cols
+        and "signal_id" in feature_cols
+        and "is_synthetic" in feature_cols
+    )
+
+
 def _hash_query(query: str) -> str:
     """Privacy-preserving SHA-256[:16] query hash."""
     return hashlib.sha256(query.encode("utf-8")).hexdigest()[:16]
@@ -109,7 +189,12 @@ class FeedbackCollector:
     def __init__(self, db_path: Path) -> None:
         self._db_path = Path(db_path)
         self._lock = threading.Lock()
+        # Latched once the canonical LLD-02 tables are confirmed present, so
+        # the sqlite_master probe runs at most once per collector instead of
+        # on every feedback write.
+        self._canonical_ready = False
         self._ensure_schema()
+        self._bootstrap_canonical_schema()
 
     # ------------------------------------------------------------------
     # Schema
@@ -226,16 +311,45 @@ class FeedbackCollector:
         query: str = "",
         channel: str = "unknown",
     ) -> Optional[int]:
+        """Record explicit user feedback on a specific fact.
+
+        Back-compatible wrapper: returns the ``learning_feedback`` row id.
+        Callers that must tell a user whether the feedback actually reached
+        the store the phase counters read should use
+        :meth:`record_explicit_event` and check ``FeedbackWrite.canonical`` —
+        a legacy row id on its own influences no consumer.
         """
-        Record explicit user feedback on a specific fact.
+        return self.record_explicit_event(
+            profile_id=profile_id, fact_id=fact_id, signal_type=signal_type,
+            value=value, query=query, channel=channel,
+        ).feedback_row_id
 
-        This is the canonical durable write for the learning system. Recall
-        itself is deliberately read-only (it must never open a writer — see
+    def record_explicit_event(
+        self,
+        profile_id: str,
+        fact_id: str,
+        signal_type: str,
+        value: float,
+        query: str = "",
+        channel: str = "unknown",
+    ) -> FeedbackWrite:
+        """
+        Record explicit user feedback as ONE atomic learning event.
+
+        Writes three rows in a single transaction: the historic
+        ``learning_feedback`` row (kept one more release for ``pattern_miner``
+        channel mining and GDPR erasure) plus the canonical
+        ``learning_signals`` + ``learning_features`` pair that the dashboard,
+        the recall phase gate, and the retrainer all read. Either the whole
+        event is durable or none of it is — a partial write would leave the
+        legacy table and the phase counters permanently disagreeing, which is
+        the shape of issue #106.
+
+        Recall itself is deliberately read-only (it must never open a writer —
+        see
         ``test_readonly_bandit_uses_uri_read_connection_and_never_records_play``),
-        so explicit feedback is the ONLY path that grows ``learning_feedback``.
-        Every downstream consumer reads this table: the phase gate
-        (``_ReadOnlyLearningView.count_feedback`` unlocks adaptive ranking at
-        50 rows), ``pattern_miner`` (channel_performance), and the dashboard.
+        so explicit feedback is the only path that grows these tables outside
+        the signal worker.
 
         Args:
             profile_id:  Profile providing feedback.
@@ -248,10 +362,10 @@ class FeedbackCollector:
             channel:     Retrieval channel that surfaced the fact.
 
         Returns:
-            Row ID of the inserted record, or None on error.
+            A :class:`FeedbackWrite` describing exactly which rows landed.
         """
         if not profile_id or not fact_id:
-            return None
+            return FeedbackWrite(None, None, False)
 
         clamped = max(0.0, min(1.0, float(value)))
         now = _utcnow_iso()
@@ -268,10 +382,159 @@ class FeedbackCollector:
                     (profile_id, fact_id, signal_type, clamped, query_hash,
                      now, None, channel or "unknown"),
                 )
+                feedback_row_id = cursor.lastrowid
+                signal_row_id = self._insert_canonical_pair(
+                    conn,
+                    profile_id=profile_id,
+                    fact_id=fact_id,
+                    value=clamped,
+                    query_hash=query_hash,
+                    created_at=now,
+                    feedback_row_id=feedback_row_id,
+                )
                 conn.commit()
-                return cursor.lastrowid
+                return FeedbackWrite(
+                    feedback_row_id, signal_row_id, signal_row_id is not None,
+                )
+            except sqlite3.Error:
+                conn.rollback()
+                raise
+            finally:
+                conn.close()
+
+    # ------------------------------------------------------------------
+    # Canonical store
+    # ------------------------------------------------------------------
+
+    def _insert_canonical_pair(
+        self,
+        conn: sqlite3.Connection,
+        *,
+        profile_id: str,
+        fact_id: str,
+        value: float,
+        query_hash: Optional[str],
+        created_at: str,
+        feedback_row_id: Optional[int],
+    ) -> Optional[int]:
+        """Insert the ``learning_signals`` + ``learning_features`` pair.
+
+        Runs inside the caller's open transaction so the canonical rows commit
+        with the legacy row or not at all. Returns the new signal row id, or
+        None when the canonical tables are absent — they are owned by the
+        migration runner (LLD-06 H15 forbids DDL here), so on a database that
+        predates them the caller is told the truth rather than handed a
+        fabricated success.
+        """
+        if feedback_row_id is None:
+            return None
+        if not self._canonical_tables_present(conn):
+            return None
+
+        from superlocalmemory.learning.legacy_migration import legacy_query_id
+
+        query_id = legacy_query_id(feedback_row_id)
+        # Pad to 32 hex chars so an eagerly-written row has the same shape as
+        # both a migrated row and a fresh signal-worker row.
+        padded_hash = ((query_hash or "") + ("0" * 32))[:32]
+
+        cursor = conn.execute(
+            "INSERT INTO learning_signals "
+            "(profile_id, query, fact_id, signal_type, value, created_at, "
+            " query_id, query_text_hash, position, channel_scores, "
+            " cross_encoder) "
+            "VALUES (?, '', ?, ?, ?, ?, ?, ?, 0, '{}', NULL)",
+            (profile_id, fact_id, CANONICAL_SIGNAL_TYPE, value, created_at,
+             query_id, padded_hash),
+        )
+        signal_row_id = cursor.lastrowid
+        conn.execute(
+            "INSERT INTO learning_features "
+            "(profile_id, query_id, fact_id, features_json, label, "
+            " created_at, signal_id, is_synthetic) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, 1)",
+            (profile_id, query_id, fact_id, _SYNTHETIC_FEATURES_JSON,
+             value, created_at, signal_row_id),
+        )
+        return signal_row_id
+
+    def _bootstrap_canonical_schema(self) -> None:
+        """Make sure the canonical store exists before any feedback arrives.
+
+        Explicit feedback can be the very first learning write on a machine —
+        a user can rate a recall before the daemon has ever run the migration
+        runner. Without this the write would honestly, but uselessly, report
+        that it never reached the store that gates ranking.
+
+        No DDL is authored here (LLD-06 H15). The base tables come from
+        ``LearningDatabase``, which the migration runner itself calls as its
+        first-boot bootstrap, and the LLD-02 columns come from M001's own DDL.
+        Applying M001's DDL without a ``migration_log`` row is safe: when the
+        real runner reaches M001 its ALTERs fail, ``M001.verify`` passes, and
+        the runner records it as "already applied (verified via schema
+        inspection)".
+
+        Never fatal — a read-only or unwritable learning.db must not stop a
+        collector from being constructed.
+        """
+        try:
+            from superlocalmemory.learning.database import LearningDatabase
+            from superlocalmemory.storage.migrations import (
+                M001_add_signal_features_columns as _m001,
+            )
+
+            LearningDatabase(self._db_path)
+            conn = self._connect()
+            try:
+                if not _m001.verify(conn):
+                    conn.executescript(_m001.DDL)
+                    conn.commit()
+                self._canonical_ready = _canonical_schema_ready(conn)
             finally:
                 conn.close()
+        except Exception as exc:  # noqa: BLE001 — construction must not fail
+            logger.warning(
+                "canonical learning schema bootstrap failed for %s: %s",
+                self._db_path, exc,
+            )
+
+    def _canonical_tables_present(self, conn: sqlite3.Connection) -> bool:
+        """Return True when both canonical LLD-02 tables exist.
+
+        Re-probes while unready so a collector constructed before the
+        migration runner ran starts writing canonically as soon as the tables
+        appear, instead of degrading for the life of the process.
+        """
+        if self._canonical_ready:
+            return True
+        self._canonical_ready = _canonical_schema_ready(conn)
+        if not self._canonical_ready:
+            logger.warning(
+                "learning.db at %s has no usable learning_signals/"
+                "learning_features schema; explicit feedback cannot reach the "
+                "store that gates adaptive ranking.",
+                self._db_path,
+            )
+        return self._canonical_ready
+
+    def get_signal_count(self, profile_id: str) -> int:
+        """Return the canonical signal count that gates the ranking phase.
+
+        This is the single number the recall phase gate, the dashboard's
+        Living Brain panel, and the ranker-phase card all resolve their phase
+        from. Reporting anything else to a user — as ``report_feedback`` did
+        with ``feedback_records`` before issue #106 — shows progress toward a
+        threshold nothing is actually measuring.
+        """
+        conn = self._connect()
+        try:
+            row = conn.execute(
+                "SELECT COUNT(*) FROM learning_signals WHERE profile_id = ?",
+                (profile_id,),
+            ).fetchone()
+            return row[0] if row else 0
+        finally:
+            conn.close()
 
     # ------------------------------------------------------------------
     # Public API: record dashboard feedback
@@ -295,30 +558,26 @@ class FeedbackCollector:
         This method restores the dashboard feedback path: the HTTP routes in
         ``server/routes/learning.py`` called it before it existed, so every
         thumbs/pin/dwell write raised ``AttributeError`` (issues #53/#59).
+
+        Routed through :meth:`record_explicit_event` so a thumbs-up from the
+        dashboard lands in exactly the same canonical store as a thumbs-up
+        from MCP. Before issue #106 this path wrote only ``learning_feedback``,
+        so the dashboard's own Living Brain counter — which reads
+        ``learning_signals`` — never moved in response to its own buttons.
         """
         if not memory_id:
             return None
         signal_type, value = _DASHBOARD_SIGNAL_MAP.get(
             feedback_type, ("user_correction", 0.5),
         )
-        qhash = _hash_query(query) if query else None
-        now = _utcnow_iso()
-
-        with self._lock:
-            conn = self._connect()
-            try:
-                cursor = conn.execute(
-                    "INSERT INTO learning_feedback "
-                    "(profile_id, fact_id, signal_type, signal_value, "
-                    "query_hash, created_at, metadata) "
-                    "VALUES (?, ?, ?, ?, ?, ?, ?)",
-                    (profile_id or "default", str(memory_id), signal_type,
-                     value, qhash, now, None),
-                )
-                conn.commit()
-                return cursor.lastrowid
-            finally:
-                conn.close()
+        return self.record_explicit_event(
+            profile_id=profile_id or "default",
+            fact_id=str(memory_id),
+            signal_type=signal_type,
+            value=value,
+            query=query,
+            channel="dashboard",
+        ).feedback_row_id
 
     # ------------------------------------------------------------------
     # Public API: read feedback

package/src/superlocalmemory/learning/legacy_migration.py

@@ -41,6 +41,20 @@ MIGRATION_NAME = "LEG001_feedback_to_signals"
 _COPY_BATCH_SIZE = 500
 
 
+def legacy_query_id(feedback_row_id: int | str) -> str:
+    """Return the canonical ``learning_signals.query_id`` for a feedback row.
+
+    One explicit-feedback event has exactly ONE canonical identity, whichever
+    path writes it: this batch migration, or ``FeedbackCollector`` writing the
+    canonical pair eagerly at feedback time. Both derive the id from the
+    ``learning_feedback`` row id through this function, which is what lets the
+    migration recognise — and skip — rows already carried forward. Without a
+    shared identity the two writers would double-count the same event into the
+    store that gates the ranking phase.
+    """
+    return f"legacy:{feedback_row_id}"
+
+
 def migrate_legacy_feedback(
     learning_db: Path,
     *,
@@ -156,10 +170,18 @@ def _copy_rows(conn: sqlite3.Connection) -> tuple[int, int]:
 
     Returns ``(copied, failed)``. Does not raise. Commits per batch so
     a later failure still leaves the earlier batches durable.
+
+    Rows whose canonical ``query_id`` is already present in
+    ``learning_signals`` are skipped rather than copied a second time.
+    ``FeedbackCollector`` writes the canonical pair at feedback time, so on
+    any install where this migration has not yet been sentinel-marked the
+    newest rows are already carried forward; copying them again would
+    inflate the very counter that gates the ranking phase.
     """
     copied = 0
     failed = 0
     offset = 0
+    already_present = _existing_legacy_query_ids(conn)
     while True:
         try:
             batch = conn.execute(
@@ -179,6 +201,8 @@ def _copy_rows(conn: sqlite3.Connection) -> tuple[int, int]:
         try:
             conn.execute("BEGIN IMMEDIATE")
             for row in batch:
+                if legacy_query_id(row["id"]) in already_present:
+                    continue
                 try:
                     _copy_single_row(conn, row)
                     copied += 1
@@ -199,6 +223,23 @@ def _copy_rows(conn: sqlite3.Connection) -> tuple[int, int]:
     return copied, failed
 
 
+def _existing_legacy_query_ids(conn: sqlite3.Connection) -> set[str]:
+    """Return every ``legacy:`` query_id already present in learning_signals.
+
+    Read once up front: a per-row EXISTS probe over a signals table that grows
+    to tens of thousands of rows turns an O(n) copy into O(n*m).
+    """
+    try:
+        rows = conn.execute(
+            "SELECT DISTINCT query_id FROM learning_signals "
+            "WHERE query_id LIKE 'legacy:%'",
+        ).fetchall()
+    except sqlite3.Error as exc:
+        logger.warning("legacy migration: dedupe probe failed: %s", exc)
+        return set()
+    return {str(row[0]) for row in rows}
+
+
 def _copy_single_row(conn: sqlite3.Connection, row: sqlite3.Row) -> None:
     """Insert one legacy row into learning_signals + learning_features.
 
@@ -218,7 +259,7 @@ def _copy_single_row(conn: sqlite3.Connection, row: sqlite3.Row) -> None:
                      datetime.now(timezone.utc).isoformat(timespec="seconds"))
     profile_id = str(row["profile_id"] or "default")
     fact_id = str(row["fact_id"] or "")
-    legacy_query_id = f"legacy:{row['id']}"
+    query_id = legacy_query_id(row["id"])
 
     # Insert the signal row. ``signal_type='legacy_feedback'`` marks it
     # clearly so consumers (dashboard, labeler) can treat it correctly.
@@ -229,7 +270,7 @@ def _copy_single_row(conn: sqlite3.Connection, row: sqlite3.Row) -> None:
         "VALUES (?, '', ?, 'legacy_feedback', ?, ?, ?, ?, 0, '{}', NULL)",
         (profile_id, fact_id,
          float(row["signal_value"] or 1.0),
-         created_at, legacy_query_id, query_hash),
+         created_at, query_id, query_hash),
     )
     sid = cur.lastrowid
 
@@ -241,7 +282,7 @@ def _copy_single_row(conn: sqlite3.Connection, row: sqlite3.Row) -> None:
         "(profile_id, query_id, fact_id, features_json, label, created_at, "
         " signal_id, is_synthetic) "
         "VALUES (?, ?, ?, '{}', 0.0, ?, ?, 1)",
-        (profile_id, legacy_query_id, fact_id, created_at, sid),
+        (profile_id, query_id, fact_id, created_at, sid),
     )
 
 
@@ -274,4 +315,4 @@ def _record_migration(
         logger.warning("legacy migration: log record failed: %s", exc)
 
 
-__all__ = ("migrate_legacy_feedback", "MIGRATION_NAME")
+__all__ = ("migrate_legacy_feedback", "MIGRATION_NAME", "legacy_query_id")

package/src/superlocalmemory/mcp/_daemon_proxy.py

@@ -23,6 +23,28 @@ from typing import Any
 
 logger = logging.getLogger(__name__)
 
+_OPAQUE_UNAVAILABLE = "DAEMON_UNAVAILABLE: owned daemon is unavailable; retry later."
+
+
+def daemon_unavailable_error() -> str:
+    """Return a one-line, *diagnosed* daemon-unavailable message.
+
+    The opaque wording this replaces described a stopped daemon, a recycled
+    PID, an unreachable port and an identity mismatch identically (issue #104).
+    Diagnosis is best effort: if it fails for any reason the caller still gets
+    the original, retryable message rather than an exception.
+    """
+    try:
+        from superlocalmemory.cli.daemon import describe_daemon_unavailability
+
+        diagnosis = describe_daemon_unavailability()
+        return (
+            f"DAEMON_UNAVAILABLE ({diagnosis['reason']}): "
+            f"{diagnosis['message']} {diagnosis['hint']}"
+        )
+    except Exception:  # noqa: BLE001 - diagnosis must never mask the failure
+        return _OPAQUE_UNAVAILABLE
+
 
 class DaemonPoolProxy:
     """:class:`WorkerPool`-shaped facade that talks to the daemon over HTTP.
@@ -52,7 +74,7 @@ class DaemonPoolProxy:
             "ok": False,
             "code": "DAEMON_UNAVAILABLE",
             "retryable": True,
-            "error": "DAEMON_UNAVAILABLE: owned daemon is unavailable; retry later.",
+            "error": daemon_unavailable_error(),
         }
 
     def recall(