superlocalmemory 3.4.18 → 3.4.21

package/src/superlocalmemory/learning/labeler.py

@@ -0,0 +1,87 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory v3.4.21 — LLD-02 §4.7
+
+"""Integer-label mapping for LightGBM ``lambdarank`` training.
+
+LLD reference: ``.backup/active-brain/lld/LLD-02-signal-pipeline-and-lightgbm.md``
+Section 4.7 — single source of truth for outcome-reward / position → int label.
+
+Rules:
+    - Labels are integers in ``[0, 4]`` (5 relevance tiers).
+    - ``label_gain`` length MUST be ``>= max(label) + 1``; we ship
+      ``label_gain=[0, 1, 3, 7, 15]`` (length 5).
+    - Prefer ``outcome_reward`` (v3.4.21) if present; else position proxy.
+    - ``NaN`` reward is treated as missing and falls through to position.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+# Exactly five tiers — do not widen without updating ``label_gain`` callers.
+_LABEL_GAIN: tuple[int, ...] = (0, 1, 3, 7, 15)
+
+
+def label_gain() -> list[int]:
+    """Return the canonical ``label_gain`` list for LightGBM.
+
+    Length is ``max(label) + 1 = 5``. Must be passed verbatim to
+    ``lgb.train(params=..., label_gain=...)``.
+    """
+    return list(_LABEL_GAIN)
+
+
+def _coerce_reward(raw: Any) -> float | None:
+    """Coerce a reward-ish input to float, rejecting None / NaN."""
+    if raw is None:
+        return None
+    try:
+        val = float(raw)
+    except (TypeError, ValueError):
+        return None
+    if math.isnan(val):
+        return None
+    return val
+
+
+def label_for_row(row: dict) -> int:
+    """Map a training row to integer relevance in ``[0, 4]``.
+
+    Args:
+        row: Dict with optional ``outcome_reward`` (float in [0, 1]) and
+             ``position`` (int, 0-based rank at recall time).
+
+    Returns:
+        An integer label in ``[0, 4]``. Higher = more relevant.
+    """
+    reward = _coerce_reward(row.get("outcome_reward"))
+    if reward is not None:
+        if reward >= 0.90:
+            return 4
+        if reward >= 0.60:
+            return 3
+        if reward >= 0.30:
+            return 2
+        if reward > 0.00:
+            return 1
+        return 0
+
+    # 3.4.21 proxy: position (0 = best, higher = worse).
+    try:
+        pos = int(row.get("position", 99))
+    except (TypeError, ValueError):
+        return 0
+    if pos == 0:
+        return 4
+    if pos <= 2:
+        return 3
+    if pos <= 4:
+        return 2
+    if pos <= 9:
+        return 1
+    return 0
+
+
+__all__ = ("label_for_row", "label_gain")

package/src/superlocalmemory/learning/legacy_migration.py

@@ -0,0 +1,277 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory v3.4.21
+
+"""Legacy ``learning_feedback`` row migration — data movement only.
+
+LLD reference: ``.backup/active-brain/lld/LLD-07-schema-migrations-and-security-primitives.md``
+Section 5 (Legacy learning_feedback Migration).
+
+Hard rule H15 (LLD-06 §10): this module MUST NOT contain any schema
+DDL. All schema definitions live in
+``src/superlocalmemory/storage/migrations/`` modules. This file is
+data-copy only — the tables it reads from and writes into are created
+by the LLD-07 runner before this runs.
+
+S8-ARC-01 fix: the earlier stub always returned ``copied=0`` while the
+``/api/v3/brain`` endpoint surfaced the pre-existing ``learning_feedback``
+row count as ``legacy_migrated_count`` — a silent integrity lie. This
+implementation does the real row copy and is idempotent (guarded by a
+``migration_log`` sentinel row under the name ``LEG001_feedback_to_signals``).
+D5 (keep ``learning_feedback`` for one release) still holds — this
+migration does NOT delete from ``learning_feedback``; it only inserts
+into ``learning_signals`` + ``learning_features``.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# Marker used in migration_log to signal the data-copy has run.
+MIGRATION_NAME = "LEG001_feedback_to_signals"
+
+# Batch size for the row copy. 500 keeps the single transaction short
+# enough that any concurrent SQLite reader only sees brief locks, while
+# still amortising the per-row overhead.
+_COPY_BATCH_SIZE = 500
+
+
+def migrate_legacy_feedback(
+    learning_db: Path,
+    *,
+    dry_run: bool = False,
+) -> dict:
+    """Copy ``learning_feedback`` rows forward into LLD-02 tables.
+
+    Policy (LLD-07 §5 + D5 + D9):
+      * Idempotent: if ``migration_log`` already has ``LEG001_feedback_to_signals``
+        with status 'complete', returns a stats dict with ``already_done=True``.
+      * Preserves the original ``learning_feedback`` table (D5 — one release).
+      * Flags synthetic rows: ``learning_features.is_synthetic=1`` so the
+        LightGBM trainer in ``consolidation_worker._retrain_ranker`` can
+        exclude them from training (``WHERE is_synthetic=0``).
+      * Never raises; any sqlite3 error collapses into ``failed=N`` and a
+        WARN log so the daemon lifespan can continue.
+
+    Returns:
+        ``{"copied": int, "skipped": int, "failed": int,
+          "already_done": bool, "details": {...}}``
+    """
+    stats = {
+        "copied": 0, "skipped": 0, "failed": 0,
+        "already_done": False,
+        "details": {"name": MIGRATION_NAME, "dry_run": dry_run},
+    }
+    if not learning_db.exists():
+        stats["details"]["reason"] = "db_missing"
+        return stats
+
+    try:
+        conn = sqlite3.connect(str(learning_db), isolation_level=None, timeout=10.0)
+        conn.row_factory = sqlite3.Row
+    except sqlite3.Error as exc:
+        logger.warning("legacy migration: open failed: %s", exc)
+        stats["failed"] = 1
+        stats["details"]["reason"] = f"open_failed:{type(exc).__name__}"
+        return stats
+
+    try:
+        # Every required table must exist. If any is missing we defer to
+        # the migration runner (LLD-07 M001/M003) and record a skip.
+        if not _tables_exist(conn, ("migration_log", "learning_feedback",
+                                     "learning_signals", "learning_features")):
+            stats["details"]["reason"] = "required_tables_missing"
+            return stats
+
+        if _already_complete(conn, MIGRATION_NAME):
+            stats["already_done"] = True
+            stats["details"]["reason"] = "migration_log_complete"
+            return stats
+
+        # Row count of candidates up front so the stats dict can report
+        # progress even under a dry_run.
+        total = _count_feedback(conn)
+        stats["details"]["source_rows"] = total
+        if total == 0:
+            _record_migration(conn, MIGRATION_NAME,
+                              status="complete", rows_affected=0,
+                              dry_run=dry_run)
+            return stats
+
+        if dry_run:
+            stats["details"]["reason"] = "dry_run_noop"
+            stats["skipped"] = total
+            return stats
+
+        copied, failed = _copy_rows(conn)
+        stats["copied"] = copied
+        stats["failed"] = failed
+        _record_migration(conn, MIGRATION_NAME,
+                          status="complete" if failed == 0 else "partial",
+                          rows_affected=copied, dry_run=False)
+        return stats
+    finally:
+        try:
+            conn.close()
+        except sqlite3.Error:  # pragma: no cover — defensive close
+            pass
+
+
+# ---------------------------------------------------------------------------
+# Helpers — parameterised SQL only; no DDL here per H15.
+# ---------------------------------------------------------------------------
+
+
+def _tables_exist(conn: sqlite3.Connection, names: tuple[str, ...]) -> bool:
+    rows = conn.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name IN "
+        "(" + ",".join("?" * len(names)) + ")",
+        names,
+    ).fetchall()
+    return len(rows) == len(names)
+
+
+def _already_complete(conn: sqlite3.Connection, name: str) -> bool:
+    row = conn.execute(
+        "SELECT status FROM migration_log WHERE name = ?",
+        (name,),
+    ).fetchone()
+    return row is not None and str(row[0]).lower() == "complete"
+
+
+def _count_feedback(conn: sqlite3.Connection) -> int:
+    row = conn.execute(
+        "SELECT COUNT(*) FROM learning_feedback",
+    ).fetchone()
+    return int(row[0]) if row else 0
+
+
+def _copy_rows(conn: sqlite3.Connection) -> tuple[int, int]:
+    """Copy all ``learning_feedback`` rows forward.
+
+    Returns ``(copied, failed)``. Does not raise. Commits per batch so
+    a later failure still leaves the earlier batches durable.
+    """
+    copied = 0
+    failed = 0
+    offset = 0
+    while True:
+        try:
+            batch = conn.execute(
+                "SELECT id, profile_id, query_hash, fact_id, "
+                "       signal_type, signal_value, created_at "
+                "FROM learning_feedback "
+                "ORDER BY id LIMIT ? OFFSET ?",
+                (_COPY_BATCH_SIZE, offset),
+            ).fetchall()
+        except sqlite3.Error as exc:
+            logger.warning("legacy migration: batch read failed: %s", exc)
+            failed += 1
+            break
+        if not batch:
+            break
+
+        try:
+            conn.execute("BEGIN IMMEDIATE")
+            for row in batch:
+                try:
+                    _copy_single_row(conn, row)
+                    copied += 1
+                except sqlite3.Error as exc:
+                    logger.debug("legacy migration: row %s failed: %s",
+                                 row[0], exc)
+                    failed += 1
+            conn.execute("COMMIT")
+        except sqlite3.Error as exc:
+            logger.warning("legacy migration: batch commit failed: %s", exc)
+            try:
+                conn.execute("ROLLBACK")
+            except sqlite3.Error:  # pragma: no cover
+                pass
+            failed += len(batch)
+        offset += len(batch)
+
+    return copied, failed
+
+
+def _copy_single_row(conn: sqlite3.Connection, row: sqlite3.Row) -> None:
+    """Insert one legacy row into learning_signals + learning_features.
+
+    Synthetic features are minimal (position from the legacy row if present,
+    zeros elsewhere). The LightGBM trainer filters these out via
+    ``is_synthetic=0``; the bandit/heuristic layer can still learn from
+    signal counts.
+
+    Note: ``learning_feedback.query_hash`` is already a privacy-hashed
+    digest (LLD-02 §4.1 S2), so we copy it forward as-is. Padding to
+    32 chars keeps a stable shape when upstream hashes are shorter.
+    """
+    stored_hash = str(row["query_hash"] or "")
+    # Keep the hash at 32 hex chars for shape-stability with fresh signals.
+    query_hash = (stored_hash + ("0" * 32))[:32]
+    created_at = str(row["created_at"] or
+                     datetime.now(timezone.utc).isoformat(timespec="seconds"))
+    profile_id = str(row["profile_id"] or "default")
+    fact_id = str(row["fact_id"] or "")
+    legacy_query_id = f"legacy:{row['id']}"
+
+    # Insert the signal row. ``signal_type='legacy_feedback'`` marks it
+    # clearly so consumers (dashboard, labeler) can treat it correctly.
+    cur = conn.execute(
+        "INSERT INTO learning_signals "
+        "(profile_id, query, fact_id, signal_type, value, created_at, "
+        " query_id, query_text_hash, position, channel_scores, cross_encoder) "
+        "VALUES (?, '', ?, 'legacy_feedback', ?, ?, ?, ?, 0, '{}', NULL)",
+        (profile_id, fact_id,
+         float(row["signal_value"] or 1.0),
+         created_at, legacy_query_id, query_hash),
+    )
+    sid = cur.lastrowid
+
+    # Synthetic 20-dim feature vector (zeros). The real FEATURE_NAMES
+    # come from ``learning.features`` at training time; we don't import
+    # it here to keep this module boot-order-independent.
+    conn.execute(
+        "INSERT INTO learning_features "
+        "(profile_id, query_id, fact_id, features_json, label, created_at, "
+        " signal_id, is_synthetic) "
+        "VALUES (?, ?, ?, '{}', 0.0, ?, ?, 1)",
+        (profile_id, legacy_query_id, fact_id, created_at, sid),
+    )
+
+
+def _record_migration(
+    conn: sqlite3.Connection,
+    name: str,
+    *,
+    status: str,
+    rows_affected: int,
+    dry_run: bool,
+) -> None:
+    """Record migration status in ``migration_log``. No DDL here (H15).
+
+    The ``migration_log`` schema uses ``applied_at`` + ``ddl_sha256``
+    (see storage/migrations/M003_migration_log.py). This is a data-only
+    migration, so ``ddl_sha256`` is the empty string — the sentinel row
+    is for idempotency, not schema-drift detection.
+    """
+    applied_at = datetime.now(timezone.utc).isoformat(timespec="seconds")
+    if dry_run:
+        return
+    try:
+        conn.execute(
+            "INSERT OR REPLACE INTO migration_log "
+            "(name, applied_at, ddl_sha256, rows_affected, status) "
+            "VALUES (?, ?, '', ?, ?)",
+            (name, applied_at, rows_affected, status),
+        )
+    except sqlite3.Error as exc:
+        logger.warning("legacy migration: log record failed: %s", exc)
+
+
+__all__ = ("migrate_legacy_feedback", "MIGRATION_NAME")

package/src/superlocalmemory/learning/memory_merge.py

@@ -0,0 +1,160 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory v3.4.21 — LLD-12 §2.3 + §1.4
+
+"""Reversible merge-log writer + unmerge for consolidation.
+
+LLD-12 §1 hard invariant: consolidation NEVER issues
+``DELETE FROM atomic_facts``. Duplicates merge by:
+
+1. INSERT into ``memory_merge_log`` (canonical + merged fact_ids, scores,
+   timestamp, reversible flag).
+2. UPDATE ``atomic_facts`` SET archive_status='merged',
+   merged_into=<canonical>. Row stays; only status flips.
+
+``unmerge(merge_id)`` reverses the operation by flipping archive_status
+back to 'live' and clearing merged_into. The log row is marked
+``reversible=0`` once reversed so a second unmerge is a no-op.
+
+All operations run inside a single SQLite transaction with
+``busy_timeout=2000``. Partial failures roll back cleanly, leaving the
+DB in its pre-merge state.
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+def _iso_now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def apply_merges(
+    memory_db_path: str | Path,
+    candidates: list[tuple[str, str, float, float]],
+    *,
+    profile_id: str,
+) -> int:
+    """Apply merge candidates transactionally. Returns number applied.
+
+    Each candidate is ``(canonical_fact_id, merged_fact_id, cosine, jaccard)``.
+
+    Never deletes from ``atomic_facts``. Always writes a row to
+    ``memory_merge_log`` per applied merge.
+
+    Idempotent: if ``merged_fact_id`` is already ``archive_status='merged'``
+    from a prior run, the candidate is skipped (count not incremented).
+    """
+    if not candidates:
+        return 0
+
+    conn = sqlite3.connect(str(memory_db_path), timeout=10.0)
+    conn.execute("PRAGMA busy_timeout=2000")
+    applied = 0
+    # S-L02: track the candidate list in flight so a rollback diagnostic
+    # can blame the exact set of (canonical, merged) pairs instead of a
+    # blanket "rollback" message. Operators on the dashboard previously
+    # saw zero fidelity about which candidates were in the transaction
+    # at commit-time.
+    in_flight: list[tuple[str, str]] = []
+    try:
+        conn.execute("BEGIN IMMEDIATE")
+        for canonical_id, merged_id, cos, jac in candidates:
+            # Skip if already merged in a prior cycle.
+            row = conn.execute(
+                "SELECT archive_status FROM atomic_facts WHERE fact_id=?",
+                (merged_id,),
+            ).fetchone()
+            if row is None:
+                continue
+            if row[0] == "merged":
+                continue
+
+            conn.execute(
+                "INSERT INTO memory_merge_log "
+                "(merge_id, profile_id, canonical_fact_id, merged_fact_id, "
+                " cosine_sim, entity_jaccard, merged_at, reversible) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, 1)",
+                (
+                    str(uuid.uuid4()),
+                    profile_id,
+                    canonical_id,
+                    merged_id,
+                    float(cos),
+                    float(jac),
+                    _iso_now(),
+                ),
+            )
+            conn.execute(
+                "UPDATE atomic_facts "
+                "SET archive_status='merged', "
+                "    archive_reason='cosine_dup', "
+                "    merged_into=? "
+                "WHERE fact_id=?",
+                (canonical_id, merged_id),
+            )
+            applied += 1
+            in_flight.append((canonical_id, merged_id))
+        conn.commit()
+    except sqlite3.Error as exc:
+        conn.rollback()
+        logger.warning(
+            "apply_merges rollback: profile=%s pre-rollback_applied=%d "
+            "in_flight=%s error=%s",
+            profile_id, applied, in_flight, exc,
+        )
+        applied = 0
+    finally:
+        conn.close()
+    return applied
+
+
+def unmerge(memory_db_path: str | Path, merge_id: str) -> bool:
+    """Reverse a merge by merge_id. Returns True on success.
+
+    Flips the merged fact's archive_status back to 'live', clears
+    merged_into, and marks the log row ``reversible=0``.
+    """
+    conn = sqlite3.connect(str(memory_db_path), timeout=10.0)
+    conn.execute("PRAGMA busy_timeout=2000")
+    try:
+        row = conn.execute(
+            "SELECT merged_fact_id, reversible FROM memory_merge_log "
+            "WHERE merge_id=?",
+            (merge_id,),
+        ).fetchone()
+        if row is None:
+            return False
+        merged_fid, reversible = row
+        if not reversible:
+            return False
+
+        conn.execute("BEGIN IMMEDIATE")
+        conn.execute(
+            "UPDATE atomic_facts "
+            "SET archive_status='live', archive_reason=NULL, merged_into=NULL "
+            "WHERE fact_id=?",
+            (merged_fid,),
+        )
+        conn.execute(
+            "UPDATE memory_merge_log SET reversible=0 WHERE merge_id=?",
+            (merge_id,),
+        )
+        conn.commit()
+        return True
+    except sqlite3.Error as exc:
+        conn.rollback()
+        logger.warning("unmerge rollback: %s", exc)
+        return False
+    finally:
+        conn.close()
+
+
+__all__ = ("apply_merges", "unmerge")