superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/learning/legacy_migration.py

@@ -1,316 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory v3.4.22
-
-"""Legacy ``learning_feedback`` row migration — data movement only.
-
-
-Hard rule H15 (LLD-06 §10): this module MUST NOT contain any schema
-DDL. All schema definitions live in
-``src/superlocalmemory/storage/migrations/`` modules. This file is
-data-copy only — the tables it reads from and writes into are created
-by the LLD-07 runner before this runs.
-
-S8-ARC-01 fix: the earlier stub always returned ``copied=0`` while the
-``/api/v3/brain`` endpoint surfaced the pre-existing ``learning_feedback``
-row count as ``legacy_migrated_count`` — a silent integrity lie. This
-implementation does the real row copy and is idempotent (guarded by a
-``migration_log`` sentinel row under the name ``LEG001_feedback_to_signals``).
-D5 (keep ``learning_feedback`` for one release) still holds — this
-migration does NOT delete from ``learning_feedback``; it only inserts
-into ``learning_signals`` + ``learning_features``.
-"""
-from __future__ import annotations
-
-import json
-import logging
-import sqlite3
-from datetime import datetime, timezone
-from pathlib import Path
-
-logger = logging.getLogger(__name__)
-
-# Marker used in migration_log to signal the data-copy has run.
-MIGRATION_NAME = "LEG001_feedback_to_signals"
-
-# Batch size for the row copy. 500 keeps the single transaction short
-# enough that any concurrent SQLite reader only sees brief locks, while
-# still amortising the per-row overhead.
-_COPY_BATCH_SIZE = 500
-
-
-def legacy_query_id(feedback_row_id: int | str) -> str:
-    """Return the canonical ``learning_signals.query_id`` for a feedback row.
-
-    One explicit-feedback event has exactly ONE canonical identity, whichever
-    path writes it: this batch migration, or ``FeedbackCollector`` writing the
-    canonical pair eagerly at feedback time. Both derive the id from the
-    ``learning_feedback`` row id through this function, which is what lets the
-    migration recognise — and skip — rows already carried forward. Without a
-    shared identity the two writers would double-count the same event into the
-    store that gates the ranking phase.
-    """
-    return f"legacy:{feedback_row_id}"
-
-
-def migrate_legacy_feedback(
-    learning_db: Path,
-    *,
-    dry_run: bool = False,
-) -> dict:
-    """Copy ``learning_feedback`` rows forward into LLD-02 tables.
-
-    Policy (LLD-07 §5 + D5 + D9):
-      * Idempotent: if ``migration_log`` already has ``LEG001_feedback_to_signals``
-        with status 'complete', returns a stats dict with ``already_done=True``.
-      * Preserves the original ``learning_feedback`` table (D5 — one release).
-      * Flags synthetic rows: ``learning_features.is_synthetic=1`` so the
-        LightGBM trainer in ``consolidation_worker._retrain_ranker`` can
-        exclude them from training (``WHERE is_synthetic=0``).
-      * Never raises; any sqlite3 error collapses into ``failed=N`` and a
-        WARN log so the daemon lifespan can continue.
-
-    Returns:
-        ``{"copied": int, "skipped": int, "failed": int,
-          "already_done": bool, "details": {...}}``
-    """
-    stats = {
-        "copied": 0, "skipped": 0, "failed": 0,
-        "already_done": False,
-        "details": {"name": MIGRATION_NAME, "dry_run": dry_run},
-    }
-    if not learning_db.exists():
-        stats["details"]["reason"] = "db_missing"
-        return stats
-
-    try:
-        conn = sqlite3.connect(str(learning_db), isolation_level=None, timeout=10.0)
-        conn.row_factory = sqlite3.Row
-    except sqlite3.Error as exc:
-        logger.warning("legacy migration: open failed: %s", exc)
-        stats["failed"] = 1
-        stats["details"]["reason"] = f"open_failed:{type(exc).__name__}"
-        return stats
-
-    try:
-        # Every required table must exist. If any is missing we defer to
-        # the migration runner (LLD-07 M001/M003) and record a skip.
-        if not _tables_exist(conn, ("migration_log", "learning_feedback",
-                                     "learning_signals", "learning_features")):
-            stats["details"]["reason"] = "required_tables_missing"
-            return stats
-
-        if _already_complete(conn, MIGRATION_NAME):
-            stats["already_done"] = True
-            stats["details"]["reason"] = "migration_log_complete"
-            return stats
-
-        # Row count of candidates up front so the stats dict can report
-        # progress even under a dry_run.
-        total = _count_feedback(conn)
-        stats["details"]["source_rows"] = total
-        if total == 0:
-            _record_migration(conn, MIGRATION_NAME,
-                              status="complete", rows_affected=0,
-                              dry_run=dry_run)
-            return stats
-
-        if dry_run:
-            stats["details"]["reason"] = "dry_run_noop"
-            stats["skipped"] = total
-            return stats
-
-        copied, failed = _copy_rows(conn)
-        stats["copied"] = copied
-        stats["failed"] = failed
-        _record_migration(conn, MIGRATION_NAME,
-                          status="complete" if failed == 0 else "partial",
-                          rows_affected=copied, dry_run=False)
-        return stats
-    finally:
-        try:
-            conn.close()
-        except sqlite3.Error:  # pragma: no cover — defensive close
-            pass
-
-
-# ---------------------------------------------------------------------------
-# Helpers — parameterised SQL only; no DDL here per H15.
-# ---------------------------------------------------------------------------
-
-
-def _tables_exist(conn: sqlite3.Connection, names: tuple[str, ...]) -> bool:
-    rows = conn.execute(
-        "SELECT name FROM sqlite_master WHERE type='table' AND name IN "
-        "(" + ",".join("?" * len(names)) + ")",
-        names,
-    ).fetchall()
-    return len(rows) == len(names)
-
-
-def _already_complete(conn: sqlite3.Connection, name: str) -> bool:
-    row = conn.execute(
-        "SELECT status FROM migration_log WHERE name = ?",
-        (name,),
-    ).fetchone()
-    return row is not None and str(row[0]).lower() == "complete"
-
-
-def _count_feedback(conn: sqlite3.Connection) -> int:
-    row = conn.execute(
-        "SELECT COUNT(*) FROM learning_feedback",
-    ).fetchone()
-    return int(row[0]) if row else 0
-
-
-def _copy_rows(conn: sqlite3.Connection) -> tuple[int, int]:
-    """Copy all ``learning_feedback`` rows forward.
-
-    Returns ``(copied, failed)``. Does not raise. Commits per batch so
-    a later failure still leaves the earlier batches durable.
-
-    Rows whose canonical ``query_id`` is already present in
-    ``learning_signals`` are skipped rather than copied a second time.
-    ``FeedbackCollector`` writes the canonical pair at feedback time, so on
-    any install where this migration has not yet been sentinel-marked the
-    newest rows are already carried forward; copying them again would
-    inflate the very counter that gates the ranking phase.
-    """
-    copied = 0
-    failed = 0
-    offset = 0
-    already_present = _existing_legacy_query_ids(conn)
-    while True:
-        try:
-            batch = conn.execute(
-                "SELECT id, profile_id, query_hash, fact_id, "
-                "       signal_type, signal_value, created_at "
-                "FROM learning_feedback "
-                "ORDER BY id LIMIT ? OFFSET ?",
-                (_COPY_BATCH_SIZE, offset),
-            ).fetchall()
-        except sqlite3.Error as exc:
-            logger.warning("legacy migration: batch read failed: %s", exc)
-            failed += 1
-            break
-        if not batch:
-            break
-
-        try:
-            conn.execute("BEGIN IMMEDIATE")
-            for row in batch:
-                if legacy_query_id(row["id"]) in already_present:
-                    continue
-                try:
-                    _copy_single_row(conn, row)
-                    copied += 1
-                except sqlite3.Error as exc:
-                    logger.debug("legacy migration: row %s failed: %s",
-                                 row[0], exc)
-                    failed += 1
-            conn.execute("COMMIT")
-        except sqlite3.Error as exc:
-            logger.warning("legacy migration: batch commit failed: %s", exc)
-            try:
-                conn.execute("ROLLBACK")
-            except sqlite3.Error:  # pragma: no cover
-                pass
-            failed += len(batch)
-        offset += len(batch)
-
-    return copied, failed
-
-
-def _existing_legacy_query_ids(conn: sqlite3.Connection) -> set[str]:
-    """Return every ``legacy:`` query_id already present in learning_signals.
-
-    Read once up front: a per-row EXISTS probe over a signals table that grows
-    to tens of thousands of rows turns an O(n) copy into O(n*m).
-    """
-    try:
-        rows = conn.execute(
-            "SELECT DISTINCT query_id FROM learning_signals "
-            "WHERE query_id LIKE 'legacy:%'",
-        ).fetchall()
-    except sqlite3.Error as exc:
-        logger.warning("legacy migration: dedupe probe failed: %s", exc)
-        return set()
-    return {str(row[0]) for row in rows}
-
-
-def _copy_single_row(conn: sqlite3.Connection, row: sqlite3.Row) -> None:
-    """Insert one legacy row into learning_signals + learning_features.
-
-    Synthetic features are minimal (position from the legacy row if present,
-    zeros elsewhere). The LightGBM trainer filters these out via
-    ``is_synthetic=0``; the bandit/heuristic layer can still learn from
-    signal counts.
-
-    Note: ``learning_feedback.query_hash`` is already a privacy-hashed
-    digest (LLD-02 §4.1 S2), so we copy it forward as-is. Padding to
-    32 chars keeps a stable shape when upstream hashes are shorter.
-    """
-    stored_hash = str(row["query_hash"] or "")
-    # Keep the hash at 32 hex chars for shape-stability with fresh signals.
-    query_hash = (stored_hash + ("0" * 32))[:32]
-    created_at = str(row["created_at"] or
-                     datetime.now(timezone.utc).isoformat(timespec="seconds"))
-    profile_id = str(row["profile_id"] or "default")
-    fact_id = str(row["fact_id"] or "")
-    query_id = legacy_query_id(row["id"])
-
-    # Insert the signal row. ``signal_type='legacy_feedback'`` marks it
-    # clearly so consumers (dashboard, labeler) can treat it correctly.
-    cur = conn.execute(
-        "INSERT INTO learning_signals "
-        "(profile_id, query, fact_id, signal_type, value, created_at, "
-        " query_id, query_text_hash, position, channel_scores, cross_encoder) "
-        "VALUES (?, '', ?, 'legacy_feedback', ?, ?, ?, ?, 0, '{}', NULL)",
-        (profile_id, fact_id,
-         float(row["signal_value"] or 1.0),
-         created_at, query_id, query_hash),
-    )
-    sid = cur.lastrowid
-
-    # Synthetic 20-dim feature vector (zeros). The real FEATURE_NAMES
-    # come from ``learning.features`` at training time; we don't import
-    # it here to keep this module boot-order-independent.
-    conn.execute(
-        "INSERT INTO learning_features "
-        "(profile_id, query_id, fact_id, features_json, label, created_at, "
-        " signal_id, is_synthetic) "
-        "VALUES (?, ?, ?, '{}', 0.0, ?, ?, 1)",
-        (profile_id, query_id, fact_id, created_at, sid),
-    )
-
-
-def _record_migration(
-    conn: sqlite3.Connection,
-    name: str,
-    *,
-    status: str,
-    rows_affected: int,
-    dry_run: bool,
-) -> None:
-    """Record migration status in ``migration_log``. No DDL here (H15).
-
-    The ``migration_log`` schema uses ``applied_at`` + ``ddl_sha256``
-    (see storage/migrations/M003_migration_log.py). This is a data-only
-    migration, so ``ddl_sha256`` is the empty string — the sentinel row
-    is for idempotency, not schema-drift detection.
-    """
-    applied_at = datetime.now(timezone.utc).isoformat(timespec="seconds")
-    if dry_run:
-        return
-    try:
-        conn.execute(
-            "INSERT OR REPLACE INTO migration_log "
-            "(name, applied_at, ddl_sha256, rows_affected, status) "
-            "VALUES (?, ?, '', ?, ?)",
-            (name, applied_at, rows_affected, status),
-        )
-    except sqlite3.Error as exc:
-        logger.warning("legacy migration: log record failed: %s", exc)
-
-
-__all__ = ("migrate_legacy_feedback", "MIGRATION_NAME", "legacy_query_id")

package/src/superlocalmemory/learning/lightgbm_subprocess.py

@@ -1,236 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Isolated LightGBM work (retrain + consolidation) — OpenMP-crash guard.
-
-**Why this exists** — the unified daemon serves the HTTP API in-process and,
-by then, has already loaded PyTorch's OpenMP runtime (``torch/lib/libomp.dylib``)
-with *warm* worker threads (the cross-encoder reranker + embedding warm-up at
-startup). Any code path that imports ``lightgbm`` in that same process loads a
-*second* OpenMP runtime — the Homebrew ``libomp.dylib`` that
-``lib_lightgbm.dylib`` links against — which corrupts the shared ``__kmp``
-global state and SIGSEGVs a pre-existing torch OMP worker thread. That
-hard-crashes the whole daemon (observed on macOS / Apple Silicon;
-``DiagnosticReports/Python-*.ips`` shows ``libomp.dylib __kmp_launch_worker``).
-
-Two in-daemon entry points reach LightGBM training:
-  * ``POST /api/learning/retrain``  → legacy ``_retrain_ranker_impl``;
-  * ``POST /api/v3/learning/consolidate`` → ``ConsolidationWorker.run`` whose
-    step 5 trains via the online ``_run_shadow_cycle`` (active model) or the
-    legacy cold-start path.
-
-Both are funnelled through this module so LightGBM only ever trains in a
-**fresh subprocess** that never runs a torch tensor op — torch's OMP pool stays
-dormant there and lightgbm's runtime is the only active one. The mechanism
-mirrors the existing ``embedding_worker`` / ``reranker_worker`` isolation.
-
-Spawned via :func:`run_retrain_isolated` / :func:`run_consolidation_isolated`,
-which use a ``python -c`` bootstrap that imports lightgbm BEFORE the
-``superlocalmemory`` package. That ordering is load-bearing: importing the
-package transitively pulls in torch, and torch-OMP-first-then-lightgbm is
-exactly the sequence that segfaults. A plain ``python -m
-superlocalmemory.learning.lightgbm_subprocess`` would run the package
-``__init__`` (torch) first and crash — do not invoke it that way.
-
-The child emits a single JSON line on stdout — ``{"error": str|null, ...}`` —
-plus a task payload (``trained`` for retrain, ``stats`` for consolidate).
-Exit 0 = ran to completion; non-zero = handled failure. Either way the parent
-(the daemon) keeps running.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import logging
-import subprocess
-import sys
-from pathlib import Path
-
-logger = logging.getLogger(__name__)
-
-# Wall-clock ceilings. A retrain is 50 boosting rounds on ≤2000 rows (seconds);
-# a full consolidation cycle also decays/dedups/mines/graph-analyses, so it
-# gets more headroom.
-RETRAIN_TIMEOUT_SEC = 300
-CONSOLIDATE_TIMEOUT_SEC = 600
-
-
-# ---------------------------------------------------------------------------
-# Child-side task implementations (run in the isolated subprocess).
-# ---------------------------------------------------------------------------
-
-def _retrain(learning_db: str, profile_id: str, *, include_synthetic: bool) -> bool:
-    """Legacy ranker retrain in this (lightgbm-first) process. Returns trained."""
-    import lightgbm  # noqa: F401  (import-order side effect is intentional)
-
-    from superlocalmemory.learning.ranker_retrain_legacy import (
-        _retrain_ranker_impl,
-    )
-
-    return bool(
-        _retrain_ranker_impl(
-            learning_db, profile_id, include_synthetic=include_synthetic,
-        )
-    )
-
-
-def _consolidate(
-    memory_db: str, learning_db: str, profile_id: str, *, dry_run: bool,
-) -> dict:
-    """Full consolidation cycle (incl. step-5 LightGBM training). Returns stats."""
-    import lightgbm  # noqa: F401  (import-order side effect is intentional)
-
-    from superlocalmemory.learning.consolidation_worker import (
-        ConsolidationWorker,
-    )
-
-    worker = ConsolidationWorker(memory_db, learning_db)
-    return worker.run(profile_id, dry_run=dry_run)
-
-
-def _emit(obj: dict) -> None:
-    """Write the verdict as a single JSON line (default=str for safety)."""
-    sys.stdout.write(json.dumps(obj, default=str) + "\n")
-    sys.stdout.flush()
-
-
-def main(argv: list[str] | None = None) -> int:
-    parser = argparse.ArgumentParser(prog="slm-lightgbm-subprocess")
-    parser.add_argument("--task", choices=["retrain", "consolidate"], default="retrain")
-    parser.add_argument("--learning-db", required=True)
-    parser.add_argument("--memory-db")
-    parser.add_argument("--profile", default="default")
-    parser.add_argument("--include-synthetic", action="store_true")
-    parser.add_argument("--dry-run", action="store_true")
-    args = parser.parse_args(argv)
-
-    result: dict = {"error": None}
-    try:
-        if args.task == "retrain":
-            result["trained"] = _retrain(
-                args.learning_db, args.profile,
-                include_synthetic=args.include_synthetic,
-            )
-        else:
-            if not args.memory_db:
-                raise ValueError("--memory-db is required for the consolidate task")
-            result["stats"] = _consolidate(
-                args.memory_db, args.learning_db, args.profile,
-                dry_run=args.dry_run,
-            )
-    except Exception as exc:  # noqa: BLE001
-        result["error"] = f"{type(exc).__name__}: {exc}"
-        _emit(result)
-        return 1
-
-    _emit(result)
-    return 0
-
-
-# ---------------------------------------------------------------------------
-# Caller-side helpers — spawn the child in an isolated process.
-# ---------------------------------------------------------------------------
-
-def _run_isolated(task_args: list[str], *, timeout_sec: int) -> dict:
-    """Spawn the child with lightgbm imported first; return its JSON verdict.
-
-    Never raises for the expected failure modes (non-zero exit, timeout,
-    native crash with no JSON) — encodes them in an ``error`` key so the
-    calling daemon stays up no matter what.
-    """
-    # The leading ``import lightgbm`` is load-bearing: it must run BEFORE the
-    # superlocalmemory package (which transitively imports torch). ``-c`` is
-    # the only way to guarantee that ordering — ``-m superlocalmemory.…`` runs
-    # the package __init__ (torch) first and would reintroduce the crash.
-    bootstrap = (
-        "import lightgbm; "
-        "from superlocalmemory.learning.lightgbm_subprocess import main; "
-        "raise SystemExit(main())"
-    )
-    cmd = [sys.executable, "-c", bootstrap, *task_args]
-
-    try:
-        proc = subprocess.run(
-            cmd,
-            capture_output=True,
-            text=True,
-            timeout=timeout_sec,
-            check=False,
-        )
-    except subprocess.TimeoutExpired:
-        return {"error": f"timed out after {timeout_sec}s"}
-    except Exception as exc:  # noqa: BLE001
-        return {"error": f"spawn failed: {exc}"}
-
-    # The verdict is the last JSON line on stdout. Parse defensively — a native
-    # crash in the child would leave no JSON, which we surface plainly.
-    for line in reversed((proc.stdout or "").splitlines()):
-        line = line.strip()
-        if not line:
-            continue
-        try:
-            return json.loads(line)
-        except ValueError:
-            continue
-
-    tail = (proc.stderr or "").strip()[-500:]
-    return {
-        "error": (
-            f"subprocess produced no verdict (exit={proc.returncode}). "
-            f"stderr tail: {tail}"
-        ),
-    }
-
-
-def run_retrain_isolated(
-    learning_db: str | Path,
-    profile_id: str,
-    *,
-    include_synthetic: bool = False,
-    timeout_sec: int = RETRAIN_TIMEOUT_SEC,
-) -> dict:
-    """Train the ranker in a subprocess. Returns ``{"trained": bool, "error": ...}``."""
-    args = [
-        "--task", "retrain",
-        "--learning-db", str(learning_db),
-        "--profile", profile_id,
-    ]
-    if include_synthetic:
-        args.append("--include-synthetic")
-    verdict = _run_isolated(args, timeout_sec=timeout_sec)
-    verdict.setdefault("trained", False)
-    verdict.setdefault("error", None)
-    return verdict
-
-
-def run_consolidation_isolated(
-    memory_db: str | Path,
-    learning_db: str | Path,
-    profile_id: str,
-    *,
-    dry_run: bool = False,
-    timeout_sec: int = CONSOLIDATE_TIMEOUT_SEC,
-) -> dict:
-    """Run the full consolidation cycle in a subprocess (its step-5 training
-    would otherwise crash the torch-warm daemon).
-
-    Returns ``{"stats": dict|None, "error": str|None}``.
-    """
-    args = [
-        "--task", "consolidate",
-        "--memory-db", str(memory_db),
-        "--learning-db", str(learning_db),
-        "--profile", profile_id,
-    ]
-    if dry_run:
-        args.append("--dry-run")
-    verdict = _run_isolated(args, timeout_sec=timeout_sec)
-    verdict.setdefault("stats", None)
-    verdict.setdefault("error", None)
-    return verdict
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())