superlocalmemory 4.1.13 → 4.1.15

package/src/superlocalmemory/storage/generation_fence.py

@@ -1,63 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-
-from __future__ import annotations
-
-import threading
-import time
-
-_TTL_SECONDS = 300.0
-# Sentinel epoch that can never equal a real generation (which is >= 0), so a
-# conflicted key is always rejected by the fence.
-_CONFLICT_EPOCH = -1
-
-_lock = threading.RLock()
-_epochs: dict[tuple[str, str], tuple[int, float]] = {}
-
-
-def record_admission_epoch(profile_id: str, idempotency_key: str, epoch: int) -> None:
-    if not idempotency_key:
-        return
-    now = time.time()
-    with _lock:
-        _prune(now)
-        key = (profile_id, idempotency_key)
-        existing = _epochs.get(key)
-        if existing is not None and existing[0] not in (epoch, _CONFLICT_EPOCH):
-            # Two concurrent admits recorded different epochs for one key: fail
-            # closed so neither can satisfy the fence — both retry against the
-            # current binding rather than letting a stale epoch slip through.
-            _epochs[key] = (_CONFLICT_EPOCH, now)
-        else:
-            _epochs.setdefault(key, (epoch, now))
-
-
-def admitted_epoch(profile_id: str, idempotency_key: str) -> int | None:
-    if not idempotency_key:
-        return None
-    now = time.time()
-    with _lock:
-        entry = _epochs.get((profile_id, idempotency_key))
-        if entry is None:
-            return None
-        epoch, ts = entry
-        if now - ts > _TTL_SECONDS:
-            _epochs.pop((profile_id, idempotency_key), None)
-            return None
-        return epoch
-
-
-def clear_admission_epoch(profile_id: str, idempotency_key: str) -> None:
-    if not idempotency_key:
-        return
-    with _lock:
-        _epochs.pop((profile_id, idempotency_key), None)
-
-
-def _prune(now: float) -> None:
-    expired = [key for key, (_, ts) in _epochs.items() if now - ts > _TTL_SECONDS]
-    for key in expired:
-        _epochs.pop(key, None)
-
-
-__all__ = ["record_admission_epoch", "admitted_epoch", "clear_admission_epoch"]

package/src/superlocalmemory/storage/lineage_retention.py

@@ -1,236 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Forget the provenance of things that no longer exist.
-
-`derivation_lineage` records where each derived object came from — which source
-span produced this fact, this graph edge, this scene. Every ingestion operation
-re-captures lineage for the objects present at that moment, so the table grows
-with use, and nothing has ever deleted from it.
-
-Measured on a real 447 MB store:
-
-    derivation_lineage           256,885 rows, 64 MB, plus 65 MB of indexes
-    ... describing a graph edge
-        that no longer exists    100,581 rows — 39.2% of the table
-    growth                       ~9,000 rows/day, sustained
-
-The graph is pruned. Its lineage was not, so the record of how a deleted edge
-came to exist outlives the edge forever. Those rows answer no question: the
-evidence bundle computes lineage coverage over the objects it exports, and an
-object that is gone is not exported.
-
-WHAT IS AND IS NOT DELETED
-
-Only a row whose object is provably absent — the type is one this module knows
-how to resolve, the table exists, and no row with that id is in it. An
-unrecognised object type is left alone, because "I do not know what this
-describes" is not evidence that it describes nothing.
-
-There is deliberately no age rule. Lineage is what an audit reads to answer
-"where did this come from", and a fact can be years old and still current.
-Deleting provenance for something that still exists would be destroying the
-answer while keeping the question.
-"""
-
-from __future__ import annotations
-
-import logging
-import sqlite3
-from dataclasses import dataclass, field
-
-logger = logging.getLogger(__name__)
-
-__all__ = ["OBJECT_SOURCES", "LineagePruneReport", "count_orphan_lineage",
-           "prune_orphan_lineage"]
-
-#: object_type -> (table holding it, column carrying its id).
-#: Mirrors what ``core/derivation_lineage.capture_operation_lineage`` writes.
-#: A type absent from this map is never deleted.
-OBJECT_SOURCES: dict[str, tuple[str, str]] = {
-    "fact": ("atomic_facts", "fact_id"),
-    "graph_edge": ("graph_edges", "edge_id"),
-    "memory_scene": ("memory_scenes", "scene_id"),
-    "entity_summary": ("entity_profiles", "profile_entry_id"),
-    "index_bm25": ("bm25_tokens", "fact_id"),
-    "profile": ("profiles", "profile_id"),
-}
-
-#: Rows per transaction. Big enough that commit overhead vanishes, small enough
-#: that an interrupted run has done most of its work and holds no long lock.
-_BATCH = 2_000
-
-
-class _Rows:
-    """Read and write through either a raw connection or the DatabaseManager.
-
-    The maintenance cycle holds a ``DatabaseManager``, whose lock serialises
-    every write; a migration or a test holds a plain ``sqlite3.Connection``.
-    Both are legitimate callers, and the difference is two method names.
-    """
-
-    def __init__(self, db: object) -> None:
-        self._db = db
-        self._managed = hasattr(db, "transaction") and not isinstance(
-            db, sqlite3.Connection
-        )
-
-    def query(self, sql: str, params: tuple = ()) -> list:
-        rows = self._db.execute(sql, tuple(params))
-        return list(rows) if self._managed else rows.fetchall()
-
-    def write(self, sql: str, params: tuple = ()) -> None:
-        if self._managed:
-            with self._db.transaction():
-                self._db.execute(sql, tuple(params))
-            return
-        self._db.execute("BEGIN IMMEDIATE")
-        try:
-            self._db.execute(sql, tuple(params))
-            self._db.commit()
-        except Exception:
-            self._db.rollback()
-            raise
-
-
-@dataclass(frozen=True)
-class LineagePruneReport:
-    """What was removed, by object type, and what was left alone."""
-
-    deleted: dict[str, int] = field(default_factory=dict)
-    skipped_types: tuple[str, ...] = ()
-
-    @property
-    def total(self) -> int:
-        return sum(self.deleted.values())
-
-
-def _table_exists(rows: _Rows, table: str) -> bool:
-    return bool(rows.query(
-        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
-    ))
-
-
-def _resolvable(rows: _Rows) -> tuple[dict[str, tuple[str, str]], tuple[str, ...]]:
-    """Split the types present in the table into resolvable and not."""
-    present = {
-        str(r[0]) for r in rows.query(
-            "SELECT DISTINCT object_type FROM derivation_lineage"
-        )
-    }
-    resolvable: dict[str, tuple[str, str]] = {}
-    skipped: list[str] = []
-    for object_type in sorted(present):
-        source = OBJECT_SOURCES.get(object_type)
-        if source is None or not _table_exists(rows, source[0]):
-            skipped.append(object_type)
-            continue
-        resolvable[object_type] = source
-    return resolvable, tuple(skipped)
-
-
-def count_orphan_lineage(
-    db: object, *, profile_id: str | None = None,
-) -> LineagePruneReport:
-    """How many rows describe something absent, without deleting anything."""
-    rows = _Rows(db)
-    if not _table_exists(rows, "derivation_lineage"):
-        return LineagePruneReport()
-
-    resolvable, skipped = _resolvable(rows)
-    counts: dict[str, int] = {}
-    for object_type, (table, column) in resolvable.items():
-        sql = (
-            "SELECT COUNT(*) FROM derivation_lineage d WHERE d.object_type = ? "
-            f"AND NOT EXISTS (SELECT 1 FROM {table} t WHERE t.{column} = d.object_id)"
-        )
-        params: list[object] = [object_type]
-        if profile_id is not None:
-            sql += " AND d.profile_id = ?"
-            params.append(profile_id)
-        count = int(rows.query(sql, tuple(params))[0][0])
-        if count:
-            counts[object_type] = count
-    return LineagePruneReport(counts, skipped)
-
-
-def prune_orphan_lineage(
-    db: object,
-    *,
-    profile_id: str | None = None,
-    dry_run: bool = False,
-) -> LineagePruneReport:
-    """Delete lineage rows whose object is provably gone.
-
-    Batched and committed as it goes, so an interrupted run keeps the work it
-    already did and the next one continues. Nothing here reads a clock: what is
-    deleted depends only on what exists.
-    """
-    rows = _Rows(db)
-    if not _table_exists(rows, "derivation_lineage"):
-        return LineagePruneReport()
-
-    report = count_orphan_lineage(db, profile_id=profile_id)
-    if dry_run or not report.total:
-        return report
-
-    resolvable, skipped = _resolvable(rows)
-    deleted: dict[str, int] = {}
-
-    for object_type, (table, column) in resolvable.items():
-        if object_type not in report.deleted:
-            continue
-        removed = 0
-        while True:
-            sql = (
-                "SELECT lineage_id FROM derivation_lineage d "
-                "WHERE d.object_type = ? "
-                f"AND NOT EXISTS (SELECT 1 FROM {table} t WHERE t.{column} = d.object_id)"
-            )
-            params: list[object] = [object_type]
-            if profile_id is not None:
-                sql += " AND d.profile_id = ?"
-                params.append(profile_id)
-            sql += f" LIMIT {_BATCH}"
-
-            ids = [r[0] for r in rows.query(sql, tuple(params))]
-            if not ids:
-                break
-            placeholders = ",".join("?" * len(ids))
-            # Re-check absence in the DELETE itself. Selecting orphans and then
-            # deleting them by id is a check-then-act: another process can
-            # recreate the object between the two statements, and the row would
-            # be deleted anyway. Repeating the predicate makes the decision and
-            # the deletion one statement.
-            delete_sql = (
-                f"DELETE FROM derivation_lineage WHERE lineage_id IN ({placeholders}) "
-                f"AND NOT EXISTS (SELECT 1 FROM {table} t "
-                f"WHERE t.{column} = derivation_lineage.object_id)"
-            )
-            rows.write(delete_sql, tuple(ids))
-            # Count what survived the re-check rather than what was offered.
-            still = rows.query(
-                f"SELECT COUNT(*) FROM derivation_lineage "
-                f"WHERE lineage_id IN ({placeholders})",
-                tuple(ids),
-            )
-            remaining_here = int(still[0][0]) if still else 0
-            removed += len(ids) - remaining_here
-            if remaining_here == len(ids):
-                # Every row in this batch was rescued by a concurrent writer.
-                # Another pass would select the same rows forever.
-                break
-        if removed:
-            deleted[object_type] = removed
-            logger.info(
-                "lineage retention: removed %d row(s) describing a %s that no "
-                "longer exists", removed, object_type,
-            )
-
-    if skipped:
-        logger.info(
-            "lineage retention: left %s alone — no table is known to hold them",
-            ", ".join(skipped),
-        )
-    return LineagePruneReport(deleted, skipped)

package/src/superlocalmemory/storage/logical_edges.py

@@ -1,86 +0,0 @@
-"""Canonical logical-edge projection shared by scale backends.
-
-Legacy databases can contain multiple physical ``graph_edges`` rows for one
-logical relationship. Current writes define identity as profile, source,
-target, and edge type, retaining the strongest weight. Derived projections
-must use that same contract without rewriting canonical SQLite history.
-"""
-from __future__ import annotations
-
-import sqlite3
-from collections.abc import Iterator
-from typing import Any
-
-_LOGICAL_EDGE_SELECT = """
-    SELECT
-        source_id,
-        target_id,
-        COALESCE(edge_type, 'related') AS edge_type,
-        MAX(COALESCE(weight, 1.0)) AS weight,
-        profile_id
-    FROM graph_edges
-    WHERE profile_id = ?
-      AND NOT EXISTS (
-            SELECT 1 FROM atomic_facts f
-            WHERE f.fact_id IN (graph_edges.source_id, graph_edges.target_id)
-              AND NOT ({visible})
-      )
-    GROUP BY profile_id, source_id, target_id, COALESCE(edge_type, 'related')
-"""
-
-
-def _edge_select(conn: sqlite3.Connection) -> str:
-    """The logical-edge query, with the withheld-endpoint exclusion resolved.
-
-    WHY AN EDGE WITH A WITHHELD ENDPOINT IS NOT A LOGICAL EDGE
-    ----------------------------------------------------------
-    The retrieval channel this projection stands in for does not traverse them.
-    It loads ``graph_edges`` by scope and then prunes the result: "Edge scope
-    alone cannot authorize an endpoint. Prune both endpoints against the visible
-    fact corpus so denied facts cannot influence an allowed candidate indirectly
-    through propagation." Its entity map is filtered the same way, for a reason
-    it spells out — a withheld row carries its whole cluster's pooled entity
-    list, so it out-ranks real memories and then gets discarded at hydration,
-    spending the channel's budget on nothing.
-
-    The export predated that fix and kept the withheld endpoints. Measured on a
-    copy of the author's store: Cozo's bridge held 1,257 facts the store may not
-    return and its edges touched 805, and the graph search diverged from SQLite
-    on **every** query — three shadow checks, three mismatches. One query
-    returned 9 results against SQLite's 20, because withheld facts had taken the
-    top-k budget. The projection failed closed every time, so recall was correct
-    and the projection was dead weight.
-
-    The predicate is resolved against the passed connection because
-    ``archive_status`` and ``quarantined`` each arrive with a migration and may
-    be absent on a store the engine has not opened.
-    """
-    from superlocalmemory.storage.database import visible_fact_clause_for_connection
-
-    # The helper returns a leading-AND clause for appending; here it is needed as
-    # a standalone predicate, so strip the connective and default to "always
-    # visible" on a store that has neither column yet.
-    clause = visible_fact_clause_for_connection(conn, prefix="f").strip()
-    predicate = clause[4:].strip() if clause.upper().startswith("AND ") else clause
-    return _LOGICAL_EDGE_SELECT.format(visible=predicate or "1=1")
-
-
-def iter_logical_edges(
-    conn: sqlite3.Connection, profile_id: str
-) -> Iterator[tuple[Any, ...]]:
-    """Yield normalized graph edges in deterministic fingerprint order."""
-    return iter(
-        conn.execute(
-            _edge_select(conn) + " ORDER BY source_id, target_id, edge_type",
-            (profile_id,),
-        )
-    )
-
-
-def count_logical_edges(conn: sqlite3.Connection, profile_id: str) -> int:
-    """Count relationships using the canonical logical identity."""
-    row = conn.execute(
-        "SELECT COUNT(*) FROM (" + _edge_select(conn) + ")",
-        (profile_id,),
-    ).fetchone()
-    return int(row[0] if row else 0)

package/src/superlocalmemory/storage/memory_write.py

@@ -1,115 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com
-
-"""Canonical short-lived WRITE / READ connections for memory.db.
-
-Why this exists
----------------
-memory.db is a WAL SQLite database with exactly one writer allowed at any
-instant.  Historically many subsystems opened their own bare
-``sqlite3.connect(...)`` and wrote without any coordination, which produced
-two failure modes under real multi-agent load:
-
-1. **In-process races** — daemon worker threads racing each other at the WAL
-   layer, retrying ``SQLITE_BUSY`` until they time out.
-2. **Cross-process races** — Claude Code hooks and the ``slm ingest`` CLI run
-   as *separate OS processes* and write memory.db directly.  An in-process
-   ``threading`` lock cannot serialise those; only SQLite's own
-   ``PRAGMA busy_timeout`` makes them WAIT for the writer instead of erroring.
-
-``memory_write()`` closes BOTH gaps with one helper that every writer uses:
-
-* Acquires the process-level :func:`get_write_lock` (the OUTERMOST lock — see
-  ``write_lock.py``) so in-process writers serialise cleanly with **no**
-  ``SQLITE_BUSY`` spin.
-* Opens the connection with ``PRAGMA busy_timeout`` so that a *different
-  process* (hook / CLI) writing at the same instant WAITS rather than failing.
-* Commits on success, rolls back on error, always closes — a short critical
-  section.
-
-The single hard rule for callers
----------------------------------
-**Never hold this connection across a slow operation** — no embedding /
-network call / large unbounded transaction inside the ``with`` block.  Do the
-slow work first, then open ``memory_write()`` only for the fast INSERT/UPDATE.
-Holding the writer across a multi-second op is exactly what starves everyone
-else (SQLITE_BUSY after retries).  Batch large writes into bounded chunks that
-each commit quickly.
-
-Reads
------
-Use :func:`memory_read` for read-only access.  WAL allows concurrent readers
-without blocking the writer, so reads do NOT take the write lock.  The helper
-opens SQLite with ``mode=ro`` and ``PRAGMA query_only=ON``; it uses the
-3.8.6 read-path budget of at most 250ms during a brief checkpoint window.
-"""
-from __future__ import annotations
-
-import os
-import sqlite3
-from contextlib import contextmanager
-from pathlib import Path
-from typing import Generator
-
-from superlocalmemory.storage.read_connection import ReadConnectionFactory
-from superlocalmemory.storage.write_lock import get_write_lock
-
-
-def _busy_timeout_ms() -> int:
-    """Busy-timeout in ms, env-overridable, matching DatabaseManager's default.
-
-    Kept in sync with ``storage/database.py::_BUSY_TIMEOUT_MS`` (env
-    ``SLM_DB_BUSY_TIMEOUT_MS``, default 10_000) so every memory.db connection
-    in the process — and in hook/CLI child processes that import this — waits
-    the same amount for the single writer.
-    """
-    try:
-        return max(0, int(os.environ.get("SLM_DB_BUSY_TIMEOUT_MS", "10000")))
-    except (TypeError, ValueError):
-        return 10000
-
-
-@contextmanager
-def memory_write(db_path: str | Path) -> Generator[sqlite3.Connection, None, None]:
-    """Yield a serialised, busy-timeout-guarded WRITE connection to memory.db.
-
-    Acquires the process write lock (outermost), opens a short-lived
-    connection with ``busy_timeout``, commits on success / rolls back on
-    error, and always closes.
-
-    HARD RULE: keep the ``with`` block short — never embed / call the network /
-    run an unbounded transaction while holding it.
-    """
-    ms = _busy_timeout_ms()
-    lock = get_write_lock(db_path)
-    with lock:
-        conn = sqlite3.connect(str(db_path), timeout=ms / 1000.0)
-        try:
-            conn.execute(f"PRAGMA busy_timeout={ms}")
-            conn.row_factory = sqlite3.Row
-            yield conn
-            conn.commit()
-        except Exception:
-            try:
-                conn.rollback()
-            except Exception:
-                pass
-            raise
-        finally:
-            conn.close()
-
-
-@contextmanager
-def memory_read(db_path: str | Path) -> Generator[sqlite3.Connection, None, None]:
-    """Yield a physically read-only connection with no write lock.
-
-    WAL permits concurrent readers, so this deliberately does NOT take the
-    write lock.  A 250ms bounded wait protects the query path from a brief
-    checkpoint without converting a dashboard recall into a long hang.
-    """
-    with ReadConnectionFactory(db_path, timeout_ms=min(_busy_timeout_ms(), 250)).snapshot() as conn:
-        yield conn
-
-
-__all__ = ["memory_write", "memory_read"]