superlocalmemory 4.1.13 → 4.1.15

package/src/superlocalmemory/storage/retention_policy.py

@@ -1,860 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""What bounds every table that only ever grows.
-
-WHY A REGISTRY RATHER THAN FOUR MORE PRUNERS
---------------------------------------------
-Three tables already had a pruner and each was written separately, wired
-separately, and configured separately. The fourth unbounded table was found by
-someone looking at a disk-usage report, and the fifth by someone looking at the
-fourth. That is not a maintenance strategy, it is a sequence of accidents.
-
-So the policy is declared here, per table, and one pass enforces all of them. A
-table that appends without a policy is a defect the test suite can see, which is
-the property none of the individual pruners could give us.
-
-MEASURED, ON TWO REAL STORES
-----------------------------
-The live store is small enough that nothing looks urgent. The larger one shows
-where each table is going:
-
-    table                    live rows   larger store   what it was doing
-    derivation_lineage         260,346         22,020   48.8 rows per memory,
-                                                        690 for one of them
-    temporal_events             16,837        118,210   70.7% pointing at
-                                                        memories already deleted
-    fact_access_log              5,388         90,759   34% older than 90 days
-    consolidation_log            4,318         29,743   99.5% older than 30 days
-    tool_events                  2,002         39,444   28 MB, no policy at all
-    schema_version               3,496        234,348   SEVEN distinct versions
-
-That last row is not a retention problem and a TTL would be the wrong fix for
-it: the table has no unique constraint, so the six ``INSERT OR IGNORE`` call
-sites that all believe they are idempotent append a duplicate every time. It is
-registered here as ``NONE`` with that reason, and repaired by a migration.
-
-WHAT EACH KIND MEANS, AND WHY THE CONJUNCTION EXISTS
-----------------------------------------------------
-``TTL_AND_CAP`` deletes a row only when it is BOTH older than the window AND
-beyond the per-key cap. Either rule alone is unsafe here. A pure TTL on
-``fact_access_log`` would delete the access history of a memory nobody has
-touched for months -- which is precisely the memory whose history the decay
-dynamics need. A pure cap would delete this morning's accesses on a busy
-memory. The conjunction cannot destroy recent signal and cannot leave a key
-unbounded, because time eventually satisfies the first clause for every row.
-
-``CAP_PER_KEY`` without a window is right where old rows are *superseded* rather
-than historical: re-deriving an object writes its lineage again, and the tenth
-re-derivation of the same object tells you nothing the newest one does not.
-
-``ORPHAN`` is for a table with no insertion timestamp. ``temporal_events`` has
-``observation_date`` and ``referenced_date``, which are when the event happened,
-not when the row was written -- so a TTL against either would delete a memory of
-something long ago that was recorded this morning. Its bound is that the memory
-it describes must still exist.
-"""
-
-from __future__ import annotations
-
-import logging
-import sqlite3
-import time
-from dataclasses import dataclass
-from enum import Enum
-from typing import Any
-
-logger = logging.getLogger(__name__)
-
-
-class RetentionKind(Enum):
-    """How a table is kept from growing without limit."""
-
-    TTL_AND_CAP = "ttl_and_cap"
-    TTL = "ttl"
-    CAP_PER_KEY = "cap_per_key"
-    ORPHAN = "orphan"
-    #: Already pruned by a named module. Registered so the gate can see it is
-    #: covered, and so the module is discoverable from here.
-    EXTERNAL = "external"
-    #: Bounded by construction, or a growth defect that a policy would mask. The
-    #: reason is mandatory and is the whole value of the entry.
-    NONE = "none"
-    #: Measured growing faster than the store and the right rule is NOT yet
-    #: known. This exists so the registry can be honest: writing NONE here would
-    #: silence the gate on a table we know is a problem, and inventing a TTL for
-    #: data whose lifecycle nobody has established is how a cleanup job deletes
-    #: something load-bearing. An entry of this kind must carry its measurement.
-    UNRESOLVED = "unresolved"
-
-
-@dataclass(frozen=True)
-class RetentionPolicy:
-    """One table's bound, with the reasoning attached to it."""
-
-    table: str
-    kind: RetentionKind
-    reason: str
-    ttl_days: int | None = None
-    timestamp_column: str | None = None
-    key_column: str | None = None
-    cap_per_key: int | None = None
-    #: ``(local_column, "referent_table.referent_column")`` for ORPHAN.
-    referent: tuple[str, str] | None = None
-    pruned_by: str | None = None
-    #: Extra columns that, with ``key_column``, name one key. A cap counts
-    #: "the newest N per key", so a key that is not unique across the whole
-    #: table counts other rows' entries against a key's allowance. ``profile_id``
-    #: is added automatically wherever the table has it and does not need to be
-    #: listed here -- see ``_partition_columns``.
-    also_partition_by: tuple[str, ...] = ()
-
-    def __post_init__(self) -> None:
-        if not self.reason.strip():
-            raise ValueError(f"{self.table}: a policy without a reason is not a policy")
-        if self.kind in (RetentionKind.TTL, RetentionKind.TTL_AND_CAP):
-            if not self.ttl_days or not self.timestamp_column:
-                raise ValueError(f"{self.table}: a time rule needs a window and a column")
-        if self.kind in (RetentionKind.CAP_PER_KEY, RetentionKind.TTL_AND_CAP):
-            if not self.cap_per_key or not self.key_column:
-                raise ValueError(f"{self.table}: a cap rule needs a key and a limit")
-        if self.kind is RetentionKind.ORPHAN and not self.referent:
-            raise ValueError(f"{self.table}: an orphan rule needs a referent")
-        if self.kind is RetentionKind.EXTERNAL and not self.pruned_by:
-            raise ValueError(f"{self.table}: say which module prunes it")
-        if self.kind is RetentionKind.UNRESOLVED and not any(
-            character.isdigit() for character in self.reason
-        ):
-            raise ValueError(
-                f"{self.table}: an unresolved entry must carry the measurement "
-                f"that made it unresolved"
-            )
-
-
-REGISTERED_POLICIES: dict[str, RetentionPolicy] = {
-    policy.table: policy
-    for policy in (
-        RetentionPolicy(
-            table="derivation_lineage",
-            kind=RetentionKind.CAP_PER_KEY,
-            key_column="object_id",
-            also_partition_by=("object_type",),
-            cap_per_key=10,
-            timestamp_column="created_at",
-            reason=(
-                "Provenance of a derived object. Re-deriving it appends another "
-                "row, so the older ones are superseded rather than historical -- "
-                "one object had 690. No time window: the newest ten answer every "
-                "question the table is asked, at any age. The orphan half of this "
-                "table's upkeep is lineage_retention.prune_orphan_lineage. The "
-                "key is the object's id together with its type and workspace, "
-                "because the id alone is unique in none of those directions."
-            ),
-        ),
-        RetentionPolicy(
-            table="fact_access_log",
-            kind=RetentionKind.TTL_AND_CAP,
-            ttl_days=90,
-            timestamp_column="accessed_at",
-            key_column="fact_id",
-            cap_per_key=100,
-            reason=(
-                "Feeds tier demotion and the outcome signal, which both read "
-                "recent access. Bounded by the conjunction so neither rule can "
-                "take the signal on its own: a memory untouched for months keeps "
-                "its history, and a busy memory keeps today's."
-            ),
-        ),
-        RetentionPolicy(
-            table="consolidation_log",
-            kind=RetentionKind.TTL,
-            ttl_days=30,
-            timestamp_column="timestamp",
-            reason=(
-                "Read only as recent history -- the dashboard timeline and "
-                "get_consolidation_history, which takes the newest 50. The column "
-                "is `timestamp`; a rule written against `logged_at` would match "
-                "nothing and quietly do nothing."
-            ),
-        ),
-        RetentionPolicy(
-            table="tool_events",
-            kind=RetentionKind.TTL,
-            ttl_days=180,
-            timestamp_column="created_at",
-            reason=(
-                "Input to the assertion miner and the evolution triggers, which "
-                "read recent activity. The window is long because no reader "
-                "declares one and the cost of guessing short is a lost signal, "
-                "not a bigger file."
-            ),
-        ),
-        RetentionPolicy(
-            table="temporal_events",
-            kind=RetentionKind.ORPHAN,
-            referent=("fact_id", "atomic_facts.fact_id"),
-            reason=(
-                "No insertion timestamp: observation_date and referenced_date are "
-                "when the event happened, so a time rule would delete a memory of "
-                "something long ago that was recorded today. Its bound is that the "
-                "memory it describes still exists -- 70.7% of the rows on the "
-                "larger store point at memories already deleted."
-            ),
-        ),
-        RetentionPolicy(
-            table="provenance",
-            kind=RetentionKind.ORPHAN,
-            referent=("fact_id", "atomic_facts.fact_id"),
-            reason=(
-                "Where a memory came from. 45.8% of the rows on the larger store "
-                "(9,124 of 19,914) name a memory that no longer exists, so the "
-                "cascade does not reach here. Bound is the memory's existence: "
-                "provenance of a deleted memory is not provenance."
-            ),
-        ),
-        RetentionPolicy(
-            table="fact_context",
-            kind=RetentionKind.ORPHAN,
-            referent=("fact_id", "atomic_facts.fact_id"),
-            reason=(
-                "Generated description and keywords per memory. Tracks the memory "
-                "count (2.94x against 2.27x more memories) but leaks on delete: "
-                "3.9% orphaned on the larger store, 0.1% on the live one."
-            ),
-        ),
-        RetentionPolicy(
-            table="memory_events",
-            kind=RetentionKind.TTL,
-            ttl_days=180,
-            timestamp_column="created_at",
-            reason=(
-                "An event log: 0% orphaned, so nothing leaks, but it grew 13.9x "
-                "between two stores whose memory counts differ by 2.27x -- it "
-                "tracks activity, not content. The window is long because no "
-                "reader declares one."
-            ),
-        ),
-        RetentionPolicy(
-            table="action_outcomes",
-            kind=RetentionKind.NONE,
-            reason=(
-                "MUST NOT be pruned. This is the feedback that teaches the ranker "
-                "which answers were useful, and it is already sparse -- two rows "
-                "in the whole store carry an informative signal, against the fifty "
-                "needed to retrain. It grows 35.8x faster than the memory count, "
-                "so it will need a rule eventually; taking one now would delete "
-                "the evidence the ranker is waiting for."
-            ),
-        ),
-        RetentionPolicy(
-            table="compliance_audit",
-            kind=RetentionKind.NONE,
-            reason=(
-                "The erasure audit trail. Two rows today. Its whole purpose is to "
-                "outlive the data it describes, so bounding it by age would "
-                "destroy the only record that an erasure happened."
-            ),
-        ),
-        RetentionPolicy(
-            table="atomic_facts",
-            kind=RetentionKind.NONE,
-            reason=(
-                "These ARE the memories. What bounds them is the forgetting curve, "
-                "tier demotion and erasure on request -- decisions about meaning, "
-                "made elsewhere and visible to the user. A retention job here "
-                "would delete someone's memories on a timer."
-            ),
-        ),
-        RetentionPolicy(
-            table="memories",
-            kind=RetentionKind.NONE,
-            reason=(
-                "The raw records the memories are extracted from, and user data on "
-                "the same footing. Grew 7.53x against 2.27x more memories because "
-                "it tracks conversations rather than extracted facts, but the same "
-                "argument holds: this is content, not bookkeeping."
-            ),
-        ),
-        RetentionPolicy(
-            table="embedding_metadata",
-            kind=RetentionKind.NONE,
-            reason=(
-                "Exactly one row per memory -- 5,338 for 5,338 and 12,102 for "
-                "12,102 on the two stores, and 0% orphaned. Bounded by the memory "
-                "count by construction, and removed with the memory."
-            ),
-        ),
-        RetentionPolicy(
-            table="fact_entity_associations",
-            kind=RetentionKind.NONE,
-            reason=(
-                "Which subjects each memory mentions. 3.16x against 2.27x more "
-                "memories, and 0% orphaned on both stores, so it is bounded by the "
-                "memory count and the cascade reaches it."
-            ),
-        ),
-        RetentionPolicy(
-            table="core_memory_blocks",
-            kind=RetentionKind.NONE,
-            reason=(
-                "The always-present context blocks: 10 rows on one store and 20 on "
-                "the other. Bounded by the number of block categories, which is a "
-                "number we choose."
-            ),
-        ),
-        RetentionPolicy(
-            table="memory_scenes",
-            kind=RetentionKind.UNRESOLVED,
-            reason=(
-                "Grew 27.15x between two stores whose memory counts differ by "
-                "2.27x: 422 rows against 11,459. Keyed by a JSON list of memory "
-                "ids rather than a column, so no single-column rule reaches it, "
-                "and whether an old scene is superseded by a newer one or is "
-                "history worth keeping has not been established. Naming that "
-                "here rather than inventing a window for it."
-            ),
-        ),
-        RetentionPolicy(
-            table="mesh_events",
-            kind=RetentionKind.UNRESOLVED,
-            reason=(
-                "Grew 65.83x between the two stores -- 6 rows against 395 -- the "
-                "fastest of any table measured. Small in absolute terms because "
-                "the mesh has never run with a second node, which is also why its "
-                "real growth rate is unknown. A window guessed from 395 rows "
-                "would be a guess."
-            ),
-        ),
-        RetentionPolicy(
-            table="soft_prompt_templates",
-            kind=RetentionKind.CAP_PER_KEY,
-            key_column="category",
-            cap_per_key=5,
-            timestamp_column="created_at",
-            reason=(
-                "Versioned: storing a prompt deactivates the previous one for its "
-                "category and inserts a new row, so the count rises by one per "
-                "category per consolidation cycle and never falls. 44 rows on the "
-                "live store, of which 2 are active -- two categories times "
-                "twenty-two cycles. Keeping five versions leaves room to see what "
-                "changed without keeping every cycle forever."
-            ),
-        ),
-        RetentionPolicy(
-            table="rbac_sessions",
-            kind=RetentionKind.TTL,
-            ttl_days=30,
-            timestamp_column="created_at",
-            reason=(
-                "Login sessions. 0 rows today because company mode has never run "
-                "with users, which is exactly why it needs a rule before it does. "
-                "Expired sessions are also purged once when the daemon starts, "
-                "which is not a bound: a daemon that has been up for 25 hours "
-                "accumulates for 25 hours. This is the recurring half."
-            ),
-        ),
-        RetentionPolicy(
-            table="skill_evolution_log",
-            kind=RetentionKind.TTL,
-            ttl_days=365,
-            timestamp_column="created_at",
-            reason=(
-                "The record of automatic changes to an agent's own instructions. "
-                "0 rows today. A year is deliberately long: this is the audit "
-                "trail for a mutation a person approved, and it should outlive "
-                "any question about why the behaviour changed."
-            ),
-        ),
-        RetentionPolicy(
-            table="ccq_audit_log",
-            kind=RetentionKind.NONE,
-            reason=(
-                "An audit log, 0 rows today. Its purpose is to outlive what it "
-                "describes, so bounding it by age would destroy the record rather "
-                "than the storage cost."
-            ),
-        ),
-        RetentionPolicy(
-            table="projection_obligations",
-            kind=RetentionKind.NONE,
-            reason=(
-                "Outstanding projection work: 1,305 rows on the live store and a "
-                "row is removed when the obligation is discharged. Bounded by work "
-                "in flight, and a rule here would drop unfinished work exactly the "
-                "way a timeout on the outbox would."
-            ),
-        ),
-        RetentionPolicy(
-            table="projection_tombstones",
-            kind=RetentionKind.NONE,
-            reason=(
-                "One row per deletion still to be applied to a projection -- 1 on "
-                "the live store. Bounded by pending deletions, and dropping one "
-                "would leave a deleted memory in a projection."
-            ),
-        ),
-        RetentionPolicy(
-            table="completion_manifests",
-            kind=RetentionKind.NONE,
-            reason=(
-                "One per completed ingestion batch, 435 on the live store, removed "
-                "with the batch it describes. Bounded by ingestion in flight."
-            ),
-        ),
-        RetentionPolicy(
-            table="fact_consolidations",
-            kind=RetentionKind.NONE,
-            reason=(
-                "Which memories were merged into which: 2,432 rows against 5,338 "
-                "memories. Bounded by the memory count, and the row is the only "
-                "record that a merge happened, so age is the wrong axis."
-            ),
-        ),
-        RetentionPolicy(
-            table="consolidated_summaries",
-            kind=RetentionKind.NONE,
-            reason=(
-                "One summary per group of memories, 1,137 on the live store. "
-                "Bounded by the number of groups, which is bounded by the memory "
-                "count; a summary is replaced rather than appended."
-            ),
-        ),
-        RetentionPolicy(
-            table="ingestion_operations",
-            kind=RetentionKind.NONE,
-            reason=(
-                "742 rows on the live store and 117 on the larger one -- it went "
-                "DOWN as the store grew, so something already collects it. "
-                "Bounded, and adding a second rule would race the first."
-            ),
-        ),
-        RetentionPolicy(
-            table="correction_cases",
-            kind=RetentionKind.NONE,
-            reason=(
-                "One per correction a person proposed, 90 on the live store. This "
-                "is the ledger of who changed what and why; deleting an old case "
-                "removes the answer to a question about a memory that still exists."
-            ),
-        ),
-        RetentionPolicy(
-            table="behavioral_assertions",
-            kind=RetentionKind.NONE,
-            reason=(
-                "What the system has concluded about how it is used: 9 rows. "
-                "Reinforced in place rather than appended, so bounded by the number "
-                "of distinct conclusions."
-            ),
-        ),
-        RetentionPolicy(
-            table="feedback_records",
-            kind=RetentionKind.NONE,
-            reason=(
-                "Explicit feedback on an answer, 1 row. Sparse and load-bearing "
-                "for ranking quality; the same argument as the outcome feedback "
-                "above, at an even smaller count."
-            ),
-        ),
-        RetentionPolicy(
-            table="profiles",
-            kind=RetentionKind.NONE,
-            reason=(
-                "2 rows. Bounded by the number of workspaces a person creates, and "
-                "removed by erasure rather than by age."
-            ),
-        ),
-        RetentionPolicy(
-            table="polar_embeddings",
-            kind=RetentionKind.NONE,
-            reason=(
-                "0 rows; an alternate vector representation, at most one per "
-                "memory when written. Bounded by the memory count by construction."
-            ),
-        ),
-        RetentionPolicy(
-            table="embedding_quantization_metadata",
-            kind=RetentionKind.NONE,
-            reason=(
-                "0 rows; at most one per memory when quantization runs. Bounded by "
-                "the memory count by construction."
-            ),
-        ),
-        RetentionPolicy(
-            table="ccq_consolidated_blocks",
-            kind=RetentionKind.NONE,
-            reason=(
-                "0 rows; a compiled block is replaced rather than appended, so it "
-                "is bounded by the number of block categories."
-            ),
-        ),
-        RetentionPolicy(
-            table="backup_destinations",
-            kind=RetentionKind.NONE,
-            reason=(
-                "0 rows; one per configured backup target. Bounded by a number the "
-                "user chooses, and a destination does not expire."
-            ),
-        ),
-        RetentionPolicy(
-            table="retention_rules",
-            kind=RetentionKind.NONE,
-            reason=(
-                "0 rows; per-memory retention rules for compliance, one per rule a "
-                "person writes. Bounded by that, and a rule that expired would "
-                "silently stop being enforced."
-            ),
-        ),
-        RetentionPolicy(
-            table="mesh_messages",
-            kind=RetentionKind.UNRESOLVED,
-            reason=(
-                "0 rows, because the mesh has never run with a second node -- so "
-                "its growth rate is unmeasured, exactly like mesh_events at 65.83x "
-                "between two stores. A message is presumably consumed and "
-                "removable, but presumably is not a rule."
-            ),
-        ),
-        RetentionPolicy(
-            table="mesh_sent_ops",
-            kind=RetentionKind.UNRESOLVED,
-            reason=(
-                "0 rows, same reason as the other mesh tables: it records "
-                "operations already sent, which sounds collectable once "
-                "acknowledged, but with 0 rows and no second node there is nothing "
-                "to measure an acknowledgement window against."
-            ),
-        ),
-        RetentionPolicy(
-            table="rbac_users",
-            kind=RetentionKind.NONE,
-            reason=(
-                "0 rows; one per member of a workspace. Bounded by team size, and "
-                "removed when a person is removed, not when their row gets old."
-            ),
-        ),
-        RetentionPolicy(
-            table="graph_edges",
-            kind=RetentionKind.EXTERNAL,
-            pruned_by="core.graph_pruner.prune_graph",
-            reason="Orphans, duplicates, weak edges and hub degree, every cycle.",
-        ),
-        RetentionPolicy(
-            table="association_edges",
-            kind=RetentionKind.EXTERNAL,
-            pruned_by="core.graph_pruner.prune_graph",
-            reason="Orphan sweep in the same pass as graph_edges.",
-        ),
-        RetentionPolicy(
-            table="activation_cache",
-            kind=RetentionKind.EXTERNAL,
-            pruned_by="storage.database.DatabaseManager.cleanup_activation_cache",
-            reason="Expiry-based GC once per maintenance cycle.",
-        ),
-        RetentionPolicy(
-            table="schema_version",
-            kind=RetentionKind.NONE,
-            reason=(
-                "Should hold one row per schema version and holds thousands -- "
-                "3,496 rows for 7 versions on one store, 234,348 on another. The "
-                "table has no unique constraint, so the six INSERT OR IGNORE call "
-                "sites that believe they are idempotent each append a duplicate. "
-                "A retention rule here would hide a correctness bug behind a "
-                "cleanup job; the unique index is the fix."
-            ),
-        ),
-        RetentionPolicy(
-            table="migration_log",
-            kind=RetentionKind.NONE,
-            reason=(
-                "One row per migration, and there are 48. Bounded by the number "
-                "of migrations ever written, which is a number we control."
-            ),
-        ),
-        RetentionPolicy(
-            table="projection_outbox",
-            kind=RetentionKind.NONE,
-            reason=(
-                "Keyed on fact_id, so it cannot exceed the fact count, and a row "
-                "is deleted when the projection accepts it. Bounded by design; a "
-                "TTL here would silently drop unprojected work."
-            ),
-        ),
-    )
-}
-
-
-def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
-    row = conn.execute(
-        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
-    ).fetchone()
-    return row is not None
-
-
-def _columns(conn: sqlite3.Connection, table: str) -> set[str]:
-    """Column names, or empty when the table cannot be introspected.
-
-    A virtual table raises rather than answering when its module is not loaded
-    on this connection -- ``sqlite_vec`` is loaded per connection, so a
-    maintenance connection that never asked for it gets "no such module: vec0"
-    from a plain PRAGMA. An unreadable table is one this pass leaves alone,
-    which is the right answer for a vector index anyway: it is derived from a
-    base table and bounded by writing to it.
-    """
-    try:
-        return {row[1] for row in conn.execute(f"PRAGMA table_info({table})")}
-    except sqlite3.Error as exc:
-        logger.debug("retention: cannot introspect %s: %s", table, exc)
-        return set()
-
-
-def apply_policy(
-    conn: sqlite3.Connection,
-    policy: RetentionPolicy,
-    *,
-    dry_run: bool = False,
-    limit: int | None = None,
-) -> int:
-    """Enforce one policy. Returns rows deleted, or would-be deleted.
-
-    A policy whose table or column is absent is a no-op, not an error: these
-    tables arrive with migrations and a store may predate any of them.
-
-    ``limit`` caps how many rows one call removes, so a caller that has to hand
-    the write lock back can do this in bounded pieces. Without it a single call
-    deleted 123,888 rows in 1,480 ms on a 1 GB store, holding the lock for the
-    whole of it -- and a memory being saved in that window waits behind it.
-    """
-    # A kind with no rule of its own returns before any SQL is built. Without
-    # UNRESOLVED in this set it fell through to the cap branch with every field
-    # None and emitted "no such column: None" on two real stores -- and the test
-    # meant to catch that passed anyway, because run_retention swallows a
-    # per-table error and the table was simply absent from the result either way.
-    if policy.kind in (
-        RetentionKind.EXTERNAL, RetentionKind.NONE, RetentionKind.UNRESOLVED,
-    ):
-        return 0
-    if not _table_exists(conn, policy.table):
-        return 0
-    present = _columns(conn, policy.table)
-    needed = [
-        column
-        for column in (policy.timestamp_column, policy.key_column)
-        if column is not None
-    ]
-    if policy.referent is not None:
-        needed.append(policy.referent[0])
-    missing = [column for column in needed if column not in present]
-    if missing:
-        logger.debug(
-            "retention: %s has no %s; skipping", policy.table, ", ".join(missing)
-        )
-        return 0
-
-    if policy.kind is RetentionKind.ORPHAN:
-        local, target = policy.referent  # type: ignore[misc]
-        referent_table, referent_column = target.split(".", 1)
-        if not _table_exists(conn, referent_table):
-            return 0
-        where = (
-            f"{local} IS NOT NULL AND {local} NOT IN "
-            f"(SELECT {referent_column} FROM {referent_table})"
-        )
-        params: tuple[Any, ...] = ()
-    elif policy.kind is RetentionKind.TTL:
-        where = f"{policy.timestamp_column} < datetime('now', ?)"
-        params = (f"-{policy.ttl_days} days",)
-    else:
-        # Beyond the cap AND (for TTL_AND_CAP) older than the window. The window
-        # function ranks newest-first per key so "beyond the cap" means "not one
-        # of the newest N", which is the only reading that keeps recent rows.
-        age = ""
-        params = ()
-        if policy.kind is RetentionKind.TTL_AND_CAP:
-            age = f" AND {policy.timestamp_column} < datetime('now', ?)"
-            params = (f"-{policy.ttl_days} days",)
-        partition = ", ".join(_partition_columns(conn, policy))
-        where = (
-            f"rowid IN (SELECT rowid FROM (SELECT rowid, ROW_NUMBER() OVER ("
-            f"PARTITION BY {partition} "
-            f"ORDER BY {policy.timestamp_column} DESC, rowid DESC) AS rn "
-            f"FROM {policy.table}) WHERE rn > {int(policy.cap_per_key or 0)})"
-            f"{age}"
-        )
-
-    if dry_run:
-        row = conn.execute(
-            f"SELECT COUNT(*) FROM {policy.table} WHERE {where}", params
-        ).fetchone()
-        return int(row[0] if row else 0)
-
-    if limit is not None and limit > 0:
-        # Two statements rather than DELETE ... LIMIT, which needs a compile
-        # option SQLite is not built with everywhere.
-        cursor = conn.execute(
-            f"DELETE FROM {policy.table} WHERE rowid IN ("
-            f"SELECT rowid FROM {policy.table} WHERE {where} LIMIT ?)",
-            (*params, int(limit)),
-        )
-        return int(cursor.rowcount or 0)
-    cursor = conn.execute(f"DELETE FROM {policy.table} WHERE {where}", params)
-    return int(cursor.rowcount or 0)
-
-
-def _partition_columns(
-    conn: sqlite3.Connection, policy: RetentionPolicy
-) -> tuple[str, ...]:
-    """Every column that has to match for two rows to be under the same cap.
-
-    A cap keeps the newest N rows per key, so anything the key does not
-    distinguish gets counted against somebody else's allowance. Two workspaces
-    are the case that matters: a shared table's ids are unique per workspace,
-    not across the store, so partitioning on the id alone lets one workspace's
-    rows evict another's. Nothing on a single-workspace store notices, which is
-    why this has to be structural rather than something a policy author
-    remembers.
-
-    ``profile_id`` is added wherever the table has it, checked against the live
-    schema rather than assumed, because these policies also run on stores old
-    enough to predate the column.
-    """
-    columns: list[str] = []
-    if _has_column(conn, policy.table, "profile_id"):
-        columns.append("profile_id")
-    for extra in policy.also_partition_by:
-        if _has_column(conn, policy.table, extra) and extra not in columns:
-            columns.append(extra)
-    if policy.key_column and policy.key_column not in columns:
-        columns.append(policy.key_column)
-    return tuple(columns)
-
-
-def _has_column(conn: sqlite3.Connection, table: str, column: str) -> bool:
-    """Whether ``table`` has ``column`` on this store."""
-    try:
-        rows = conn.execute(f"PRAGMA table_info({table})").fetchall()
-    except sqlite3.Error:
-        return False
-    return any(str(row[1]) == column for row in rows)
-
-
-#: Rows one piece removes before handing the write lock back. Chosen from
-#: measurement on a 1 GB store, sweeping the same 123,888 rows every time, with
-#: the page cache warm because that is the state a running store is in:
-#:
-#:     unbounded      1,291 ms in a single hold
-#:     2,000/piece      255 ms longest hold, 8,425 ms total
-#:     8,000/piece      216 ms longest hold, 2,925 ms total, median hold 0.5 ms
-#:    20,000/piece      349 ms longest hold, 2,160 ms total
-#:
-#: Smaller is not automatically shorter, which is the non-obvious part: a
-#: piece costs mostly the scan that finds the rows, not the removing of them,
-#: so halving the piece nearly doubles the number of scans and barely moves the
-#: hold. 8,000 is where both numbers are at their best.
-#:
-#: The total goes up. That is the trade being made on purpose -- this is a
-#: background sweep, and the number that matters is how long someone saving a
-#: memory has to wait for the lock, not how long the sweep takes.
-BOUNDED_BATCH = 8000
-
-#: Seconds between pieces. Long enough for a waiting writer to be handed the
-#: lock by the OS rather than losing the race back to this loop every time.
-BOUNDED_YIELD_S = 0.002
-
-
-def run_retention_bounded(
-    open_connection: Any,
-    *,
-    batch_size: int = BOUNDED_BATCH,
-    yield_seconds: float = BOUNDED_YIELD_S,
-    max_batches_per_table: int = 500,
-) -> dict[str, int]:
-    """The same sweep, in pieces, releasing the write lock between each.
-
-    ``open_connection`` is a context manager factory -- typically the database
-    manager's ``raw_connection`` -- and is entered once per piece. That is the
-    point: entering it is what takes the process write lock, so a caller that
-    enters it once for the whole sweep holds the lock for the whole sweep,
-    however carefully the sweep batches inside.
-
-    ``max_batches_per_table`` is a stop, not a target. A policy that keeps
-    reporting deletions forever is a bug in that policy, and this pass declining
-    to loop on it indefinitely is how the rest of the tables still get swept.
-    """
-    removed: dict[str, int] = {}
-    for policy in REGISTERED_POLICIES.values():
-        table_total = 0
-        try:
-            for _ in range(max_batches_per_table):
-                with open_connection() as conn:
-                    count = apply_policy(conn, policy, limit=batch_size)
-                if count <= 0:
-                    break
-                table_total += count
-                if count < batch_size:
-                    break
-                time.sleep(yield_seconds)
-            else:
-                logger.warning(
-                    "retention: %s still had rows to remove after %d pieces; "
-                    "stopping here and continuing with the next table",
-                    policy.table, max_batches_per_table,
-                )
-        except Exception as exc:  # noqa: BLE001
-            logger.warning("retention: %s skipped (%s)", policy.table, exc)
-            continue
-        if table_total:
-            removed[policy.table] = table_total
-    return removed
-
-
-def run_retention(
-    conn: sqlite3.Connection, *, dry_run: bool = False
-) -> dict[str, int]:
-    """Enforce every policy that has one. Returns rows removed per table.
-
-    One statement per table, each committed by the caller's connection. A table
-    that fails does not stop the others: an unbounded table is a slow problem and
-    stopping the pass would leave every table after it in the dict unbounded too.
-    """
-    removed: dict[str, int] = {}
-    for policy in REGISTERED_POLICIES.values():
-        try:
-            count = apply_policy(conn, policy, dry_run=dry_run)
-        except sqlite3.Error as exc:
-            logger.warning("retention: %s failed: %s", policy.table, exc)
-            continue
-        if count:
-            removed[policy.table] = count
-    return removed
-
-
-def undeclared_growing_tables(conn: sqlite3.Connection) -> list[str]:
-    """Tables that look append-shaped and have no policy.
-
-    "Append-shaped" is taken from the schema, not from a hand-written list, so a
-    table added next release shows up here without anyone remembering to add it:
-    it carries a timestamp-ish column and is not a registered policy, an FTS
-    shadow, or SQLite's own bookkeeping.
-    """
-    candidates: list[str] = []
-    rows = conn.execute(
-        "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
-    ).fetchall()
-    for (name,) in rows:
-        if name in REGISTERED_POLICIES or name.startswith("sqlite_"):
-            continue
-        # FTS5 external-content shadow tables are derived from their base table
-        # and are pruned by writing to it.
-        if any(
-            name.endswith(suffix)
-            for suffix in ("_data", "_idx", "_content", "_docsize", "_config")
-        ):
-            continue
-        columns = _columns(conn, name)
-        if not columns & {
-            "created_at", "accessed_at", "occurred_at", "timestamp", "logged_at",
-        }:
-            continue
-        candidates.append(name)
-    return candidates