superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/encoding/cognitive_consolidator.py

@@ -1,899 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Cognitive Consolidation Quantization (CCQ) — Phase E.
-
-Sleep-time consolidation that mirrors hippocampal-neocortical transfer:
-  1. IDENTIFY: warm/cold facts below retention threshold
-  2. CLUSTER: group by entity overlap + temporal proximity (Union-Find)
-  3. EXTRACT GIST: rules (Mode A) or LLM (Mode B/C) summary
-  4. COMPRESS: source embeddings -> PolarQuant 2-bit
-  5. STORE: gist block at float32 + archive source facts
-  6. AUDIT: complete audit trail
-
-Biological analogy:
-  Hippocampus (atomic_facts) -> replay during sleep (CCQ pipeline)
-  -> Neocortex (ccq_consolidated_blocks with full-precision gist)
-
-Hard rules:
-  - Already-consolidated facts NEVER re-consolidated (HR-01)
-  - Minimum cluster size 3 (HR-02)
-  - Gist must cover 50% shared entities (HR-03)
-  - Source facts soft-archived, NEVER deleted (HR-04)
-  - Gist embedding always float32 (HR-05)
-  - Parameterized SQL only (HR-06)
-  - Per-cluster error isolation (HR-07)
-  - Idempotent (HR-08)
-  - PolarQuant optional (HR-10)
-  - Audit trail mandatory (HR-11)
-
-References:
-  McClelland et al. (1995). Complementary Learning Systems.
-  SimpleMem (arXiv 2601.02553). Semantic lossless compression.
-  TurboQuant (ICLR 2026). Recursive polar quantization.
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: AGPL-3.0-or-later
-IP Novelty: 92% (no prior art for retention-gated consolidation + polar quantization)
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-from collections import defaultdict
-from dataclasses import dataclass
-from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Any, Protocol
-
-from superlocalmemory.storage.models import _new_id
-
-if TYPE_CHECKING:
-    from superlocalmemory.core.config import CCQConfig
-    from superlocalmemory.storage.database import DatabaseManager
-
-logger = logging.getLogger(__name__)
-
-
-# ---------------------------------------------------------------------------
-# Protocols
-# ---------------------------------------------------------------------------
-
-class Embedder(Protocol):
-    """Anything that produces an embedding vector from text."""
-
-    def encode(self, text: str) -> list[float]: ...
-
-
-class LLM(Protocol):
-    """Anything that can generate text from a prompt."""
-
-    def generate(self, prompt: str, system: str = "") -> str: ...
-
-    def is_available(self) -> bool: ...
-
-
-# ---------------------------------------------------------------------------
-# Data classes (frozen, immutable)
-# ---------------------------------------------------------------------------
-
-
-@dataclass(frozen=True)
-class ConsolidationCluster:
-    """A group of related fading memories identified for consolidation."""
-
-    cluster_id: str
-    fact_ids: tuple[str, ...]
-    shared_entities: tuple[str, ...]
-    temporal_centroid: str
-    avg_retention: float
-    fact_count: int
-
-
-@dataclass(frozen=True)
-class GistResult:
-    """Output of the gist extraction step."""
-
-    gist_text: str
-    key_entities: tuple[str, ...]
-    extraction_mode: str       # 'rules' or 'llm'
-    representative_fact_id: str
-
-
-@dataclass(frozen=True)
-class CCQPipelineResult:
-    """Full result of one CCQ pipeline execution."""
-
-    clusters_processed: int
-    blocks_created: int
-    facts_archived: int
-    total_bytes_before: int
-    total_bytes_after: int
-    compression_ratio: float
-    audit_entries: tuple[str, ...]
-    errors: tuple[str, ...]
-
-
-# ---------------------------------------------------------------------------
-# Union-Find helper
-# ---------------------------------------------------------------------------
-
-
-class _UnionFind:
-    """Minimal Union-Find for entity-based clustering."""
-
-    __slots__ = ("_parent", "_rank")
-
-    def __init__(self, elements: list[str]) -> None:
-        self._parent: dict[str, str] = {e: e for e in elements}
-        self._rank: dict[str, int] = {e: 0 for e in elements}
-
-    def find(self, x: str) -> str:
-        if self._parent[x] != x:
-            self._parent[x] = self.find(self._parent[x])
-        return self._parent[x]
-
-    def union(self, a: str, b: str) -> None:
-        ra, rb = self.find(a), self.find(b)
-        if ra == rb:
-            return
-        if self._rank[ra] < self._rank[rb]:
-            ra, rb = rb, ra
-        self._parent[rb] = ra
-        if self._rank[ra] == self._rank[rb]:
-            self._rank[ra] += 1
-
-    def components(self) -> dict[str, list[str]]:
-        groups: dict[str, list[str]] = defaultdict(list)
-        for element in self._parent:
-            groups[self.find(element)].append(element)
-        return dict(groups)
-
-
-# ---------------------------------------------------------------------------
-# Date helpers
-# ---------------------------------------------------------------------------
-
-
-def _parse_date(raw: str | None) -> datetime | None:
-    """Parse ISO-8601 dates with multiple format fallbacks."""
-    if not raw:
-        return None
-    for fmt in (
-        "%Y-%m-%dT%H:%M:%S.%f",
-        "%Y-%m-%dT%H:%M:%S",
-        "%Y-%m-%d %H:%M:%S",
-        "%Y-%m-%d",
-    ):
-        try:
-            return datetime.strptime(raw, fmt)
-        except ValueError:
-            continue
-    return None
-
-
-def _temporal_midpoint(dates: list[datetime]) -> str:
-    """Compute ISO-8601 midpoint of a list of datetimes."""
-    if not dates:
-        return datetime.now(timezone.utc).isoformat()
-    ts = [d.timestamp() for d in dates]
-    mid = sum(ts) / len(ts)
-    return datetime.fromtimestamp(mid).isoformat()
-
-
-# ---------------------------------------------------------------------------
-# CognitiveConsolidator
-# ---------------------------------------------------------------------------
-
-
-def _config_for_this_store() -> "CCQConfig":
-    """The settings this store is actually running under.
-
-    A caller that supplies only a database used to get the bare defaults, which
-    say a language model may be used to summarise. Three of the four places
-    that build this pass only a database, so on a store configured to use no
-    language model at all, the refusal below rested on nobody having handed one
-    in -- true today, and not a property of the mode. Reading the store's own
-    settings makes it a property of the mode.
-
-    Falls back to the defaults if the settings cannot be read, because a
-    consolidation pass that cannot start is worse than one that starts with the
-    documented defaults.
-    """
-    from superlocalmemory.core.config import CCQConfig as _CCQConfig
-
-    try:
-        from superlocalmemory.core.config import SLMConfig
-
-        resolved = getattr(SLMConfig.load(), "ccq", None)
-        if isinstance(resolved, _CCQConfig):
-            return resolved
-    except Exception as exc:  # noqa: BLE001
-        logger.debug("consolidator: using default settings (%s)", exc)
-    return _CCQConfig()
-
-
-class CognitiveConsolidator:
-    """CCQ engine: sleep-time consolidation with quantization.
-
-    Executes 6-step pipeline: identify -> cluster -> gist -> compress
-    -> store -> audit. Each cluster failure is isolated (HR-07).
-    """
-
-    __slots__ = ("_db", "_embedder", "_llm", "_config")
-
-    def __init__(
-        self,
-        db: DatabaseManager,
-        embedder: Embedder | None = None,
-        llm: LLM | None = None,
-        config: CCQConfig | None = None,
-    ) -> None:
-        self._db = db
-        self._embedder = embedder
-        self._llm = llm
-        self._config = config if config is not None else _config_for_this_store()
-
-    # ------------------------------------------------------------------
-    # Public API
-    # ------------------------------------------------------------------
-
-    def run_pipeline(
-        self, profile_id: str, dry_run: bool = False,
-    ) -> CCQPipelineResult:
-        """Execute the full 6-step CCQ pipeline.
-
-        Per-cluster error isolation: one cluster failure does NOT
-        abort the pipeline (HR-07).
-
-        Args:
-            profile_id: Target profile.
-            dry_run: If True, identify clusters but don't apply changes.
-        """
-        # Step 1: Identify candidates
-        candidates = self._step1_identify(profile_id)
-        if not candidates:
-            return self._empty_result()
-
-        # Step 2: Cluster by entity overlap + temporal proximity
-        clusters = self._step2_cluster(candidates, profile_id)
-        if not clusters:
-            return self._empty_result()
-
-        if dry_run:
-            return CCQPipelineResult(
-                clusters_processed=len(clusters),
-                blocks_created=0,
-                facts_archived=len(candidates),
-                total_bytes_before=0,
-                total_bytes_after=0,
-                compression_ratio=0.0,
-                audit_entries=(),
-                errors=(),
-            )
-
-        # Process each cluster
-        blocks_created = 0
-        facts_archived = 0
-        bytes_before = 0
-        bytes_after = 0
-        audit_ids: list[str] = []
-        errors: list[str] = []
-
-        for cluster in clusters:
-            try:
-                # Step 3: Extract gist
-                gist = self._step3_extract_gist(cluster, profile_id)
-
-                # Step 4: Compress source embeddings
-                cb, ca = self._step4_compress_embeddings(cluster, profile_id)
-
-                # Step 5: Store block + archive source facts
-                block_id = self._step5_store_block(cluster, gist, profile_id)
-
-                # Step 6: Audit trail
-                audit_id = self._step6_audit(
-                    cluster, gist, cb, ca, block_id, profile_id,
-                )
-
-                blocks_created += 1
-                facts_archived += cluster.fact_count
-                bytes_before += cb
-                bytes_after += ca
-                audit_ids.append(audit_id)
-
-            except Exception as exc:
-                logger.warning(
-                    "CCQ cluster %s failed (non-fatal): %s",
-                    cluster.cluster_id, exc,
-                )
-                errors.append(
-                    f"cluster={cluster.cluster_id}: {exc!s}",
-                )
-
-        compression_ratio = (
-            bytes_before / bytes_after if bytes_after > 0 else 0.0
-        )
-
-        return CCQPipelineResult(
-            clusters_processed=len(clusters),
-            blocks_created=blocks_created,
-            facts_archived=facts_archived,
-            total_bytes_before=bytes_before,
-            total_bytes_after=bytes_after,
-            compression_ratio=round(compression_ratio, 2),
-            audit_entries=tuple(audit_ids),
-            errors=tuple(errors),
-        )
-
-    # ------------------------------------------------------------------
-    # Step 1: Identify candidates
-    # ------------------------------------------------------------------
-
-    def _step1_identify(self, profile_id: str) -> list[dict]:
-        """Identify warm/cold facts not yet consolidated.
-
-        Excludes:
-          - Active/archive/forgotten lifecycle zones
-          - Facts above retention threshold
-          - Facts already in ccq_consolidated_blocks (HR-01, HR-08)
-        """
-        rows = self._db.execute(
-            """
-            SELECT f.fact_id, f.content, f.canonical_entities_json,
-                   f.created_at, f.observation_date, f.importance,
-                   f.confidence,
-                   r.retention_score, r.lifecycle_zone, r.memory_strength
-            FROM atomic_facts f
-            INNER JOIN fact_retention r
-                ON f.fact_id = r.fact_id AND r.profile_id = ?
-            WHERE f.profile_id = ?
-              AND r.lifecycle_zone IN ('warm', 'cold')
-              AND r.retention_score < ?
-              AND f.lifecycle != 'forgotten'
-              -- Withheld rows are not candidates, and this is not tidiness.
-              -- 304 of them remain warm/cold in atomic_facts after 4.0.10
-              -- withholds them, and each still carries its cluster's POOLED
-              -- canonical_entities_json. CCQ clusters on entity overlap, so one
-              -- withheld summary naming ten entities joins a cluster of real
-              -- memories, contributes model prose to the gist, and then this
-              -- pass archives every source in the cluster -- including the real
-              -- memories, at scores M043's restore would not bring back.
-              -- Exactly the damage this release exists to stop, on a path the
-              -- release did not otherwise touch.
-              AND COALESCE(f.quarantined, 0) = 0
-              AND f.fact_id NOT IN (
-                  SELECT je.value
-                  FROM ccq_consolidated_blocks ccb,
-                       json_each(ccb.source_fact_ids) je
-                  WHERE ccb.profile_id = ?
-              )
-            ORDER BY r.retention_score ASC
-            LIMIT ?
-            """,
-            (
-                profile_id,
-                profile_id,
-                self._config.retention_threshold,
-                profile_id,
-                self._config.max_candidates_per_run,
-            ),
-        )
-
-        candidates: list[dict] = []
-        for row in rows or []:
-            d = dict(row)
-            raw_entities = d.get("canonical_entities_json") or "[]"
-            try:
-                d["canonical_entities"] = json.loads(raw_entities)
-            except (json.JSONDecodeError, TypeError):
-                d["canonical_entities"] = []
-            candidates.append(d)
-
-        return candidates
-
-    # ------------------------------------------------------------------
-    # Step 2: Cluster by entity overlap + temporal proximity
-    # ------------------------------------------------------------------
-
-    def _step2_cluster(
-        self,
-        candidates: list[dict],
-        profile_id: str,
-    ) -> list[ConsolidationCluster]:
-        """Group candidates via Union-Find (entity overlap) + temporal sub-clustering."""
-        if len(candidates) < self._config.min_cluster_size:
-            return []
-
-        fact_ids = [c["fact_id"] for c in candidates]
-        fact_map = {c["fact_id"]: c for c in candidates}
-
-        # Build entity-to-fact index
-        entity_index: dict[str, list[str]] = defaultdict(list)
-        for c in candidates:
-            for entity in c.get("canonical_entities", []):
-                entity_index[entity].append(c["fact_id"])
-
-        # Union-Find by entity overlap
-        uf = _UnionFind(fact_ids)
-        for i, fid_a in enumerate(fact_ids):
-            entities_a = set(fact_map[fid_a].get("canonical_entities", []))
-            for fid_b in fact_ids[i + 1:]:
-                entities_b = set(
-                    fact_map[fid_b].get("canonical_entities", []),
-                )
-                if len(entities_a & entities_b) >= self._config.min_entity_overlap:
-                    uf.union(fid_a, fid_b)
-
-        # Extract components and sub-cluster temporally
-        clusters: list[ConsolidationCluster] = []
-        for _root, group_ids in uf.components().items():
-            if len(group_ids) < self._config.min_cluster_size:
-                continue
-            sub_clusters = self._temporal_subcluster(group_ids, fact_map)
-            for sc_ids in sub_clusters:
-                if len(sc_ids) < self._config.min_cluster_size:
-                    continue
-                # Cap cluster size (HR: prevents huge gists)
-                sc_ids = sc_ids[: self._config.max_cluster_size]
-
-                # Compute shared entities (appear in ALL facts)
-                entity_sets = [
-                    set(fact_map[fid].get("canonical_entities", []))
-                    for fid in sc_ids
-                ]
-                shared = (
-                    set.intersection(*entity_sets) if entity_sets else set()
-                )
-
-                # Compute temporal centroid
-                dates = [
-                    _parse_date(
-                        fact_map[fid].get("observation_date")
-                        or fact_map[fid].get("created_at"),
-                    )
-                    for fid in sc_ids
-                ]
-                valid_dates = [d for d in dates if d is not None]
-                centroid = _temporal_midpoint(valid_dates)
-
-                # Average retention
-                avg_ret = sum(
-                    fact_map[fid].get("retention_score", 0.0)
-                    for fid in sc_ids
-                ) / len(sc_ids)
-
-                clusters.append(
-                    ConsolidationCluster(
-                        cluster_id=_new_id(),
-                        fact_ids=tuple(sc_ids),
-                        shared_entities=tuple(sorted(shared)),
-                        temporal_centroid=centroid,
-                        avg_retention=round(avg_ret, 4),
-                        fact_count=len(sc_ids),
-                    ),
-                )
-
-        return clusters
-
-    def _temporal_subcluster(
-        self,
-        fact_ids: list[str],
-        fact_map: dict[str, dict],
-    ) -> list[list[str]]:
-        """Split an entity group into temporal sub-clusters within window."""
-        dated: list[tuple[str, datetime | None]] = []
-        for fid in fact_ids:
-            raw = (
-                fact_map[fid].get("observation_date")
-                or fact_map[fid].get("created_at")
-            )
-            dated.append((fid, _parse_date(raw)))
-
-        dated.sort(key=lambda t: t[1] or datetime.min)
-
-        sub_clusters: list[list[str]] = [[]]
-        prev_dt: datetime | None = None
-        window_seconds = self._config.temporal_window_days * 86400
-
-        for fid, dt in dated:
-            if dt is None:
-                sub_clusters[-1].append(fid)
-                continue
-            if (
-                prev_dt is not None
-                and (dt - prev_dt).total_seconds() > window_seconds
-            ):
-                sub_clusters.append([])
-            sub_clusters[-1].append(fid)
-            prev_dt = dt
-
-        return [sc for sc in sub_clusters if sc]
-
-    # ------------------------------------------------------------------
-    # Step 3: Extract gist
-    # ------------------------------------------------------------------
-
-    def _step3_extract_gist(
-        self,
-        cluster: ConsolidationCluster,
-        profile_id: str,
-    ) -> GistResult:
-        """Extract a single gist from the cluster.
-
-        Mode B (LLM) attempted first if available; falls back to Mode A (rules).
-        """
-        # Fetch fact content
-        placeholders = ",".join("?" for _ in cluster.fact_ids)
-        rows = self._db.execute(
-            f"SELECT fact_id, content, importance, confidence, "
-            f"       canonical_entities_json "
-            f"FROM atomic_facts "
-            f"WHERE fact_id IN ({placeholders}) AND profile_id = ? "
-            # Belt and braces with the identify query above: a cluster assembled
-            # before a row was withheld must not contribute its text to a gist.
-            f"  AND COALESCE(quarantined, 0) = 0",
-            (*cluster.fact_ids, profile_id),
-        )
-
-        facts: list[dict] = []
-        for r in rows or []:
-            d = dict(r)
-            raw = d.get("canonical_entities_json") or "[]"
-            try:
-                d["canonical_entities"] = json.loads(raw)
-            except (json.JSONDecodeError, TypeError):
-                d["canonical_entities"] = []
-            facts.append(d)
-
-        if not facts:
-            return GistResult(
-                gist_text="[empty cluster]",
-                key_entities=(),
-                extraction_mode="rules",
-                representative_fact_id="",
-            )
-
-        # Configuration first, then availability. Mode A sets use_llm_gist=False,
-        # so this refuses because the mode says to — not because an LLM happened
-        # not to be passed in. The previous order tested availability first,
-        # which meant a caller that supplied a model to a Mode A engine would
-        # have got an LLM call on a mode whose whole promise is that nothing
-        # does.
-        if (
-            self._config.use_llm_gist
-            and self._llm is not None
-        ):
-            try:
-                gist = self._extract_gist_llm(
-                    facts, cluster.shared_entities,
-                )
-                if gist is not None:
-                    return gist
-            except Exception as exc:
-                logger.warning("LLM gist failed, falling back to rules: %s", exc)
-
-        # Mode A: rules-based
-        return self._extract_gist_mode_a(facts, cluster.shared_entities)
-
-    def _extract_gist_llm(
-        self,
-        facts: list[dict],
-        shared_entities: tuple[str, ...],
-    ) -> GistResult | None:
-        """LLM-based gist extraction (Mode B). Returns None if validation fails."""
-        if self._llm is None:
-            return None
-
-        fact_lines = "\n".join(
-            f"{i + 1}. {f['content']}" for i, f in enumerate(facts)
-        )
-        entity_str = ", ".join(shared_entities)
-
-        prompt = (
-            f"Summarize these {len(facts)} related memories into one "
-            f"concise factual statement.\n"
-            f"Preserve all key entities: {entity_str}.\n\n"
-            f"Memories:\n{fact_lines}\n\n"
-            f"Consolidated statement:"
-        )
-
-        response = self._llm.generate(
-            prompt,
-            system="You are a precise memory consolidator.",
-        )
-
-        # Validate entity coverage (HR-03)
-        if shared_entities:
-            mentioned = sum(
-                1 for e in shared_entities
-                if e.lower() in response.lower()
-            )
-            coverage = mentioned / len(shared_entities)
-            if coverage < self._config.min_entity_coverage:
-                logger.info(
-                    "LLM gist entity coverage %.2f < %.2f, falling back",
-                    coverage, self._config.min_entity_coverage,
-                )
-                return None
-
-        # Truncate if needed
-        gist_text = response
-        if len(gist_text) > self._config.max_gist_chars:
-            gist_text = gist_text[: self._config.max_gist_chars - 3] + "..."
-
-        return GistResult(
-            gist_text=gist_text,
-            key_entities=shared_entities,
-            extraction_mode="llm",
-            representative_fact_id="",
-        )
-
-    def _extract_gist_mode_a(
-        self,
-        facts: list[dict],
-        shared_entities: tuple[str, ...],
-    ) -> GistResult:
-        """Rules-based gist: representative fact + entity summary."""
-        # Find representative (highest importance * confidence)
-        best = max(
-            facts,
-            key=lambda f: f.get("importance", 0) * f.get("confidence", 0),
-        )
-
-        # Entity frequency across cluster
-        entity_freq: dict[str, int] = defaultdict(int)
-        for f in facts:
-            for e in f.get("canonical_entities", []):
-                entity_freq[e] += 1
-        top_entities = sorted(
-            entity_freq, key=lambda k: entity_freq[k], reverse=True,
-        )[:5]
-
-        entity_summary = ", ".join(top_entities) if top_entities else ""
-        gist = f"{best['content']} [Entities: {entity_summary}]"
-
-        if len(gist) > self._config.max_gist_chars:
-            gist = gist[: self._config.max_gist_chars - 3] + "..."
-
-        return GistResult(
-            gist_text=gist,
-            key_entities=tuple(top_entities),
-            extraction_mode="rules",
-            representative_fact_id=best["fact_id"],
-        )
-
-    # ------------------------------------------------------------------
-    # Step 4: Compress source embeddings
-    # ------------------------------------------------------------------
-
-    def _step4_compress_embeddings(
-        self,
-        cluster: ConsolidationCluster,
-        profile_id: str,
-    ) -> tuple[int, int]:
-        """Compress source fact embeddings. Returns (bytes_before, bytes_after).
-
-        PolarQuant is optional (HR-10). If not available, marks as pending.
-        """
-        if not self._config.compress_embeddings:
-            return (0, 0)
-
-        total_before = 0
-        total_after = 0
-
-        for fact_id in cluster.fact_ids:
-            bb, ba = self._compress_single_embedding(fact_id, profile_id)
-            total_before += bb
-            total_after += ba
-
-        return (total_before, total_after)
-
-    def _compress_single_embedding(
-        self,
-        fact_id: str,
-        profile_id: str,
-    ) -> tuple[int, int]:
-        """Compress one fact's embedding. Returns (bytes_before, bytes_after).
-
-        PolarQuant is optional (HR-10). When unavailable, records the
-        uncompressed byte count so the pipeline can still report metrics.
-        Does NOT write invalid values to embedding_quantization_metadata.
-        """
-        meta = self._db.execute(
-            "SELECT vec_rowid, dimension FROM embedding_metadata "
-            "WHERE fact_id = ? AND profile_id = ?",
-            (fact_id, profile_id),
-        )
-        if not meta:
-            return (0, 0)
-
-        d = dict(meta[0])
-        dim = d.get("dimension", 768)
-
-        # Check existing quantization status
-        eq_meta = self._db.execute(
-            "SELECT quantization_level FROM embedding_quantization_metadata "
-            "WHERE fact_id = ? AND profile_id = ?",
-            (fact_id, profile_id),
-        )
-        if eq_meta:
-            level = dict(eq_meta[0]).get("quantization_level", "float32")
-            if level in ("polar2", "polar4", "deleted"):
-                return (0, 0)  # Already compressed
-
-        bytes_before = dim * 4  # float32
-
-        # Try PolarQuant (Phase B) — optional dependency (HR-10)
-        try:
-            from superlocalmemory.math.polar_quant import PolarQuantEncoder
-            # Phase B exists. Mark as polar2 for the scheduler to actually
-            # perform the quantization with the raw embedding data.
-            self._db.execute(
-                "INSERT OR REPLACE INTO embedding_quantization_metadata "
-                "(fact_id, profile_id, quantization_level, bit_width) "
-                "VALUES (?, ?, 'polar2', 2)",
-                (fact_id, profile_id),
-            )
-            # Estimated compressed size: radius(4 bytes) + packed angles
-            bytes_after = 4 + (dim * self._config.target_bit_width + 7) // 8
-            return (bytes_before, bytes_after)
-        except ImportError:
-            pass
-
-        # PolarQuant not available: no compression, no metadata change.
-        # The bytes are uncompressed but we track them for metrics.
-        return (bytes_before, bytes_before)
-
-    # ------------------------------------------------------------------
-    # Step 5: Store block + archive source facts
-    # ------------------------------------------------------------------
-
-    def _step5_store_block(
-        self,
-        cluster: ConsolidationCluster,
-        gist: GistResult,
-        profile_id: str,
-    ) -> str:
-        """Create CCQ consolidated block and archive source facts.
-
-        Source facts are SOFT-ARCHIVED (HR-04), never deleted.
-        Gist embedding stored at float32 (HR-05).
-
-        Ordering guarantee: the gist replacement is committed as a live
-        atomic fact BEFORE source facts are archived.  This ensures there
-        is never a window where neither the sources nor the replacement
-        are recallable.
-        """
-        block_id = _new_id()
-
-        # Generate gist embedding BEFORE the transaction.  Embedding generation
-        # is a pure CPU/ML computation with no DB side effects; running it outside
-        # the transaction avoids holding a write lock during a potentially slow
-        # model forward pass.  If generation fails, we abort before any write.
-        # Note: gist_embedding_rowid is currently always None because vec0
-        # insertion requires VectorStore integration (future work).  When that
-        # integration lands, the rowid write must move INSIDE the transaction
-        # below so a rollback removes the embedding row together with the block.
-        gist_embedding_rowid: int | None = None
-        if self._embedder is not None:
-            try:
-                self._embedder.encode(gist.gist_text)
-            except Exception as exc:
-                logger.warning("Gist embedding generation failed: %s", exc)
-
-        from superlocalmemory.storage.models import AtomicFact, MemoryRecord
-        from superlocalmemory.core.lifecycle_state import set_fact_lifecycle_zone
-
-        # All four writes are wrapped in a single transaction so a mid-write
-        # failure (e.g. store_ccq_block raising) cannot leave an orphan gist
-        # AtomicFact with no corresponding ccq_consolidated_blocks row.
-        # set_fact_lifecycle_zone is transaction-aware and reuses the active
-        # connection rather than opening a nested one.
-        with self._db.transaction():
-            # --- Store replacement fact FIRST (ordering guarantee) ---
-            # The gist is persisted as a live atomic fact so it is immediately
-            # recallable.  Source archival happens only after this write lands.
-            self._db.store_memory(
-                MemoryRecord(
-                    memory_id=block_id,
-                    profile_id=profile_id,
-                    content=gist.gist_text,
-                    session_id="ccq",
-                )
-            )
-            self._db.store_fact(
-                AtomicFact(
-                    memory_id=block_id,
-                    profile_id=profile_id,
-                    content=gist.gist_text,
-                    canonical_entities=list(gist.key_entities),
-                    importance=0.8,
-                    confidence=1.0,
-                    session_id="ccq",
-                )
-            )
-
-            # Store the consolidated block — must follow the gist fact INSERT so
-            # the step-1 identify filter (NOT IN ccq_consolidated_blocks) sees the
-            # correct state on any immediate re-run.
-            self._db.store_ccq_block(
-                block_id=block_id,
-                profile_id=profile_id,
-                content=gist.gist_text,
-                source_fact_ids=json.dumps(list(cluster.fact_ids)),
-                gist_embedding_rowid=gist_embedding_rowid,
-                char_count=len(gist.gist_text),
-                cluster_id=cluster.cluster_id,
-            )
-
-            # Archive source facts AFTER replacement is committed (HR-04: soft-archive, never delete)
-            set_fact_lifecycle_zone(
-                self._db, cluster.fact_ids, "archive", profile_id=profile_id,
-            )
-            for fact_id in cluster.fact_ids:
-                # Log access event
-                self._db.execute(
-                    "INSERT INTO fact_access_log "
-                    "(log_id, fact_id, profile_id, accessed_at, "
-                    " access_type, session_id) "
-                    "VALUES (?, ?, ?, datetime('now'), 'consolidation', 'ccq')",
-                    (_new_id(), fact_id, profile_id),
-                )
-
-        return block_id
-
-    # ------------------------------------------------------------------
-    # Step 6: Audit trail
-    # ------------------------------------------------------------------
-
-    def _step6_audit(
-        self,
-        cluster: ConsolidationCluster,
-        gist: GistResult,
-        bytes_before: int,
-        bytes_after: int,
-        block_id: str,
-        profile_id: str,
-    ) -> str:
-        """Record audit trail for this consolidation (HR-11)."""
-        audit_id = _new_id()
-        compression_ratio = (
-            bytes_before / bytes_after if bytes_after > 0 else 0.0
-        )
-
-        self._db.store_ccq_audit({
-            "audit_id": audit_id,
-            "profile_id": profile_id,
-            "cluster_id": cluster.cluster_id,
-            "block_id": block_id,
-            "fact_ids": json.dumps(list(cluster.fact_ids)),
-            "fact_count": cluster.fact_count,
-            "gist_text": gist.gist_text,
-            "extraction_mode": gist.extraction_mode,
-            "bytes_before": bytes_before,
-            "bytes_after": bytes_after,
-            "compression_ratio": round(compression_ratio, 2),
-            "shared_entities": json.dumps(list(cluster.shared_entities)),
-        })
-
-        return audit_id
-
-    # ------------------------------------------------------------------
-    # Helpers
-    # ------------------------------------------------------------------
-
-    @staticmethod
-    def _empty_result() -> CCQPipelineResult:
-        """Return a zero-work pipeline result."""
-        return CCQPipelineResult(
-            clusters_processed=0,
-            blocks_created=0,
-            facts_archived=0,
-            total_bytes_before=0,
-            total_bytes_after=0,
-            compression_ratio=0.0,
-            audit_entries=(),
-            errors=(),
-        )