cortexm 0.3.0__py3-none-any.whl

cortexm/text/tokenizer.py

@@ -0,0 +1,79 @@
+"""Deterministic tokenizer & sentence segmentation (μ=0 building block)."""
+
+from __future__ import annotations
+
+import re
+
+WORD_RE = re.compile(r"[a-zA-Z][a-zA-Z'’-]*|\d+(?:\.\d+)?")
+ABBREV = {"mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "vs", "etc",
+          "e.g", "i.e", "fig", "inc", "ltd", "co", "u.s", "u.k"}
+
+_SENT_SPLIT = re.compile(r"(?<=[.!?])[\"')\]]*\s+|\n{2,}")
+_SENT_GUARD = re.compile(r"^(?:[A-Z0-9\"'(]|…)")
+
+
+def words(text: str) -> list[str]:
+    return [w.lower().replace("’", "'") for w in WORD_RE.findall(text)]
+
+
+def sentences(text: str) -> list[tuple[int, int, str]]:
+    """Sentence segmentation with (start, end, sentence) spans."""
+    out: list[tuple[int, int, str]] = []
+    if not text or not text.strip():
+        return out
+    pos = 0
+    for raw in _SENT_SPLIT.split(text):
+        s = raw.strip()
+        if not s:
+            continue
+        start = text.find(s[:24], pos)
+        if start < 0:
+            start = pos
+        end = start + len(s)
+        out.append((start, end, s))
+        pos = end
+    # merge fragments ending in abbreviations ("I met Dr. Chen.")
+    merged: list[tuple[int, int, str]] = []
+    for item in out:
+        if merged:
+            ps, pe, ptext = merged[-1]
+            tail = re.sub(r"[^\w.]", "", ptext.split()[-1] if ptext.split() else "")
+            if tail.rstrip(".").lower() in ABBREV:
+                nxt = text[pe:item[1]]
+                merged[-1] = (ps, item[1], text[ps:item[1]])
+                continue
+        merged.append(item)
+    return merged
+
+
+def strip_punct(s: str) -> str:
+    return re.sub(r"[^\w\s'-]", " ", s).strip()
+
+
+STOPWORDS = {
+    "a", "an", "the", "and", "or", "but", "if", "of", "to", "in", "on", "at",
+    "for", "with", "is", "are", "was", "were", "be", "been", "am", "do",
+    "does", "did", "have", "has", "had", "i", "you", "he", "she", "it", "we",
+    "they", "me", "my", "your", "his", "her", "its", "our", "their", "this",
+    "that", "these", "those", "as", "so", "than", "then", "there", "here",
+    "what", "which", "who", "whom", "when", "where", "why", "how", "will",
+    "would", "can", "could", "should", "shall", "may", "might", "just",
+    "about", "from", "by", "up", "out", "not", "no", "yes", "oh", "well",
+}
+
+
+def content_words(text: str) -> list[str]:
+    return [w for w in words(text) if w not in STOPWORDS and len(w) > 1]
+
+
+def cap_sequences(text: str) -> list[str]:
+    """Capitalized multi-word sequences — poor-man's NER (μ=0)."""
+    seqs = re.findall(
+        r"\b([A-Z][a-zA-Z'&-]*(?:[ ](?:of|the|and|de|van|for)[ ])?[ ]*[A-Z][a-zA-Z'&-]*)+\b",
+        text)
+    out = []
+    for s in seqs:
+        s = " ".join(s.split())
+        if len(s) > 2 and not s.islower() and "." not in s:
+            out.append(s)
+    return out

cortexm/trace/blob_arena.py

@@ -0,0 +1,277 @@
+"""Sidecar blob arena — Aeon-inspired off-graph text storage.
+
+arXiv:2601.15311 (Aeon): large text is stored off-graph in an
+append-only mmap-backed blob file with generational GC. Graph nodes
+hold only a 64-byte preview + offset.
+
+Context-M's chunks table currently stores full text inline
+(`text TEXT NOT NULL`). For typical personas this is fine — chunks
+are 200-400 bytes. But for use cases that ingest long documents
+(research papers, meeting transcripts, issue threads), the chunks
+table balloons and the working set of the SQLite page cache gets
+polluted by long text the graph rarely needs.
+
+This module provides an OPT-IN sidecar blob arena:
+
+    BlobArena(path) — opens/creates an mmap'd blob file at `path`
+        .put(text) -> (blob_id, offset, length)
+        .get(offset, length) -> bytes
+        .preview(text, n=64) -> first n bytes (the in-graph preview)
+
+The host migrates chunks by:
+    1. creating a BlobArena
+    2. for each chunk: arena.put(text) -> (blob_id, offset, len)
+    3. updating the chunks row: text = preview, blob_offset = offset,
+       blob_len = len (schema migration adds the columns)
+    4. on retrieval: chunks.text gives the preview (fast); full text
+       is fetched via arena.get(offset, len) only when the audit
+       / retrieval path actually needs it
+
+Generational GC is out of scope for v1 (chunks are append-mostly in
+practice — retired facts are deactivated but the chunk text is kept
+for audit). The arena is a single mmap'd file; concurrent writers
+must hold a lock (the arena serializes via flock).
+
+BACKWARD COMPATIBILITY: the chunks schema migration is OPT-IN. The
+default Memory() path still stores text inline. To enable the
+sidecar, call Memory.enable_blob_arena(path) after construction.
+"""
+from __future__ import annotations
+
+import io
+import mmap
+import os
+import threading
+import zlib
+from pathlib import Path
+from typing import Iterator
+
+
+# Schema additions for the chunks table (applied by TraceStore when
+# the arena is enabled). The columns are nullable so existing rows
+# (with inline text) keep working — a NULL blob_offset means "use the
+# inline text column".
+SCHEMA_MIGRATION = """
+ALTER TABLE chunks ADD COLUMN blob_offset INTEGER DEFAULT NULL;
+ALTER TABLE chunks ADD COLUMN blob_len INTEGER DEFAULT NULL;
+ALTER TABLE chunks ADD COLUMN blob_compressed INTEGER DEFAULT 0;
+"""
+
+
+class BlobArena:
+    """Append-only mmap-backed blob file.
+
+    Layout:
+        - 8-byte header: magic + version + length
+        - records: [8-byte length | length bytes of payload]
+          (payload may be zlib-compressed if compressed=1 in the chunk
+          row; the arena itself is format-agnostic to the bytes)
+
+    The arena serializes writes via a process-local threading.Lock
+    AND an flock on the file for cross-process safety. Concurrent
+    readers don't need the lock — mmap pages are CoW.
+    """
+
+    MAGIC = b"BLB1"  # Blob Layer v1
+    HEADER_LEN = 16
+
+    def __init__(self, path: str | os.PathLike) -> None:
+        self.path = Path(path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        # create or open
+        is_new = not self.path.exists()
+        if is_new:
+            with open(self.path, "wb") as f:
+                f.write(self.MAGIC + b"\x00" * 12)  # magic + 12 bytes pad
+        self._fd = open(self.path, "r+b")
+        # write-lock for cross-process safety
+        try:
+            import fcntl
+            fcntl.flock(self._fd.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except (ImportError, OSError):
+            pass  # Windows or already locked by same process
+        self._mmap: mmap.mmap | None = None
+        self._lock = threading.Lock()
+        self._open_mmap()
+
+    def _open_mmap(self) -> None:
+        if self._mmap is None:
+            try:
+                self._mmap = mmap.mmap(
+                    self._fd.fileno(), 0, access=mmap.ACCESS_WRITE)
+            except (ValueError, OSError):
+                # empty file (just header) — fall back to file I/O
+                self._mmap = None
+
+    def put(self, data: bytes, compress: bool = True) -> tuple[int, int, int, bool]:
+        """Append `data` to the blob file.
+
+        Returns (blob_id, offset, length, was_compressed).
+            blob_id — monotonically increasing int (records / appended)
+            offset  — byte offset in the file where the record starts
+            length  — payload length in bytes
+            was_compressed — whether zlib compression was applied
+
+        Compression: enabled by default for text payloads > 256 bytes;
+        small payloads are stored raw because the zlib header (5+ bytes)
+        makes small blobs larger.
+        """
+        if isinstance(data, str):
+            data = data.encode("utf-8")
+        if compress and len(data) > 256:
+            compressed = zlib.compress(data, level=6)
+            if len(compressed) < len(data) - 8:  # only if meaningful savings
+                payload = compressed
+                was_compressed = True
+            else:
+                payload = data
+                was_compressed = False
+        else:
+            payload = data
+            was_compressed = False
+
+        with self._lock:
+            # we need to write 8 bytes (length) + len(payload) bytes
+            rec_len = len(payload)
+            # current file size = where this record starts
+            self._fd.seek(0, io.SEEK_END)
+            offset = self._fd.tell()
+            # write 8-byte length + payload
+            self._fd.write(rec_len.to_bytes(8, "little"))
+            self._fd.write(payload)
+            self._fd.flush()
+            # invalidate the mmap view so the next get() re-reads
+            if self._mmap is not None:
+                try:
+                    self._mmap.close()
+                except Exception:
+                    pass
+                self._mmap = None
+            self._open_mmap()
+            blob_id = offset  # offset IS the id (unique within file)
+            return blob_id, offset, rec_len, was_compressed
+
+    def get(self, offset: int, length: int,
+            was_compressed: bool = False) -> bytes:
+        """Read a record from the blob file.
+
+        `offset` and `length` come from the chunks table. If
+        `was_compressed` is True (stored in the chunk row), the
+        returned bytes are zlib-decompressed before return.
+        """
+        with self._lock:
+            self._fd.seek(offset)
+            raw_len = self._fd.read(8)
+            if len(raw_len) != 8:
+                raise IOError(f"short read at offset {offset}")
+            rec_len = int.from_bytes(raw_len, "little")
+            payload = self._fd.read(rec_len)
+            if len(payload) != rec_len:
+                raise IOError(
+                    f"short payload read at offset {offset}: "
+                    f"expected {rec_len}, got {len(payload)}")
+        if was_compressed:
+            try:
+                return zlib.decompress(payload)
+            except zlib.error:
+                return payload  # return raw on decompression failure
+        return payload
+
+    def get_text(self, offset: int, length: int,
+                 was_compressed: bool = False) -> str:
+        """Convenience: get bytes and decode as UTF-8."""
+        return self.get(offset, length, was_compressed).decode(
+            "utf-8", errors="replace")
+
+    def close(self) -> None:
+        with self._lock:
+            if self._mmap is not None:
+                try:
+                    self._mmap.close()
+                except Exception:
+                    pass
+                self._mmap = None
+            try:
+                self._fd.close()
+            except Exception:
+                pass
+
+    def __enter__(self) -> "BlobArena":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()
+
+
+# ---------- Migration helper ------------------------------------------
+
+def migrate_chunks_to_arena(store, arena: BlobArena,
+                             batch_size: int = 1000) -> dict:
+    """Migrate existing chunks.text rows into the sidecar arena.
+
+    For each chunk:
+        - put text into arena -> (blob_id, offset, len, compressed)
+        - update the row: blob_offset=offset, blob_len=len,
+          blob_compressed=compressed, text=preview (first 64 bytes)
+    The original text column is REPLACED with the preview — the full
+    text lives only in the arena. Existing readers that use chunks.text
+    will see the preview (good for display); readers that need full
+    text must call arena.get_text(offset, len, compressed).
+
+    Idempotent: chunks with blob_offset IS NOT NULL are skipped.
+    """
+    # ensure the schema migration has run
+    cols = {r[1] for r in store.conn.execute("PRAGMA table_info(chunks)").fetchall()}
+    if "blob_offset" not in cols:
+        for stmt in SCHEMA_MIGRATION.strip().split(";"):
+            stmt = stmt.strip()
+            if stmt:
+                store.conn.execute(stmt)
+        store.conn.commit()
+
+    migrated = 0
+    skipped = 0
+    while True:
+        rows = store.conn.execute(
+            "SELECT id, text FROM chunks WHERE blob_offset IS NULL "
+            f"LIMIT {batch_size}").fetchall()
+        if not rows:
+            break
+        for chunk_id, text in rows:
+            if not text:
+                skipped += 1
+                continue
+            data = text.encode("utf-8")
+            _, offset, length, compressed = arena.put(data, compress=True)
+            preview = text[:64] + ("..." if len(text) > 64 else "")
+            store.conn.execute(
+                "UPDATE chunks SET text=?, blob_offset=?, blob_len=?, "
+                "blob_compressed=? WHERE id=?",
+                (preview, offset, length, 1 if compressed else 0, chunk_id))
+            migrated += 1
+        store.conn.commit()
+    return {"migrated": migrated, "skipped": skipped}
+
+
+def get_chunk_text(store, arena: BlobArena, chunk_id: str) -> str:
+    """Fetch full text for a chunk — from the arena if blob_offset is
+    set, otherwise fall back to inline text.
+
+    Use this in the audit / retrieval path when the 64-byte preview
+    in chunks.text isn't enough and you need the full source.
+    """
+    row = store.conn.execute(
+        "SELECT text, blob_offset, blob_len, blob_compressed "
+        "FROM chunks WHERE id=?", (chunk_id,)).fetchone()
+    if not row:
+        return ""
+    text, offset, length, compressed = row
+    if offset is None:
+        return text or ""
+    return arena.get_text(offset, length, bool(compressed))
+
+
+__all__ = [
+    "BlobArena", "SCHEMA_MIGRATION",
+    "migrate_chunks_to_arena", "get_chunk_text",
+]

cortexm/trace/consolidate.py

@@ -0,0 +1,337 @@
+"""Dreaming / consolidation — Aeon-inspired idle-time optimization.
+
+arXiv:2601.15311 (Aeon) describes a background task analogous to
+biological sleep: defragmentation, GC, and consolidation of the
+verbose Trace into compressed long-term episodic summaries.
+
+Context-M's Memory Git is currently passive — it version-controls
+facts but doesn't optimize them. This module turns it into an ACTIVE
+memory optimizer that runs during idle periods:
+
+    consolidate(store, palace, prefetcher, ...) ->
+      - merges redundant triples in the Trace (same subject/relation,
+        near-duplicate values) via MERGED_WITH edges
+      - retires facts past their valid_to + grace period
+      - defragments the palace by rebuilding the in-memory packed matrix
+        from active facts only (drops retired/merged IDs)
+      - re-trains the MBTB prefetcher from recent query access patterns
+
+The pass is IDEMPOTENT and SAFE — every change is a commit on the
+current branch, every retired fact is still queryable via
+allow_inactive=True, every merged fact keeps a MERGED_WITH edge so
+the original triple is recoverable for audit. A failed pass rolls back
+via the existing Memory Git ancestry.
+
+NOTE: this is NOT a separate daemon — it's a function the host app
+calls when idle (e.g.overnight cron, `cortexm consolidate` CLI, or
+an `on_idle` hook in the MCP server).
+"""
+from __future__ import annotations
+
+import datetime as _dt
+from datetime import datetime, timezone
+from typing import Iterable
+
+from cortexm.trace.edges import MERGED_WITH, RETRACTED_BY
+from cortexm.util import iso, similarity
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def consolidate(store, palace=None, prefetcher=None, *,
+                user_id: str | None = None,
+                merge_threshold: float = 0.92,
+                retire_grace_days: int = 365,
+                defrag_palace: bool = True,
+                retrain_prefetcher: bool = True,
+                run_fade: bool = True,
+                run_tmt: bool = False,
+                run_cognition: bool = False,
+                dry_run: bool = False,
+                fade_cfg: dict | None = None,
+                tmt_cfg: dict | None = None,
+                cognition_cfg: dict | None = None) -> dict:
+    """Run a single consolidation pass.
+
+    Returns a stats dict:
+        {merged_pairs, retired_facts, palace_defragged,
+         prefetcher_retrained, fade_stats, tmt_stats,
+         cognition_stats, commit_id, dry_run}
+
+    Parameters:
+        store           — TraceStore
+        palace          — MemoryPalace (optional; only needed for defrag)
+        prefetcher      — Prefetcher (optional; only needed for retrain)
+        user_id         — restrict pass to one user (None = all users)
+        merge_threshold — jaccard similarity above which two facts
+                           (same subject+relation) are merged
+        retire_grace_days — facts with valid_to older than this many
+                            days are retired (deactivated)
+        defrag_palace   — rebuild the palace packed matrix from active facts
+        retrain_prefetcher — rebuild the MBTB from access_count stats
+        run_fade        — also run FadeMem sweep (decay + deactivate + merge)
+        run_tmt         — also run TiMem TMT hierarchy build
+        run_cognition   — also run the HMS-style Cognition Engine pass
+                           (PatternScanner + AbstractionEngine + GapDetector
+                           + HypothesisEngine + AnalogyDetector). Emits
+                           HYPOTHESIZED_BY edges with confidence < 0.5 —
+                           never active in retrieval unless promoted.
+        fade_cfg        — kwargs for fade_sweep (lambda_, thresholds, etc.)
+        tmt_cfg         — kwargs for tmt_build (cluster mins, etc.)
+        cognition_cfg   — kwargs for run_cognition_pass
+        dry_run         — compute the changes but don't apply them
+    """
+    stats = {
+        "merged_pairs": 0,
+        "retired_facts": 0,
+        "palace_defragged": False,
+        "prefetcher_retrained": False,
+        "fade_stats": None,
+        "tmt_stats": None,
+        "cognition_stats": None,
+        "commit_id": None,
+        "dry_run": dry_run,
+    }
+
+    # ---------- 1. Merge redundant triples -------------------------------
+    # Group active facts by (user_id, subject, relation) and find
+    # near-duplicate values within each group.
+    where = "is_active=1 AND quarantined=0"
+    args: tuple = ()
+    if user_id is not None:
+        where += " AND user_id=?"
+        args = (user_id,)
+    rows = store.conn.execute(
+        f"SELECT id, subject, relation, value, user_id, confidence, "
+        f"access_count FROM facts WHERE {where} ORDER BY user_id, subject, "
+        f"relation, valid_from", args).fetchall()
+    groups: dict[tuple, list[dict]] = {}
+    for r in rows:
+        r = dict(r)
+        key = (r["user_id"], r["subject"], r["relation"])
+        groups.setdefault(key, []).append(r)
+
+    merge_pairs: list[tuple[str, str, float]] = []
+    for key, group in groups.items():
+        if len(group) < 2:
+            continue
+        # pairwise similarity (small groups, so O(n^2) is fine)
+        for i, a in enumerate(group):
+            for b in group[i + 1:]:
+                sim = similarity(a["value"], b["value"])
+                if sim >= merge_threshold:
+                    # keep the higher-confidence one; merge the other into it
+                    if a["confidence"] >= b["confidence"]:
+                        keep, drop = a, b
+                    else:
+                        keep, drop = b, a
+                    merge_pairs.append((keep["id"], drop["id"], sim))
+
+    if not dry_run:
+        store.begin_batch()
+        commit = store.create_commit(
+            f"consolidate: merge {len(merge_pairs)} pairs, "
+            f"retire stale", n_facts=0)
+        stats["commit_id"] = commit
+
+        # apply merges
+        for keep_id, drop_id, sim in merge_pairs:
+            # deactivate the drop, wire MERGED_WITH edge so audits can
+            # always recover the original triple
+            store.update_fact(
+                drop_id, is_active=0, tx_to=iso(_now()),
+                retired_commit=commit,
+                provenance={"merged_into": keep_id,
+                            "merge_sim": round(sim, 4),
+                            "consolidated_at": iso(_now())})
+            store.add_edge(keep_id, drop_id, MERGED_WITH,
+                          {"sim": round(sim, 4),
+                           "consolidated_at": iso(_now())})
+
+    stats["merged_pairs"] = len(merge_pairs)
+
+    # ---------- 2. Retire stale facts -----------------------------------
+    # Facts whose valid_to is older than retire_grace_days and which
+    # haven't been accessed recently (access_count == 0) — these are
+    # safe to retire; the bi-temporal model still serves them via
+    # allow_inactive=True.
+    cutoff = (_now() - _dt.timedelta(days=retire_grace_days)).strftime(
+        "%Y-%m-%d")
+    retire_where = (
+        f"is_active=1 AND valid_to IS NOT NULL AND valid_to < ? "
+        f"AND valid_to != '' AND access_count = 0")
+    retire_args: tuple = (cutoff,)
+    if user_id is not None:
+        retire_where += " AND user_id=?"
+        retire_args = (cutoff, user_id)
+    retire_rows = store.conn.execute(
+        f"SELECT id FROM facts WHERE {retire_where}", retire_args).fetchall()
+    retire_ids = [r[0] for r in retire_rows]
+
+    if not dry_run:
+        for fid in retire_ids:
+            store.update_fact(
+                fid, is_active=0, tx_to=iso(_now()),
+                retired_commit=commit,
+                provenance={"retired_by_consolidate": True,
+                            "consolidated_at": iso(_now())})
+    stats["retired_facts"] = len(retire_ids)
+
+    # ---------- 3. Palace defrag ----------------------------------------
+    # Rebuild the palace's packed matrix from active facts only.
+    # This drops retired / merged IDs from the in-memory index and
+    # re-tightens the page-clustered tree (currently a no-op for the
+    # SQLite-backed palace; for the in-memory palace it compacts).
+    if defrag_palace and palace is not None and not dry_run:
+        try:
+            # the palace's _n tracks active entries — a defrag pass
+            # re-builds the matrix by re-adding only active fact IDs.
+            # For the SQLite-backed palace this is a no-op (vectors
+            # are stored in a BLOB, not a packed matrix), but the
+            # call still flushes any in-memory dirty state.
+            if hasattr(palace, "defrag"):
+                palace.defrag()
+            elif hasattr(palace, "close"):
+                palace.close()
+            stats["palace_defragged"] = True
+        except Exception:
+            pass
+
+    # ---------- 4. Prefetcher retrain ------------------------------------
+    # The MBTB prefetcher tracks co-access patterns. Re-training from
+    # access_count stats lets it pick up shifts in user behavior.
+    if retrain_prefetcher and prefetcher is not None and not dry_run:
+        try:
+            if hasattr(prefetcher, "retrain"):
+                prefetcher.retrain(store)
+            elif hasattr(prefetcher, "rebuild"):
+                prefetcher.rebuild(store)
+            stats["prefetcher_retrained"] = True
+        except Exception:
+            pass
+
+    # ---------- 5. FadeMem sweep (decay + deactivate + cluster merge) ----
+    # Biologically-inspired forgetting: exponential decay on retention
+    # scores, with access-driven reconsolidation (frequently-retrieved
+    # facts resist decay). Deactivates facts whose retention_score drops
+    # below fade_deactivate_threshold; merges clusters of low-retention
+    # siblings. Bi-temporal safe: only is_active flips.
+    if run_fade and not dry_run:
+        try:
+            from cortexm.trace.fade import fade_sweep
+            fade_kwargs = dict(
+                lambda_=0.05,
+                access_boost=0.5,
+                contradiction_penalty=0.25,
+                deactivate_threshold=0.10,
+                merge_threshold=0.30,
+                merge_similarity=merge_threshold,
+                user_id=user_id,
+                dry_run=dry_run,
+            )
+            if fade_cfg:
+                fade_kwargs.update(fade_cfg)
+            stats["fade_stats"] = fade_sweep(store, palace, **fade_kwargs)
+        except Exception as e:
+            stats["fade_stats"] = {"error": str(e)}
+
+    # ---------- 6. TiMem TMT hierarchy build -----------------------------
+    # Episodic → session → day → persona abstraction. Each higher level
+    # is a derived fact with DERIVED_FROM edges back to its constituents.
+    # Retrieval can short-circuit to the appropriate level based on
+    # query complexity.
+    if run_tmt and not dry_run:
+        try:
+            from cortexm.trace.tmt import tmt_build
+            tmt_kwargs = dict(
+                session_cluster_mins=5,
+                persona_min_sessions=3,
+                user_id=user_id,
+            )
+            if tmt_cfg:
+                tmt_kwargs.update(tmt_cfg)
+            stats["tmt_stats"] = tmt_build(store, palace, **tmt_kwargs)
+        except Exception as e:
+            stats["tmt_stats"] = {"error": str(e)}
+
+    # ---------- 7. HMS Cognition Engine pass ----------------------------
+    # PatternScanner + AbstractionEngine + GapDetector + HypothesisEngine
+    # + AnalogyDetector. Surfaces structural regularities, builds
+    # prototype categories, fills in missing relations via hypotheses,
+    # finds cross-domain analogies. Output is derived facts with
+    # confidence < 0.5 and is_derived=1, never promoted to active
+    # retrieval unless explicitly confirmed by user input.
+    if run_cognition:
+        try:
+            from cortexm.cognition import run_cognition_pass
+            cog_kwargs = dict(
+                dry_run=dry_run,
+                user_id=user_id,
+            )
+            if cognition_cfg:
+                cog_kwargs.update(cognition_cfg)
+            cog_report = run_cognition_pass(store, palace=palace,
+                                              **cog_kwargs)
+            stats["cognition_stats"] = {
+                "scan": cog_report.scan,
+                "abstraction": cog_report.abstraction,
+                "gaps": cog_report.gaps,
+                "hypotheses": cog_report.hypotheses,
+                "analogies": cog_report.analogies,
+                "total_derived_facts": cog_report.total_derived_facts,
+                "duration_ms": round(cog_report.duration_ms, 2),
+                "cognition_commit_id": cog_report.commit_id,
+            }
+        except Exception as e:
+            stats["cognition_stats"] = {"error": str(e)}
+
+    if not dry_run:
+        store.end_batch()
+        if palace is not None and hasattr(palace, "close"):
+            palace.close()
+
+    return stats
+
+
+# ---------- Audit / inspection -----------------------------------------
+
+def consolidation_report(store, since: str | None = None) -> dict:
+    """Return a summary of consolidation activity since `since` (ISO).
+
+    Counts merged pairs, retired facts, and lists the commit IDs that
+    ran consolidation. Useful for the audit / governance dashboard.
+    """
+    where = "is_active=0 AND provenance LIKE ?"
+    pat = '%"consolidated_at"%'
+    if since:
+        where += " AND tx_to >= ?"
+        args = (pat, since)
+    else:
+        args = (pat,)
+    rows = store.conn.execute(
+        f"SELECT COUNT(*) FROM facts WHERE {where}", args).fetchone()
+    merged = 0
+    retired = 0
+    for r in store.conn.execute(
+        "SELECT provenance FROM facts WHERE " + where, args).fetchall():
+        prov = r[0] or ""
+        if "merged_into" in prov:
+            merged += 1
+        if "retired_by_consolidate" in prov:
+            retired += 1
+    commits = []
+    try:
+        for r in store.conn.execute(
+            "SELECT id, message FROM commits WHERE message LIKE ?",
+            ("%consolidate%",)).fetchall():
+            commits.append({"id": r[0], "message": r[1]})
+    except Exception:
+        pass
+    return {"merged_facts": merged, "retired_facts": retired,
+            "consolidation_commits": len(commits),
+            "recent_commits": commits[:10]}
+
+
+__all__ = ["consolidate", "consolidation_report"]