cortexm 0.3.0__py3-none-any.whl

cortexm/trace/fade.py

@@ -0,0 +1,245 @@
+"""FadeMem-style forgetting — biologically-inspired decay + retention.
+
+arXiv:2601.18642 (FadeMem, 2026) implements:
+  * Dual-layer hierarchy (STM / LTM)
+  * Differential decay rates modulated by:
+    - semantic relevance (matches current query intent)
+    - access frequency (reinforcement strengthens)
+    - temporal patterns (recent > old)
+    - contradiction pressure (superseded facts fade faster)
+  * LLM-guided conflict resolution (we keep μ=0 — uses similarity)
+  * Reported: 45% storage reduction with superior multi-hop reasoning
+
+Context-M's existing `retention_score` in lifecycle.py already does
+reinforcement x recency x contradiction. This module adds:
+
+  1. EXPONENTIAL DECAY — retention_score is multiplied by
+     exp(-lambda * age_days). Old untouched facts decay to zero.
+
+  2. ACCESS-DRIVEN RECONSOLIDATION — each retrieval event bumps the
+     fact's `last_accessed` AND its `reinforcement` by a small delta
+     (0.1 per access, capped at 3.0). This is "retrieval-induced
+     reconsolidation" — the act of remembering strengthens the memory,
+     exactly as in human recollection.
+
+  3. CLUSTER CONSOLIDATION — when multiple facts in the same
+     (subject, relation) group all have retention < 0.30, they're
+     candidates for FADE MERGE: keep the highest-confidence one,
+     mark the others inactive, wire MERGED_WITH edges (audit-safe).
+
+  4. SLEEP SWEEP — `fade_sweep(store, palace, cfg)` runs the full
+     decay + deactivate + merge pass. Designed to be called from
+     `cortexm consolidate` CLI as part of the Aeon-style dreaming
+     cycle. Idempotent — safe to call repeatedly.
+
+The sweep is BI-TEMPORAL SAFE: deactivated facts keep their
+`valid_from`/`valid_to`/`tx_from`/`tx_to` windows intact, so
+`allow_inactive=True` retrieval still serves them for temporal queries.
+The sweep only changes `is_active` and `provenance.fade_state`.
+"""
+from __future__ import annotations
+
+import datetime as _dt
+import math
+from datetime import datetime, timezone
+from typing import Iterable
+
+from cortexm.trace.edges import MERGED_WITH
+from cortexm.trace.fact import Fact
+from cortexm.trace.lifecycle import retention_score
+from cortexm.util import iso, similarity
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def fade_retention_score(fact: Fact, now: datetime,
+                          *, lambda_: float = 0.05,
+                          access_boost: float = 0.5,
+                          contradiction_penalty: float = 0.25) -> float:
+    """Exponential-decay retention score.
+
+    Builds on lifecycle.retention_score (which already does
+    reinforcement x recency x contradiction) and multiplies by an
+    exponential decay envelope:
+
+        fade_factor = exp(-lambda * age_days)
+
+    The result is a more aggressive forgetting curve than the
+    hyperbolic 1/(1+age/30) in lifecycle.retention_score — long-untouched
+    facts decay to ~0 instead of plateauing at ~0.3.
+
+    Lambda=0.05 means:
+      * 7 days idle:  0.97 fade_factor
+      * 30 days idle: 0.86
+      * 90 days idle: 0.61
+      * 365 days idle: 0.16  → below deactivate_threshold
+    """
+    base = retention_score(fact, now)
+    # parse the fact's tx_from (transaction time) — that's when the fact
+    # entered the store, which is the right "age" anchor for forgetting
+    from cortexm.util import parse_ts
+    tx = parse_ts(fact.tx_from) or now
+    age_days = max(0.0, (now - tx).total_seconds() / 86400.0)
+    fade = math.exp(-lambda_ * age_days)
+    # access boost: each retrieval multiplies retention by (1 + boost)
+    # capped to avoid runaway reinforcement
+    access_mult = 1.0 + min(access_boost * fact.access_count, 2.5)
+    # contradiction pressure: each chain_updates in provenance pulls
+    # retention down (the fact has been disputed)
+    chain_updates = fact.provenance.get("chain_updates", 0) \
+        if isinstance(fact.provenance, dict) else 0
+    contradiction_mult = max(0.1, 1.0 - contradiction_penalty * chain_updates)
+    return base * fade * access_mult * contradiction_mult
+
+
+def fade_sweep(store, palace=None, *,
+               lambda_: float = 0.05,
+               access_boost: float = 0.5,
+               contradiction_penalty: float = 0.25,
+               deactivate_threshold: float = 0.10,
+               merge_threshold: float = 0.30,
+               merge_similarity: float = 0.92,
+               user_id: str | None = None,
+               dry_run: bool = False) -> dict:
+    """Run a single FadeMem sweep over the store.
+
+    Steps:
+      1. Compute fade_retention_score for every active fact.
+      2. Deactivate facts with score < deactivate_threshold.
+         (Bi-temporal safe: only `is_active` flips, all temporal
+         fields preserved.)
+      3. For each (user_id, subject, relation) cluster where all
+         members have score < merge_threshold, MERGE the lowest-
+         confidence ones into the highest-confidence one via
+         MERGED_WITH edges.
+
+    Returns a stats dict. Idempotent — safe to call repeatedly.
+    """
+    now = _now()
+    stats = {
+        "scanned": 0,
+        "deactivated": 0,
+        "merged_into": 0,
+        "merged_dropped": 0,
+        "min_score": 1.0,
+        "max_score": 0.0,
+        "mean_score": 0.0,
+        "dry_run": dry_run,
+    }
+
+    where = "is_active=1 AND quarantined=0"
+    args: tuple = ()
+    if user_id is not None:
+        where += " AND user_id=?"
+        args = (user_id,)
+    rows = store.conn.execute(
+        f"SELECT id FROM facts WHERE {where}", args).fetchall()
+    fact_ids = [r[0] for r in rows]
+
+    facts = store.get_facts(fact_ids)
+    scores: dict[str, float] = {}
+    for f in facts:
+        s = fade_retention_score(f, now, lambda_=lambda_,
+                                  access_boost=access_boost,
+                                  contradiction_penalty=contradiction_penalty)
+        scores[f.id] = s
+        stats["scanned"] += 1
+        stats["min_score"] = min(stats["min_score"], s)
+        stats["max_score"] = max(stats["max_score"], s)
+    if scores:
+        stats["mean_score"] = sum(scores.values()) / len(scores)
+
+    # ---- 1. Deactivate low-retention facts ------------------------------
+    to_deactivate = [fid for fid, s in scores.items()
+                     if s < deactivate_threshold]
+    if not dry_run and to_deactivate:
+        store.begin_batch()
+        commit = store.create_commit(
+            f"fade_sweep: deactivate {len(to_deactivate)} low-retention facts",
+            n_facts=0)
+        for fid in to_deactivate:
+            store.update_fact(
+                fid, is_active=0, tx_to=iso(now),
+                retired_commit=commit,
+                provenance={"fade_state": "deactivated",
+                            "fade_score": round(scores[fid], 4),
+                            "faded_at": iso(now)})
+        store.end_batch()
+    stats["deactivated"] = len(to_deactivate)
+
+    # ---- 2. Cluster merge (low-retention siblings) ---------------------
+    # Group remaining active facts by (user, subject, relation) and
+    # find clusters where ALL members have score < merge_threshold.
+    # These are stale repetitions — keep the highest-confidence one,
+    # merge the rest.
+    surviving = [f for f in facts
+                 if scores[f.id] >= deactivate_threshold]
+    clusters: dict[tuple, list[Fact]] = {}
+    for f in surviving:
+        key = (f.user_id, f.subject, f.relation)
+        clusters.setdefault(key, []).append(f)
+
+    merge_count = 0
+    drop_count = 0
+    if not dry_run:
+        store.begin_batch()
+        commit = store.create_commit(
+            f"fade_sweep: cluster merge pass", n_facts=0)
+    for key, group in clusters.items():
+        if len(group) < 2:
+            continue
+        group_scores = [scores[f.id] for f in group]
+        # only merge if ALL members are below merge_threshold
+        if max(group_scores) >= merge_threshold:
+            continue
+        # pick the keeper: highest score, tie-break on confidence
+        group.sort(key=lambda f: (-scores[f.id], -f.confidence))
+        keeper = group[0]
+        for f in group[1:]:
+            if similarity(keeper.value, f.value) >= merge_similarity:
+                if not dry_run:
+                    store.update_fact(
+                        f.id, is_active=0, tx_to=iso(now),
+                        retired_commit=commit,
+                        provenance={"fade_state": "merged",
+                                    "fade_score": round(scores[f.id], 4),
+                                    "merged_into": keeper.id,
+                                    "faded_at": iso(now)})
+                    store.add_edge(keeper.id, f.id, MERGED_WITH,
+                                   {"fade_score": round(scores[f.id], 4),
+                                    "faded_at": iso(now)})
+                drop_count += 1
+        merge_count += 1
+    if not dry_run:
+        store.end_batch()
+    stats["merged_into"] = merge_count
+    stats["merged_dropped"] = drop_count
+
+    return stats
+
+
+def fade_report(store, since: str | None = None) -> dict:
+    """Summary of fade activity since `since` (ISO)."""
+    where = "is_active=0 AND provenance LIKE '%fade_state%'"
+    args: tuple = ()
+    if since:
+        where += " AND tx_to >= ?"
+        args = (since,)
+    rows = store.conn.execute(
+        f"SELECT provenance FROM facts WHERE {where}", args).fetchall()
+    deactivated = 0
+    merged = 0
+    for r in rows:
+        prov = r[0] or ""
+        if '"deactivated"' in prov:
+            deactivated += 1
+        if '"merged"' in prov:
+            merged += 1
+    return {"fade_deactivated": deactivated,
+            "fade_merged": merged,
+            "fade_total": deactivated + merged}
+
+
+__all__ = ["fade_retention_score", "fade_sweep", "fade_report"]

cortexm/trace/lifecycle.py

@@ -0,0 +1,112 @@
+"""Interference-aware lifecycle — promotion, decay, consolidation.
+
+From the August-2026 memory research (Dual-Layer Agentic Memory,
+Controlled Memory Interference, LiveMem): facts are not just stored —
+before commitment each candidate is evaluated for how it *interacts*
+with existing memory; retention is shaped by reinforcement, recency
+and interference rather than a pure Ebbinghaus curve.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+from dataclasses import dataclass
+
+from cortexm.trace.contradictions import Action, Conflict, find_conflicts
+from cortexm.trace.fact import Fact
+from cortexm.trace.store import TraceStore
+from cortexm.util import parse_ts, similarity
+
+
+@dataclass
+class LifecycleDecision:
+    action: Action
+    commit_fact: bool = True
+    target_ids: list[str] | None = None
+    quarantine: bool = False
+    note: str = ""
+    interference: float = 0.0
+
+
+def assess(store: TraceStore, candidate: Fact,
+           value_match_threshold: float = 0.92) -> LifecycleDecision:
+    """Pre-commit interference evaluation (Controlled Memory Interference)."""
+    conflict = find_conflicts(store, candidate)
+    inter = 0.0
+
+    if conflict.action is Action.SUPERSEDE:
+        old = conflict.existing[0]
+        # semantic interference: how much the new fact disrupts the old one
+        inter = similarity(old.value, candidate.value)
+        note = conflict.note
+        if inter > 0.75 and candidate.confidence < 0.6:
+            note += "; low-confidence supersession flagged for audit"
+        return LifecycleDecision(
+            action=Action.SUPERSEDE, commit_fact=True,
+            target_ids=[f.id for f in conflict.existing],
+            note=note, interference=inter)
+
+    if conflict.action is Action.MERGE:
+        return LifecycleDecision(
+            Action.MERGE, commit_fact=False,
+            target_ids=[f.id for f in conflict.existing],
+            note=conflict.note, interference=1.0)
+
+    if conflict.action is Action.SKIP:
+        return LifecycleDecision(
+            Action.SKIP, commit_fact=False,
+            target_ids=[f.id for f in conflict.existing],
+            note=conflict.note)
+
+    if conflict.action is Action.COEXIST:
+        # interference over a bounded recent sample (linear at scale)
+        if candidate.relation == "mentioned":
+            inter = 0.0
+        else:
+            recent = conflict.existing[-8:]
+            inter = max((similarity(f.value, candidate.value) for f in recent),
+                        default=0.0)
+        return LifecycleDecision(
+            Action.COEXIST, commit_fact=True, interference=inter,
+            note=conflict.note)
+
+    return LifecycleDecision(Action.COMMIT, commit_fact=True, note="new fact")
+
+
+def retention_score(fact: Fact, now: _dt.datetime) -> float:
+    """Interference-aware retention: conf x reinforcement x recency decay,
+    reduced by contradiction pressure (number of supersessions the
+    subject-relation chain went through)."""
+    age_days = max(
+        0.0, (now - (parse_ts(fact.tx_from) or now)).total_seconds() / 86400.0)
+    recency = 1.0 / (1.0 + age_days / 30.0) ** 0.5
+    reinforce = 1.0 + 0.5 * (fact.reinforcement - 1) + 0.1 * min(fact.access_count, 20)
+    pressure = 1.0 + 0.25 * fact.provenance.get("chain_updates", 0)
+    return fact.confidence * recency * reinforce / pressure
+
+
+def consolidate(store: TraceStore, now: _dt.datetime | None = None,
+                promote_reinforcement: int = 2,
+                demote_threshold: float = 0.15) -> dict:
+    """Fast-write routing + slow consolidation (Dual-Layer Agentic Memory):
+    promote reinforced short-term facts, decay untouched ones."""
+    now = now or _dt.datetime.now(_dt.timezone.utc)
+    promoted, demoted, deactivated = 0, 0, 0
+    for f in store.query_facts(active=True, derived=None):
+        if f.is_derived:
+            continue
+        score = retention_score(f, now)
+        if f.memory_type == "short_term" and (
+                f.reinforcement >= promote_reinforcement or f.access_count >= 3) \
+                and score >= 0.25:
+            store.update_fact(f.id, memory_type="long_term")
+            promoted += 1
+        elif f.memory_type == "short_term" and score < demote_threshold:
+            store.update_fact(f.id, is_active=0,
+                              provenance={**f.provenance, "deactivated": "decay"})
+            deactivated += 1
+        elif f.memory_type == "long_term" and score < demote_threshold * 0.5:
+            demoted += 1
+            store.update_fact(f.id, memory_type="short_term",
+                              provenance={**f.provenance, "demoted": "decay"})
+    return {"promoted": promoted, "demoted": demoted, "deactivated": deactivated}

cortexm/trace/rebuild.py

@@ -0,0 +1,173 @@
+"""Trace-based rebuild — checksum-driven re-materialization from the
+symbolic Trace.
+
+User concern: "Self-healing is theater → Use checksums + rebuild from
+Trace". The existing palace.heal() re-encodes corrupt records from the
+symbolic Trace, which IS the canonical rebuild path — but it's
+invoked manually and doesn't cover the full lifecycle.
+
+This module formalizes the rebuild op:
+  1. Checksum audit: every vector record carries a stored vec_hash;
+     health_check() detects mismatches.
+  2. Rebuild: for every corrupt record, re-encode from the canonical
+     Fact in TraceStore. The Trace is the source of truth — vectors
+     are derived.
+  3. Full rebuild: drop all vectors, re-build from scratch from all
+     Facts in the Trace. Use after schema changes, codec swaps, or
+     suspected widespread corruption.
+  4. Audit log: every rebuild leaves a tamper-evident record in the
+     rebuild_log table.
+
+Designed for the MemoryPalace + TraceStore pair.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+import json
+from dataclasses import dataclass, asdict
+
+from cortexm.util import iso
+
+
+REBUILD_LOG_TABLE = """
+CREATE TABLE IF NOT EXISTS rebuild_log (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  ts TEXT NOT NULL,
+  kind TEXT NOT NULL,         -- 'partial' | 'full'
+  scope TEXT NOT NULL,        -- 'all' | 'user:N' | 'scope:N'
+  checked INTEGER NOT NULL,
+  rebuilt INTEGER NOT NULL,
+  note TEXT DEFAULT ''
+)
+"""
+
+
+@dataclass
+class RebuildReport:
+    kind: str
+    scope: str
+    checked: int
+    rebuilt: int
+    corrupt_ids: list[str]
+    missing_ids: list[str]
+    duration_ms: int
+    ts: str
+
+    def to_dict(self) -> dict:
+        d = asdict(self)
+        return d
+
+
+class TraceRebuilder:
+    """Drive checksum-audited rebuilds of the MemoryPalace from Trace."""
+
+    def __init__(self, palace, store) -> None:
+        self.palace = palace
+        self.store = store
+        store.conn.execute(REBUILD_LOG_TABLE)
+        store.conn.commit()
+
+    def audit(self, sample: int | None = None) -> dict:
+        """Run a checksum audit — detect but don't fix."""
+        report = self.palace.health_check(sample=sample)
+        return {
+            "checked": report["checked"],
+            "corrupt": report["corrupt"],
+            "corrupt_ids": report["corrupt_ids"],
+            "tmr_disagree_bits": report.get("tmr_disagree_bits", 0),
+        }
+
+    def rebuild_partial(self, max_records: int = 1000) -> RebuildReport:
+        """Re-encode corrupt records from the Trace.
+
+        For every corrupt fact_id, look up the canonical Fact in the
+        TraceStore, re-encode it via palace.encode_fact, and overwrite
+        the corrupt vector record. Leaves a row in rebuild_log.
+        """
+        t0 = _dt.datetime.now(_dt.timezone.utc)
+        audit = self.palace.health_check()
+        corrupt_ids = audit["corrupt_ids"][:max_records]
+        # fetch canonical facts
+        rebuilt = 0
+        missing_ids: list[str] = []
+        for fid in corrupt_ids:
+            fact = self._lookup_fact(fid)
+            if fact is None:
+                missing_ids.append(fid)
+                continue
+            vec = self.palace.encode_fact(fact)
+            self.palace.add(fid, vec)  # add overwrites by fact_id
+            rebuilt += 1
+        t1 = _dt.datetime.now(_dt.timezone.utc)
+        dur_ms = int((t1 - t0).total_seconds() * 1000)
+        report = RebuildReport(
+            kind="partial", scope="all",
+            checked=audit["checked"], rebuilt=rebuilt,
+            corrupt_ids=corrupt_ids, missing_ids=missing_ids,
+            duration_ms=dur_ms, ts=iso(t1))
+        self._log(report)
+        return report
+
+    def rebuild_full(self, user_id: str | None = None) -> RebuildReport:
+        """Drop all vectors and re-build from scratch from Trace.
+
+        Use after schema changes, codec swaps, or suspected widespread
+        corruption. Slow but authoritative.
+        """
+        t0 = _dt.datetime.now(_dt.timezone.utc)
+        # fetch all facts
+        scope = f"user:{user_id}" if user_id else "all"
+        facts = self.store.query_facts(active=True, user_id=user_id)
+        # drop all vectors
+        self.palace.remove_ids([f.id for f in facts])  # clears them
+        rebuilt = 0
+        for fact in facts:
+            try:
+                vec = self.palace.encode_fact(fact)
+                self.palace.add(fact.id, vec)
+                rebuilt += 1
+            except Exception:
+                continue
+        t1 = _dt.datetime.now(_dt.timezone.utc)
+        dur_ms = int((t1 - t0).total_seconds() * 1000)
+        report = RebuildReport(
+            kind="full", scope=scope,
+            checked=len(facts), rebuilt=rebuilt,
+            corrupt_ids=[], missing_ids=[],
+            duration_ms=dur_ms, ts=iso(t1))
+        self._log(report)
+        return report
+
+    def rebuild_log(self, limit: int = 50) -> list[dict]:
+        """Return recent rebuild log entries (tamper-evident audit)."""
+        rows = self.store.conn.execute(
+            "SELECT * FROM rebuild_log ORDER BY id DESC LIMIT ?", (limit,)
+        ).fetchall()
+        return [dict(r) for r in rows]
+
+    # --- internals ---
+    def _lookup_fact(self, fact_id: str):
+        """Fetch a fact by id from the TraceStore."""
+        try:
+            facts = self.store.query_facts(active=True)
+            for f in facts:
+                if f.id == fact_id:
+                    return f
+        except Exception:
+            pass
+        return None
+
+    def _log(self, report: RebuildReport) -> None:
+        self.store.conn.execute(
+            """INSERT INTO rebuild_log(ts, kind, scope, checked, rebuilt, note)
+               VALUES(?,?,?,?,?,?)""",
+            (report.ts, report.kind, report.scope,
+             report.checked, report.rebuilt,
+             json.dumps({"corrupt_ids": report.corrupt_ids[:10],
+                         "missing_ids": report.missing_ids[:10]}))
+        )
+        self.store.conn.commit()
+
+
+__all__ = ["TraceRebuilder", "RebuildReport", "REBUILD_LOG_TABLE"]