cortexm 0.3.0__py3-none-any.whl

cortexm/trace/contradictions.py

@@ -0,0 +1,69 @@
+"""Deterministic contradiction detection & truth maintenance.
+
+Phase rules from Section 1.1:
+  1. Classification      — relation → semantic category, single/multi valued
+  2. Contradiction       — exact + fuzzy (Jaccard/Levenshtein) match on
+                           subject-relation pairs; latest-value-wins for
+                           single-valued relations, append for multi-valued
+  3. Interference-aware lifecycle — see lifecycle.py
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+from cortexm.trace.fact import Fact, SINGLE_VALUED
+from cortexm.trace.store import TraceStore
+from cortexm.util import similarity
+
+
+class Action(str, Enum):
+    COMMIT = "commit"          # brand-new fact
+    MERGE = "merge"            # near-duplicate: reinforce existing
+    SUPERSEDE = "supersede"    # contradiction on single-valued relation
+    COEXIST = "coexist"        # contradiction on multi-valued relation
+    SKIP = "skip"              # exact duplicate
+
+
+@dataclass
+class Conflict:
+    action: Action
+    existing: list[Fact]
+    note: str = ""
+
+
+def find_conflicts(store: TraceStore, candidate: Fact) -> Conflict:
+    """Decide how ``candidate`` interacts with active memory."""
+    existing = [
+        f for f in store.query_facts(
+            subject=candidate.subject, relation=candidate.relation,
+            user_id=candidate.user_id, active=True)
+        if f.id != candidate.id
+    ]
+    if not existing:
+        return Conflict(Action.COMMIT, [])
+
+    exact = [f for f in existing if f.value.strip().lower() == candidate.value.strip().lower()]
+    if exact:
+        return Conflict(Action.SKIP, exact, "exact duplicate")
+
+    # low-salience mention anchors: exact-dup semantics only (no fuzzy
+    # quadratic scans — mention streams are high-volume, low-signal)
+    if candidate.relation in ("mentioned", "event", "instruction"):
+        if candidate.relation == "mentioned":
+            return Conflict(Action.COEXIST, existing, "mention anchor recorded")
+
+    near = [f for f in existing
+            if similarity(f.value, candidate.value) >= 0.92]
+    if near:
+        return Conflict(Action.MERGE, near, "near-duplicate merged; reinforcement +1")
+
+    single = candidate.relation in SINGLE_VALUED
+    if single:
+        # newest valid_from wins reality; old fact gets valid_to
+        target = max(existing, key=lambda f: (f.valid_from, f.tx_from))
+        return Conflict(Action.SUPERSEDE, [target],
+                        f"contradiction on single-valued '{candidate.relation}'")
+    return Conflict(Action.COEXIST, existing,
+                    f"conflicting values on multi-valued '{candidate.relation}' coexist")

cortexm/trace/dedup.py

@@ -0,0 +1,114 @@
+"""Deduplication + holographic compression audit.
+
+User concern (Con #4): "Storage Bloat — wrong. Deduplication and
+holographic compression solve this."
+
+This module audits the MemoryPalace and TraceStore for:
+  1. **Dedup ratio**: how many input tokens map to how many derived
+     facts. The README claims 10M tokens → ~590 facts. This formalizes
+     that audit.
+  2. **Holographic compression ratio**: dense vector bytes vs stored
+     bytes per codec. PQ codec achieves 96x at 768 dims (8 B vs 768 B);
+     binary codec achieves 8x (96 B vs 768 B with TMR).
+  3. **Effective memory budget per million memories**: the metric that
+     matters for edge deployment — "fits on a Raspberry Pi 5".
+
+Pure Python + numpy. Runs as part of palace.storage_stats() and exposed
+via the MCP server's `contextm_stats` tool.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, asdict
+
+
+@dataclass
+class CompressionReport:
+    """Audit of dedup + holographic compression."""
+    raw_token_count: int
+    unique_fact_count: int
+    dedup_ratio: float           # raw / unique
+    codec: str
+    dims: int
+    bytes_per_vector: int
+    dense_bytes_per_vector: int   # FP32 dense = dims * 4
+    compression_ratio: float      # dense / stored
+    total_stored_bytes: int
+    per_million_mb: float         # MB to store 1M memories
+    fits_raspberry_pi_5: bool     # 8GB RAM target
+    notes: list[str]
+
+
+class DedupAuditor:
+    """Audit dedup + compression for a MemoryPalace."""
+
+    DENSE_BYTES_PER_DIM = 4  # FP32
+
+    def __init__(self, palace, store) -> None:
+        self.palace = palace
+        self.store = store
+
+    def audit(self, raw_token_count: int | None = None) -> CompressionReport:
+        """Compute dedup ratio + compression ratio for the palace."""
+        facts = self.store.query_facts(active=True)
+        unique_facts = len(facts)
+        # raw token count: heuristic from stored chunks (rows in trace text table)
+        if raw_token_count is None:
+            raw_token_count = self._estimate_raw_tokens()
+        dedup_ratio = (raw_token_count / unique_facts
+                       if unique_facts > 0 else 0.0)
+        # compression ratio
+        codec_name = self.palace.codec.name
+        dims = self.palace.dims
+        bytes_per_vec = self.palace.codec.bytes_per_vector
+        dense_bytes = dims * self.DENSE_BYTES_PER_DIM
+        comp_ratio = dense_bytes / bytes_per_vec if bytes_per_vec > 0 else 0.0
+        total_stored = unique_facts * bytes_per_vec
+        per_million_mb = (bytes_per_vec * 1_000_000) / (1024 * 1024)
+        fits_pi = per_million_mb < 1024  # 1GB threshold
+        notes = []
+        if codec_name == "pq":
+            notes.append("PQ achieves ~96x compression at 768 dims (8 B/v)")
+            notes.append("Codebook adds ~16KB overhead (amortized at >2k facts)")
+        elif codec_name == "binary":
+            notes.append("Binary + TMR: 96 B/v at 768 dims, 8x compression")
+            notes.append("TMR adds 3x storage but enables self-healing")
+        elif codec_name == "rabitq":
+            notes.append("RaBitQ: 96 B/v at 768 dims, JL rotation + binarization")
+        elif codec_name == "int8":
+            notes.append("INT8: 770 B/v at 768 dims (baseline, 4x compression)")
+        notes.append(f"Dedup ratio {dedup_ratio:.1f}x "
+                     f"({raw_token_count} tokens → {unique_facts} facts)")
+        if dedup_ratio < 5:
+            notes.append("⚠ Dedup ratio <5x — check for redundant patterns")
+        return CompressionReport(
+            raw_token_count=raw_token_count,
+            unique_fact_count=unique_facts,
+            dedup_ratio=dedup_ratio,
+            codec=codec_name,
+            dims=dims,
+            bytes_per_vector=bytes_per_vec,
+            dense_bytes_per_vector=dense_bytes,
+            compression_ratio=comp_ratio,
+            total_stored_bytes=total_stored,
+            per_million_mb=per_million_mb,
+            fits_raspberry_pi_5=fits_pi,
+            notes=notes,
+        )
+
+    def _estimate_raw_tokens(self) -> int:
+        """Estimate raw token count from stored chunk text rows."""
+        try:
+            # trace store has raw_text rows; count chars / 4
+            row = self.store.conn.execute(
+                "SELECT COUNT(*) as n, SUM(LENGTH(text)) as chars FROM raw_text"
+            ).fetchone()
+            if row and row["chars"]:
+                return int(row["chars"] / 4)  # ~4 chars/token heuristic
+        except Exception:
+            pass
+        # fallback: estimate from fact count * 16 (heuristic)
+        return self.palace.size() * 16
+
+
+__all__ = ["DedupAuditor", "CompressionReport"]

cortexm/trace/edges.py

@@ -0,0 +1,214 @@
+"""Typed edge vocabulary for the Trace.
+
+arXiv:2601.15311 (Aeon) formalizes an episodic Trace with typed edges:
+CAUSAL, NEXT, REFERS_TO. Context-M already had EXTRACTED_FROM /
+CONTRADICTS / TEMPORALLY_PRECEDED_BY — this module adds the missing
+types and centralizes the vocabulary so:
+
+- write-side code (writer.py / query_extract.py / consolidate.py) has
+  one place to discover the canonical edge kinds
+- read-side code (reader.py / ppr.py) can enumerate them by purpose
+  ("causal", "temporal", "provenance", "semantic_ref") without hard-
+  coding strings scattered across the codebase
+- the dreaming / consolidation pass can walk the typed graph
+  intentionally (e.g. merge facts joined by NEXT, summarize chains
+  linked by CAUSAL)
+
+Edge kind -> direction convention:
+    src -> dst means "src is the cause / antecedent / referring node"
+    e.g. CAUSAL(fact_A, fact_B) reads "fact_A caused fact_B" (A is
+    the cause, B is the effect). This matches Aeon's convention.
+
+The kind column on `edges` is TEXT — adding new kinds requires no
+schema migration, only a helper here that knows how to wire them.
+"""
+from __future__ import annotations
+
+from typing import Final
+
+
+# ---------- Canonical edge vocabulary (centralized) --------------------
+
+# provenance: where a fact came from
+EXTRACTED_FROM: Final[str] = "EXTRACTED_FROM"   # fact -> chunk
+MENTIONS: Final[str] = "MENTIONS"                 # fact -> entity mention
+
+# temporal / causal ordering
+TEMPORALLY_PRECEDED_BY: Final[str] = "TEMPORALLY_PRECEDED_BY"  # fact -> fact (next is newer)
+CAUSAL: Final[str] = "CAUSAL"                      # fact -> fact (cause -> effect)
+NEXT: Final[str] = "NEXT"                          # fact -> fact (narrative next, no causal claim)
+
+# truth maintenance
+CONTRADICTS: Final[str] = "CONTRADICTS"            # fact -> fact (new contradicts old)
+SUPERSEDES: Final[str] = "SUPERSEDES"              # fact -> fact (new replaces old)
+RETRACTED_BY: Final[str] = "RETRACTED_BY"          # fact -> fact (retired by retraction)
+MERGED_WITH: Final[str] = "MERGED_WITH"            # fact -> fact (merged into target)
+
+# semantic cross-references (Aeon's REFERS_TO)
+REFERS_TO: Final[str] = "REFERS_TO"                # fact -> fact/concept (episodic → atlas)
+SAME_PERSON: Final[str] = "SAME_PERSON"            # alias fact -> canonical fact (e.g. "Priya" → "Priya Johnson")
+
+# HMS cognition engine edges — produced by cortexm.cognition.*.
+# HYPOTHESIZED_BY: hypothesis fact -> supporting fact(s) (reasoning chain)
+# PROMOTED_FROM:   user-confirmed fact -> hypothesis origin (truth
+#                   maintenance — when a hypothesis is verified by
+#                   user input, the new fact links back to its origin)
+# ABSTRACTS:       abstraction prototype -> member (categorization)
+# INSTANTIATES:    member -> abstraction prototype (inverse)
+# ANALOGOUS_TO:    relation_a -> relation_b (structural isomorphism)
+HYPOTHESIZED_BY: Final[str] = "HYPOTHESIZED_BY"
+PROMOTED_FROM: Final[str] = "PROMOTED_FROM"
+ABSTRACTS: Final[str] = "ABSTRACTS"
+INSTANTIATES: Final[str] = "INSTANTIATES"
+ANALOGOUS_TO: Final[str] = "ANALOGOUS_TO"
+
+# All kinds in one place for enumeration / display
+ALL_KINDS: Final[tuple[str, ...]] = (
+    EXTRACTED_FROM, MENTIONS,
+    TEMPORALLY_PRECEDED_BY, CAUSAL, NEXT,
+    CONTRADICTS, SUPERSEDES, RETRACTED_BY, MERGED_WITH,
+    REFERS_TO, SAME_PERSON,
+    HYPOTHESIZED_BY, PROMOTED_FROM,
+    ABSTRACTS, INSTANTIATES, ANALOGOUS_TO,
+)
+
+# Group by purpose — used by the reader / PPR / consolidator
+PROVENANCE_EDGES: Final[tuple[str, ...]] = (EXTRACTED_FROM, MENTIONS)
+TEMPORAL_EDGES: Final[tuple[str, ...]] = (TEMPORALLY_PRECEDED_BY, NEXT)
+CAUSAL_EDGES: Final[tuple[str, ...]] = (CAUSAL,)
+TRUTH_MAINTENANCE_EDGES: Final[tuple[str, ...]] = (
+    CONTRADICTS, SUPERSEDES, RETRACTED_BY, MERGED_WITH,
+    PROMOTED_FROM)
+SEMANTIC_REF_EDGES: Final[tuple[str, ...]] = (
+    REFERS_TO, SAME_PERSON,
+    ABSTRACTS, INSTANTIATES, ANALOGOUS_TO)
+COGNITION_EDGES: Final[tuple[str, ...]] = (
+    HYPOTHESIZED_BY, PROMOTED_FROM,
+    ABSTRACTS, INSTANTIATES, ANALOGOUS_TO)
+
+
+# ---------- Helpers -----------------------------------------------------
+
+def is_causal(kind: str) -> bool:
+    """Does this edge type assert a causal / temporal antecedent?"""
+    return kind in (CAUSAL, TEMPORALLY_PRECEDED_BY, NEXT)
+
+
+def is_truth_maintenance(kind: str) -> bool:
+    """Does this edge type mark a fact as no-longer-current?"""
+    return kind in (CONTRADICTS, SUPERSEDES, RETRACTED_BY, MERGED_WITH)
+
+
+def is_semantic_ref(kind: str) -> bool:
+    """Does this edge type link two facts semantically (vs. structurally)?"""
+    return kind in (REFERS_TO, SAME_PERSON)
+
+
+def direction_convention(kind: str) -> str:
+    """Return 'cause_to_effect' or 'new_to_old' or 'ref_to_target'.
+
+    Used by reader/PPR to know which way to walk the edge for a given
+    query intent (e.g. "why" queries walk CAUSAL src->dst; "what
+    replaced this" queries walk SUPERSEDES dst->src).
+    """
+    if kind in (CAUSAL, TEMPORALLY_PRECEDED_BY, NEXT):
+        return "cause_to_effect"  # src causes/precedes dst
+    if kind in (CONTRADICTS, SUPERSEDES, RETRACTED_BY, MERGED_WITH):
+        return "new_to_old"  # src is the newer fact, dst is the older
+    if kind in (REFERS_TO, SAME_PERSON, MENTIONS):
+        return "ref_to_target"
+    return "out"
+
+
+# ---------- CAUSAL / REFERS_TO writers --------------------------------
+
+def wire_causal_edge(store, cause_fact_id: str, effect_fact_id: str,
+                    reason: str = "") -> None:
+    """Wire a CAUSAL edge from cause to effect.
+
+    Used by the writer when a SUPERSEDE / retraction pattern fires —
+    e.g. "I left Google in March" (cause) caused the older works_at
+    fact to be retired (effect). Both edges are kept:
+    SUPERSEDES marks the truth-maintenance side, CAUSAL marks the
+    narrative cause. Reader can answer "why did X happen?" via CAUSAL
+    traversal; it can answer "what's the current truth about X?" via
+    SUPERSEDES traversal.
+    """
+    if cause_fact_id == effect_fact_id:
+        return
+    store.add_edge(cause_fact_id, effect_fact_id, CAUSAL,
+                   {"reason": reason} if reason else None)
+
+
+def wire_refers_to(store, ref_fact_id: str, target_id: str,
+                   target_kind: str = "fact") -> None:
+    """Wire a REFERS_TO edge from a referring fact to its target.
+
+    Used by the writer / consolidator to link an episodic fact back to
+    the atlas concept it refers to. For Context-M the "atlas" is the
+    palace (the VSA vector space) — so REFERS_TO currently points at
+    other facts (semantic cross-references) and may also point at
+    chunk_ids (episodic → raw source) when called from the
+    consolidator's branch-compression pass.
+    """
+    if ref_fact_id == target_id:
+        return
+    store.add_edge(ref_fact_id, target_id, REFERS_TO,
+                   {"target_kind": target_kind})
+
+
+def find_causal_chain(store, fact_id: str,
+                      direction: str = "ancestors",
+                      max_depth: int = 8) -> list[str]:
+    """Walk CAUSAL edges from fact_id.
+
+    direction='ancestors': walk src->dst backwards — return the chain
+        of causes that led to this fact (effect).
+    direction='descendants': walk src->dst forwards — return the chain
+        of effects this fact caused.
+    """
+    seen: set[str] = {fact_id}
+    out: list[str] = []
+    frontier = [fact_id]
+    for _ in range(max_depth):
+        next_frontier: list[str] = []
+        for fid in frontier:
+            if direction == "ancestors":
+                # find edges where dst=fid (this fact is the effect)
+                rows = store.conn.execute(
+                    "SELECT src FROM edges WHERE dst=? AND kind=?",
+                    (fid, CAUSAL)).fetchall()
+            else:
+                # find edges where src=fid (this fact is the cause)
+                rows = store.conn.execute(
+                    "SELECT dst FROM edges WHERE src=? AND kind=?",
+                    (fid, CAUSAL)).fetchall()
+            for r in rows:
+                other = r[0]
+                if other in seen:
+                    continue
+                seen.add(other)
+                out.append(other)
+                next_frontier.append(other)
+        if not next_frontier:
+            break
+        frontier = next_frontier
+    return out
+
+
+__all__ = [
+    # kinds
+    "EXTRACTED_FROM", "MENTIONS", "TEMPORALLY_PRECEDED_BY",
+    "CAUSAL", "NEXT", "CONTRADICTS", "SUPERSEDES",
+    "RETRACTED_BY", "MERGED_WITH", "REFERS_TO", "SAME_PERSON",
+    "HYPOTHESIZED_BY", "PROMOTED_FROM",
+    "ABSTRACTS", "INSTANTIATES", "ANALOGOUS_TO",
+    "ALL_KINDS", "PROVENANCE_EDGES", "TEMPORAL_EDGES",
+    "CAUSAL_EDGES", "TRUTH_MAINTENANCE_EDGES", "SEMANTIC_REF_EDGES",
+    "COGNITION_EDGES",
+    # helpers
+    "is_causal", "is_truth_maintenance", "is_semantic_ref",
+    "direction_convention",
+    # writers
+    "wire_causal_edge", "wire_refers_to", "find_causal_chain",
+]

cortexm/trace/fact.py

@@ -0,0 +1,121 @@
+"""Fact model & relation taxonomy for the Symbolic Trace.
+
+Mirrors the Section 1.1 schema: Subject-Relation-Value triples with
+bi-temporal timestamps (valid time + transaction time), confidence,
+BLAKE3 source hash, scope (user/agent/flow), memory type, access count
+and active flag — plus CONTRADICTS / TEMPORALLY_PRECEDED_BY /
+EXTRACTED_FROM edges materialized in the store.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field, asdict
+from datetime import datetime
+
+from cortexm.util import parse_ts, iso
+
+# Relations where reality has exactly one value at a time: a new value
+# SUPERSEDES the old one (contradiction resolution by truth maintenance).
+SINGLE_VALUED = {
+    "name", "works_at", "role", "lives_in", "birthday", "age",
+    "reports_to", "member_of", "studied_at", "team",
+}
+
+# Relations that accumulate: duplicates are merged, conflicts coexist.
+MULTI_VALUED = {
+    "likes", "dislikes", "prefers", "has_skill", "works_on", "completed", "alias",
+    "sibling", "spouse", "parent", "child", "friend", "uses", "manages",
+    "studied", "owns", "goal", "event", "mentioned", "joined", "left",
+    "moved_to", "allergy", "has_pet", "speaks", "hobby",
+}
+
+RELATION_CATEGORIES = {
+    "personal": {"name", "birthday", "age", "sibling", "spouse", "parent",
+                 "child", "friend", "alias", "lives_in", "speaks", "has_pet"},
+    "work": {"works_at", "role", "reports_to", "member_of", "manages",
+             "team", "joined", "left", "works_on", "completed"},
+    "preference": {"likes", "dislikes", "prefers"},
+    "skill": {"has_skill", "studied", "studied_at"},
+    "task": {"event", "goal", "mentioned", "uses", "owns"},
+    "temporal": {"moved_to"},
+}
+
+
+@dataclass
+class Fact:
+    id: str
+    subject: str
+    relation: str
+    value: str
+    valid_from: str                 # when it became true in reality
+    valid_to: str | None = None     # when it stopped being true (None = now)
+    tx_from: str = ""               # when we recorded it
+    tx_to: str | None = None
+    confidence: float = 0.8
+    source_hash: str = ""
+    source_id: str = ""
+    user_id: str = "default"
+    agent_id: str | None = None
+    run_id: str | None = None
+    memory_type: str = "short_term"
+    access_count: int = 0
+    reinforcement: int = 1
+    is_active: bool = True
+    is_derived: bool = False
+    quarantined: bool = False
+    birth_commit: str | None = None
+    retired_commit: str | None = None
+    provenance: dict = field(default_factory=dict)
+
+    # ------------------------------------------------------------------
+    def text(self) -> str:
+        return f"{self.subject} | {self.relation} | {self.value}"
+
+    def display(self) -> str:
+        return f"({self.subject}, {self.relation}, {self.value})"
+
+    def valid_window(self) -> str:
+        return f"{self.valid_from}→{self.valid_to or '∞'}"
+
+    def scope_dict(self) -> dict:
+        return {"user_id": self.user_id, "agent_id": self.agent_id, "run_id": self.run_id}
+
+    def to_row(self) -> dict:
+        d = asdict(self)
+        d["is_active"] = int(self.is_active)
+        d["is_derived"] = int(self.is_derived)
+        d["quarantined"] = int(self.quarantined)
+        d["provenance"] = json.dumps(self.provenance, default=str)
+        return d
+
+    @staticmethod
+    def from_row(row: dict) -> "Fact":
+        row = dict(row)
+        row["is_active"] = bool(row["is_active"])
+        row["is_derived"] = bool(row["is_derived"])
+        row["quarantined"] = bool(row.get("quarantined", 0))
+        row["provenance"] = json.loads(row.get("provenance") or "{}")
+        return Fact(**row)
+
+    def matches_scope(self, user_id: str | None, agent_id: str | None = None,
+                      run_id: str | None = None) -> bool:
+        if user_id is not None and self.user_id != user_id:
+            return False
+        if agent_id is not None and self.agent_id != agent_id:
+            return False
+        if run_id is not None and self.run_id != run_id:
+            return False
+        return True
+
+
+def make_fact(subject: str, relation: str, value: str, *,
+              now: datetime, valid_from: datetime | str | None = None,
+              valid_to: datetime | str | None = None, **kwargs) -> Fact:
+    """Convenience constructor normalizing timestamps to ISO strings."""
+    vf = iso(parse_ts(valid_from) or now)[:10] if valid_from else iso(now)[:10]
+    vt = iso(parse_ts(valid_to))[:10] if valid_to else None
+    return Fact(
+        id=kwargs.pop("id", "") or __import__("uuid").uuid4().hex,
+        subject=subject, relation=relation, value=value,
+        valid_from=vf, valid_to=vt, tx_from=iso(now), **kwargs)