crprotocol 2.0.0__py3-none-any.whl

crp/state/event_log.py

@@ -0,0 +1,313 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Event sourcing — append-only immutable fact lifecycle log (§3.2).
+
+Every fact mutation (create, supersede, compact, archive, restore) is recorded
+as a FactEvent.  The log supports temporal queries: state_at_window,
+facts_between, supersession_chain.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from typing import Any
+
+from crp.extraction.types import Fact, FactEdge
+
+from .fact import StateFact
+
+# ---------------------------------------------------------------------------
+# Event types
+# ---------------------------------------------------------------------------
+
+EVENT_CREATED = "created"
+EVENT_SUPERSEDED = "superseded"
+EVENT_COMPACTED = "compacted"
+EVENT_ARCHIVED = "archived"
+EVENT_RESTORED = "restored"
+EVENT_EDGE_ADDED = "edge_added"
+
+ALL_EVENT_TYPES = frozenset(
+    {EVENT_CREATED, EVENT_SUPERSEDED, EVENT_COMPACTED, EVENT_ARCHIVED, EVENT_RESTORED, EVENT_EDGE_ADDED}
+)
+
+# ---------------------------------------------------------------------------
+# FactEvent (from extraction/types.py, but we re-define here with monotonic ID)
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class FactEvent:
+    """Immutable audit-log entry for a fact lifecycle event."""
+
+    event_id: int = 0  # monotonically increasing
+    timestamp: float = field(default_factory=time.time)
+    window_id: str = ""
+    event_type: str = ""
+    fact_id: str = ""
+    payload: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "event_id": self.event_id,
+            "timestamp": self.timestamp,
+            "window_id": self.window_id,
+            "event_type": self.event_type,
+            "fact_id": self.fact_id,
+            "payload": self.payload,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> FactEvent:
+        return cls(
+            event_id=data.get("event_id", 0),
+            timestamp=data.get("timestamp", 0.0),
+            window_id=data.get("window_id", ""),
+            event_type=data.get("event_type", ""),
+            fact_id=data.get("fact_id", ""),
+            payload=data.get("payload", {}),
+        )
+
+
+# ---------------------------------------------------------------------------
+# FactEventLog — append-only, thread-safe
+# ---------------------------------------------------------------------------
+
+
+class FactEventLog:
+    """Append-only immutable event log for fact lifecycle (§3.2).
+
+    Supports:
+    - append(): add new event with monotonic ID
+    - state_at_window(): replay events to reconstruct state at a point
+    - facts_between(): range query
+    - supersession_chain(): full lifecycle trail for a fact
+    - Temporal queries: events_since, events_for_fact
+    """
+
+    def __init__(self) -> None:
+        self._events: list[FactEvent] = []
+        self._next_id: int = 1
+        self._lock = threading.Lock()
+
+        # Indexes for fast lookup
+        self._by_fact: dict[str, list[int]] = {}  # fact_id → [event indices]
+        self._by_window: dict[str, list[int]] = {}  # window_id → [event indices]
+
+    @property
+    def size(self) -> int:
+        return len(self._events)
+
+    # --- Append ---------------------------------------------------------------
+
+    def append(
+        self,
+        event_type: str,
+        fact_id: str,
+        window_id: str,
+        payload: dict[str, Any] | None = None,
+    ) -> FactEvent:
+        """Record an immutable event.  Returns the created event."""
+        with self._lock:
+            event = FactEvent(
+                event_id=self._next_id,
+                timestamp=time.time(),
+                window_id=window_id,
+                event_type=event_type,
+                fact_id=fact_id,
+                payload=payload or {},
+            )
+            idx = len(self._events)
+            self._events.append(event)
+            self._next_id += 1
+
+            # Update indexes
+            self._by_fact.setdefault(fact_id, []).append(idx)
+            self._by_window.setdefault(window_id, []).append(idx)
+
+        return event
+
+    def record_fact_created(self, fact: Fact | StateFact, window_id: str) -> FactEvent:
+        """Convenience: record a fact creation event."""
+        fid = fact.id if isinstance(fact, StateFact) else fact.id
+        text = fact.text if isinstance(fact, StateFact) else fact.text
+        return self.append(
+            EVENT_CREATED,
+            fid,
+            window_id,
+            {"text": text, "confidence": fact.confidence},
+        )
+
+    def record_supersession(
+        self, old_fact_id: str, new_fact_id: str, window_id: str, confidence: float = 1.0
+    ) -> FactEvent:
+        return self.append(
+            EVENT_SUPERSEDED,
+            old_fact_id,
+            window_id,
+            {"superseded_by": new_fact_id, "confidence": confidence},
+        )
+
+    def record_compaction(self, fact_id: str, window_id: str, summary_id: str = "") -> FactEvent:
+        return self.append(EVENT_COMPACTED, fact_id, window_id, {"summary_id": summary_id})
+
+    def record_archived(self, fact_id: str, window_id: str) -> FactEvent:
+        return self.append(EVENT_ARCHIVED, fact_id, window_id)
+
+    def record_restored(self, fact_id: str, window_id: str) -> FactEvent:
+        return self.append(EVENT_RESTORED, fact_id, window_id)
+
+    def record_edge_added(self, edge: FactEdge, window_id: str) -> FactEvent:
+        return self.append(
+            EVENT_EDGE_ADDED,
+            edge.source_id,
+            window_id,
+            {
+                "edge_id": edge.id,
+                "target_id": edge.target_id,
+                "relation_type": edge.relation_type.value if hasattr(edge.relation_type, "value") else str(edge.relation_type),
+                "confidence": edge.confidence,
+            },
+        )
+
+    # --- Temporal queries (§3.2) ----------------------------------------------
+
+    def state_at_window(self, window_id: str) -> set[str]:
+        """Replay events up to *window_id* and return the set of active fact IDs.
+
+        An active fact is one that was created and not yet superseded/compacted/archived.
+        """
+        active: set[str] = set()
+        with self._lock:
+            target_indices = self._by_window.get(window_id)
+            if target_indices is None:
+                return active
+            max_idx = max(target_indices)
+
+            for event in self._events[: max_idx + 1]:
+                if event.event_type == EVENT_CREATED:
+                    active.add(event.fact_id)
+                elif event.event_type in (EVENT_SUPERSEDED, EVENT_COMPACTED, EVENT_ARCHIVED):
+                    active.discard(event.fact_id)
+                elif event.event_type == EVENT_RESTORED:
+                    active.add(event.fact_id)
+        return active
+
+    def facts_between(self, start_window: str, end_window: str) -> list[FactEvent]:
+        """Return events between two windows (inclusive)."""
+        with self._lock:
+            start_indices = self._by_window.get(start_window)
+            end_indices = self._by_window.get(end_window)
+            if start_indices is None or end_indices is None:
+                return []
+            min_idx = min(start_indices)
+            max_idx = max(end_indices)
+            return list(self._events[min_idx: max_idx + 1])
+
+    def supersession_chain(self, fact_id: str) -> list[FactEvent]:
+        """Return the full supersession lifecycle trail for *fact_id*."""
+        chain: list[FactEvent] = []
+        current_id = fact_id
+
+        with self._lock:
+            visited: set[str] = set()
+            while current_id and current_id not in visited:
+                visited.add(current_id)
+                indices = self._by_fact.get(current_id, [])
+                for idx in indices:
+                    event = self._events[idx]
+                    chain.append(event)
+                    if event.event_type == EVENT_SUPERSEDED:
+                        next_id = event.payload.get("superseded_by", "")
+                        if next_id:
+                            current_id = next_id
+                            break
+                else:
+                    break  # no supersession event found
+        return chain
+
+    def events_since(self, timestamp: float) -> list[FactEvent]:
+        """Return all events after *timestamp*."""
+        with self._lock:
+            return [e for e in self._events if e.timestamp >= timestamp]
+
+    def events_for_fact(self, fact_id: str) -> list[FactEvent]:
+        """Return all events for a specific fact."""
+        with self._lock:
+            indices = self._by_fact.get(fact_id, [])
+            return [self._events[i] for i in indices]
+
+    def events_by_type(self, event_type: str) -> list[FactEvent]:
+        """Filter events by type."""
+        with self._lock:
+            return [e for e in self._events if e.event_type == event_type]
+
+    def events_in_window(self, window_id: str) -> list[FactEvent]:
+        """Return all events for a specific window."""
+        with self._lock:
+            indices = self._by_window.get(window_id, [])
+            return [self._events[i] for i in indices]
+
+    # --- Iteration / bulk access ----------------------------------------------
+
+    def all_events(self) -> list[FactEvent]:
+        """Return a copy of all events."""
+        with self._lock:
+            return list(self._events)
+
+    def __iter__(self) -> Iterator[FactEvent]:
+        return iter(self._events)
+
+    def __len__(self) -> int:
+        return len(self._events)
+
+    # --- Serialization --------------------------------------------------------
+
+    def to_list(self) -> list[dict[str, Any]]:
+        """Serialize all events for persistence."""
+        with self._lock:
+            return [e.to_dict() for e in self._events]
+
+    def load_from_list(self, events: list[dict[str, Any]]) -> None:
+        """Restore from serialized event list."""
+        with self._lock:
+            self._events.clear()
+            self._by_fact.clear()
+            self._by_window.clear()
+            self._next_id = 1
+            for edata in events:
+                event = FactEvent.from_dict(edata)
+                idx = len(self._events)
+                self._events.append(event)
+                self._by_fact.setdefault(event.fact_id, []).append(idx)
+                self._by_window.setdefault(event.window_id, []).append(idx)
+                if event.event_id >= self._next_id:
+                    self._next_id = event.event_id + 1
+
+    # --- Truncation (for snapshot support) ------------------------------------
+
+    def truncate_before(self, event_id: int) -> list[FactEvent]:
+        """Remove events before *event_id*.  Returns removed events for archival."""
+        with self._lock:
+            cut_idx = 0
+            for i, e in enumerate(self._events):
+                if e.event_id >= event_id:
+                    cut_idx = i
+                    break
+            else:
+                cut_idx = len(self._events)
+
+            removed = self._events[:cut_idx]
+            self._events = self._events[cut_idx:]
+
+            # Rebuild indexes
+            self._by_fact.clear()
+            self._by_window.clear()
+            for i, e in enumerate(self._events):
+                self._by_fact.setdefault(e.fact_id, []).append(i)
+                self._by_window.setdefault(e.window_id, []).append(i)
+
+        return removed

crp/state/fact.py

@@ -0,0 +1,189 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""State-layer Fact model — extends extraction Fact with lazy embedding,
+age tracking, and seen_count for the 4-tier memory hierarchy (§3.1).
+
+StateFact wraps extraction.types.Fact with additional state-management fields
+that the warm store, envelope builder, and compaction engine need.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+from crp.extraction.types import Fact
+
+logger = logging.getLogger("crp.state.fact")
+
+# ---------------------------------------------------------------------------
+# Lazy embedding support
+# ---------------------------------------------------------------------------
+
+_EMBED_FN: Any = None  # set by scoring / warm store init
+
+
+def set_embedding_function(fn: Any) -> None:
+    """Register the global embedding function for lazy compute."""
+    global _EMBED_FN  # noqa: PLW0603
+    _EMBED_FN = fn
+
+
+# ---------------------------------------------------------------------------
+# StateFact
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class StateFact:
+    """Fact extended with state-management metadata (§3.1).
+
+    Wraps an extraction ``Fact`` and adds:
+    - Lazy embedding (computed on first access, cached)
+    - ``age_in_windows`` — updated each window by the warm store
+    - ``seen_count`` — how many envelopes this fact appeared in
+    - ``consumed_by_windows`` — which windows used this fact
+    - ``graph_edges`` — IDs of connected FactEdges
+    """
+
+    fact: Fact
+    _embedding: list[float] | None = field(default=None, repr=False)
+    age_in_windows: int = 0
+    seen_count: int = 0
+    consumed_by_windows: list[str] = field(default_factory=list)
+    graph_edges: list[str] = field(default_factory=list)
+    tier: int = 2  # 0=critical, 1=hot, 2=warm, 3=cold
+
+    # --- Delegated properties from inner Fact ---------------------------------
+
+    @property
+    def id(self) -> str:
+        return self.fact.id
+
+    @property
+    def text(self) -> str:
+        return self.fact.text
+
+    @property
+    def category(self) -> str:
+        return self.fact.category
+
+    @property
+    def confidence(self) -> float:
+        return self.fact.confidence
+
+    @property
+    def source_window_id(self) -> str:
+        return self.fact.source_window_id
+
+    @property
+    def created_at(self) -> float:
+        return self.fact.created_at
+
+    @property
+    def superseded_by(self) -> str | None:
+        return self.fact.superseded_by
+
+    @superseded_by.setter
+    def superseded_by(self, value: str | None) -> None:
+        self.fact.superseded_by = value
+
+    # --- Lazy embedding -------------------------------------------------------
+
+    @property
+    def embedding(self) -> list[float] | None:
+        """Lazy-compute embedding on first access."""
+        if self._embedding is None and _EMBED_FN is not None:
+            try:
+                self._embedding = _EMBED_FN(self.text)
+            except Exception:  # noqa: BLE001
+                logger.warning("Embedding computation failed for fact %s", self.id)
+        return self._embedding
+
+    @embedding.setter
+    def embedding(self, value: list[float] | None) -> None:
+        self._embedding = value
+
+    def has_embedding(self) -> bool:
+        return self._embedding is not None
+
+    # --- State mutations ------------------------------------------------------
+
+    def mark_seen(self, window_id: str) -> None:
+        """Record that this fact was included in an envelope for *window_id*."""
+        self.seen_count += 1
+        if window_id not in self.consumed_by_windows:
+            self.consumed_by_windows.append(window_id)
+
+    def increment_age(self) -> None:
+        """Advance age by one window."""
+        self.age_in_windows += 1
+
+    def supersede(self, by_fact_id: str, confidence: float = 1.0) -> None:
+        """Mark this fact as superseded."""
+        self.fact.superseded_by = by_fact_id
+        self.fact.supersession_confidence = confidence
+
+    @property
+    def is_superseded(self) -> bool:
+        return self.fact.superseded_by is not None
+
+    # --- Serialization --------------------------------------------------------
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to dict for persistence (including embeddings §4D.1)."""
+        d: dict[str, Any] = {
+            "fact_id": self.id,
+            "text": self.text,
+            "category": self.category,
+            "confidence": self.confidence,
+            "source_window_id": self.source_window_id,
+            "created_at": self.created_at,
+            "extraction_stage": self.fact.extraction_stage,
+            "age_in_windows": self.age_in_windows,
+            "seen_count": self.seen_count,
+            "consumed_by_windows": self.consumed_by_windows,
+            "graph_edges": self.graph_edges,
+            "superseded_by": self.superseded_by,
+            "supersession_confidence": self.fact.supersession_confidence,
+            "tier": self.tier,
+            "has_embedding": self.has_embedding(),
+        }
+        # Persist actual embedding vectors when available (§4D.1)
+        if self._embedding is not None:
+            d["embedding"] = self._embedding
+        return d
+
+    @classmethod
+    def from_fact(cls, fact: Fact) -> StateFact:
+        """Wrap an extraction Fact into a StateFact."""
+        return cls(fact=fact)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> StateFact:
+        """Deserialize from dict."""
+        fact = Fact(
+            id=data["fact_id"],
+            text=data["text"],
+            category=data.get("category", ""),
+            confidence=data.get("confidence", 0.0),
+            source_window_id=data.get("source_window_id", ""),
+            created_at=data.get("created_at", time.time()),
+            extraction_stage=data.get("extraction_stage", 0),
+            superseded_by=data.get("superseded_by"),
+            supersession_confidence=data.get("supersession_confidence", 0.0),
+        )
+        sf = cls(
+            fact=fact,
+            age_in_windows=data.get("age_in_windows", 0),
+            seen_count=data.get("seen_count", 0),
+            consumed_by_windows=data.get("consumed_by_windows", []),
+            graph_edges=data.get("graph_edges", []),
+            tier=data.get("tier", 2),
+        )
+        # Restore embedding vectors if persisted (§4D.1)
+        if "embedding" in data and data["embedding"] is not None:
+            sf._embedding = data["embedding"]
+        return sf

crp/state/serialization.py

@@ -0,0 +1,189 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""FactGraph serialization — on-disk format with schema versioning (§22).
+
+Provides forward/backward compatible serialization of the full fact graph
+including nodes, edges, and metadata.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from crp.extraction.types import Fact, FactEdge, FactGraph, RelationType
+
+# ---------------------------------------------------------------------------
+# Schema version
+# ---------------------------------------------------------------------------
+
+GRAPH_SCHEMA_VERSION = 1
+
+
+@dataclass
+class GraphSerializationHeader:
+    """Header for serialized FactGraph files."""
+
+    schema_version: int = GRAPH_SCHEMA_VERSION
+    node_count: int = 0
+    edge_count: int = 0
+    checksum: str = ""
+
+
+# ---------------------------------------------------------------------------
+# FactGraphSerializer
+# ---------------------------------------------------------------------------
+
+
+class FactGraphSerializer:
+    """Serialize/deserialize FactGraph to/from disk (§22).
+
+    Format: JSON with header + nodes + edges.
+    Schema-versioned for forward/backward compatibility.
+    """
+
+    @staticmethod
+    def serialize(graph: FactGraph) -> dict[str, Any]:
+        """Serialize a FactGraph to a dict."""
+        nodes = []
+        for _fid, fact in graph.nodes.items():
+            nodes.append({
+                "id": fact.id,
+                "text": fact.text,
+                "category": fact.category,
+                "source_window_id": fact.source_window_id,
+                "confidence": fact.confidence,
+                "extraction_stage": fact.extraction_stage,
+                "created_at": fact.created_at,
+                "superseded_by": fact.superseded_by,
+                "supersession_confidence": fact.supersession_confidence,
+                "metadata": fact.metadata,
+            })
+
+        edges = []
+        for edge in graph.edges:
+            rel_type = edge.relation_type
+            if isinstance(rel_type, RelationType):
+                rel_type = rel_type.value
+            edges.append({
+                "id": edge.id,
+                "source_id": edge.source_id,
+                "target_id": edge.target_id,
+                "relation_type": str(rel_type),
+                "confidence": edge.confidence,
+                "source_stage": edge.source_stage,
+                "metadata": edge.metadata,
+            })
+
+        payload = {"nodes": nodes, "edges": edges}
+        payload_str = json.dumps(payload, sort_keys=True, default=str)
+
+        try:
+            import blake3  # type: ignore[import-untyped]
+
+            checksum = blake3.blake3(payload_str.encode()).hexdigest()
+        except ImportError:
+            checksum = hashlib.sha256(payload_str.encode()).hexdigest()
+
+        header = GraphSerializationHeader(
+            node_count=len(nodes),
+            edge_count=len(edges),
+            checksum=checksum,
+        )
+
+        return {
+            "header": {
+                "schema_version": header.schema_version,
+                "node_count": header.node_count,
+                "edge_count": header.edge_count,
+                "checksum": header.checksum,
+            },
+            "nodes": nodes,
+            "edges": edges,
+        }
+
+    @staticmethod
+    def deserialize(data: dict[str, Any]) -> tuple[FactGraph, list[str]]:
+        """Deserialize a dict into a FactGraph.  Returns (graph, warnings)."""
+        warnings: list[str] = []
+        header = data.get("header", {})
+
+        # Schema version check
+        version = header.get("schema_version", 0)
+        if version != GRAPH_SCHEMA_VERSION:
+            warnings.append(f"Schema version mismatch: {version} vs {GRAPH_SCHEMA_VERSION}")
+
+        graph = FactGraph()
+
+        # Deserialize nodes
+        for ndata in data.get("nodes", []):
+            fact = Fact(
+                id=ndata.get("id", ""),
+                text=ndata.get("text", ""),
+                category=ndata.get("category", ""),
+                source_window_id=ndata.get("source_window_id", ""),
+                confidence=ndata.get("confidence", 0.0),
+                extraction_stage=ndata.get("extraction_stage", 0),
+                created_at=ndata.get("created_at", 0.0),
+                superseded_by=ndata.get("superseded_by"),
+                supersession_confidence=ndata.get("supersession_confidence", 0.0),
+                metadata=ndata.get("metadata", {}),
+            )
+            graph.add_fact(fact)
+
+        # Deserialize edges (check orphans before add_edge, which silently drops them)
+        node_ids = set(graph.nodes.keys())
+        for edata in data.get("edges", []):
+            rel_str = edata.get("relation_type", "RELATED")
+            try:
+                rel_type: RelationType | str = RelationType(rel_str)
+            except ValueError:
+                rel_type = rel_str
+
+            edge = FactEdge(
+                id=edata.get("id", ""),
+                source_id=edata.get("source_id", ""),
+                target_id=edata.get("target_id", ""),
+                relation_type=rel_type,
+                confidence=edata.get("confidence", 0.0),
+                source_stage=edata.get("source_stage", 0),
+                metadata=edata.get("metadata", {}),
+            )
+            if edge.source_id not in node_ids:
+                warnings.append(f"Orphaned edge source: {edge.source_id}")
+            if edge.target_id not in node_ids:
+                warnings.append(f"Orphaned edge target: {edge.target_id}")
+            graph.add_edge(edge)
+
+        # Verify counts
+        expected_nodes = header.get("node_count", 0)
+        expected_edges = header.get("edge_count", 0)
+        if expected_nodes and len(graph.nodes) != expected_nodes:
+            warnings.append(f"Node count mismatch: header={expected_nodes}, loaded={len(graph.nodes)}")
+        if expected_edges and len(graph.edges) != expected_edges:
+            warnings.append(f"Edge count mismatch: header={expected_edges}, loaded={len(graph.edges)}")
+
+        return graph, warnings
+
+    @classmethod
+    def save_to_file(cls, graph: FactGraph, path: str | Path) -> None:
+        """Serialize and write to file."""
+        data = cls.serialize(graph)
+        p = Path(path)
+        p.parent.mkdir(parents=True, exist_ok=True)
+        p.write_text(json.dumps(data, default=str), encoding="utf-8")
+
+    @classmethod
+    def load_from_file(cls, path: str | Path) -> tuple[FactGraph, list[str]]:
+        """Load from file.  Returns (graph, warnings)."""
+        p = Path(path)
+        if not p.exists():
+            return FactGraph(), [f"File not found: {path}"]
+        try:
+            data = json.loads(p.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError) as exc:
+            return FactGraph(), [f"Failed to read file: {exc}"]
+        return cls.deserialize(data)