crprotocol 2.0.0__py3-none-any.whl

crp/ckf/fabric.py

@@ -0,0 +1,445 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Contextual Knowledge Fabric — unified 4-mode retrieval interface (§3.8).
+
+The CKF is the top-level interface for fact storage, retrieval, community
+detection, pub/sub events, and cross-session persistence.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from crp.extraction.types import Fact, FactEdge, RelationType
+from crp.state.cold_storage import persist_to_cold, restore_from_cold
+from crp.state.event_log import FactEventLog
+from crp.state.warm_store import WarmStateStore, WarmStoreConfig
+
+from .community import Community, CommunityDetector, CommunityResult
+from .gc import GarbageCollector, GCResult
+from .graph_walk import GraphWalkResult, graph_walk
+from .merge import MergeResult, multi_mode_merge
+from .pattern_query import PatternQueryResult, pattern_query
+from .pubsub import CKFEvent, CKFEventType, EventCallback, PubSubEventBus
+from .semantic import HNSWIndex, _check_hnswlib, semantic_fallback
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Config
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class CKFConfig:
+    """Configuration for the Contextual Knowledge Fabric."""
+
+    max_facts: int = 10_000
+    hnsw_threshold: int = 1000
+    persist_path: str = ""
+    gc_budget_bytes: int = 500 * 1024 * 1024
+    gc_trigger_ratio: float = 0.80
+    gc_target_ratio: float = 0.70
+    community_detect_enabled: bool = True
+
+
+# ---------------------------------------------------------------------------
+# Health report
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class CKFHealth:
+    """Health snapshot for monitoring."""
+
+    fact_count: int = 0
+    edge_count: int = 0
+    community_count: int = 0
+    event_count: int = 0
+    tombstoned_count: int = 0
+    estimated_bytes: int = 0
+    hnsw_active: bool = False
+    leiden_available: bool = False
+
+
+# ---------------------------------------------------------------------------
+# ContextualKnowledgeFabric
+# ---------------------------------------------------------------------------
+
+
+class ContextualKnowledgeFabric:
+    """Unified interface for fact storage and 4-mode retrieval (§3.8).
+
+    Methods (per spec 4F.1a):
+    - store(facts) / retrieve(query, modes, budget)
+    - query(pattern) / persist(path) / restore(path)
+    - fact_count() / health()
+    - temporal_query(window_range)
+    - graph_walk(seeds, hops) / community_summary(topic)
+    - subscribe(event, callback)
+    """
+
+    def __init__(self, config: CKFConfig | None = None) -> None:
+        self._config = config or CKFConfig()
+
+        # Core stores
+        self._warm = WarmStateStore(
+            WarmStoreConfig(max_facts=self._config.max_facts)
+        )
+        self._event_log = FactEventLog()
+
+        # CKF subsystems
+        self._bus = PubSubEventBus()
+        self._community = CommunityDetector()
+        self._gc = GarbageCollector(
+            budget_bytes=self._config.gc_budget_bytes,
+            trigger_ratio=self._config.gc_trigger_ratio,
+            target_ratio=self._config.gc_target_ratio,
+        )
+
+        # HNSW index (built lazily when needed)
+        self._hnsw: HNSWIndex | None = None
+        self._hnsw_dirty: bool = True
+
+        # Community cache
+        self._community_result: CommunityResult | None = None
+
+        # Cooldown for community detection to avoid excessive re-computation
+        self._last_community_detect: float = 0.0
+        _COMMUNITY_COOLDOWN_SECONDS = 30.0
+        self._community_cooldown = _COMMUNITY_COOLDOWN_SECONDS
+
+    # ====================================================================
+    # Store
+    # ====================================================================
+
+    def store(self, facts: list[Fact], window_id: str = "") -> None:
+        """Ingest facts into the warm store and emit events."""
+        # Set source_window_id on facts if not already set
+        for f in facts:
+            if not f.source_window_id and window_id:
+                f.source_window_id = window_id
+        added = self._warm.add_facts(facts)
+        for sf in added:
+            self._event_log.record_fact_created(sf.fact, window_id)
+            self._bus.publish(
+                CKFEvent(CKFEventType.FACT_CREATED, {"fact_id": sf.id, "window_id": window_id})
+            )
+        self._hnsw_dirty = True
+
+        # Auto-trigger community detection when fact count crosses thresholds
+        # (every 50 new facts or when first reaching 20 facts). This ensures
+        # community mode is usable during retrieval without explicit calls.
+        if self._config.community_detect_enabled and added:
+            fact_count = len(self._warm._facts)
+            prev_count = fact_count - len(added)
+            should_detect = (
+                (prev_count < 20 <= fact_count)
+                or (fact_count >= 20 and fact_count // 50 > prev_count // 50)
+            )
+            if should_detect:
+                if time.monotonic() - self._last_community_detect < self._community_cooldown:
+                    logger.debug("Community detection skipped (cooldown active)")
+                else:
+                    try:
+                        self.detect_communities()
+                        self._last_community_detect = time.monotonic()
+                    except Exception:
+                        logger.warning("Auto community detection failed", exc_info=True)
+
+    def store_edges(self, edges: list[FactEdge]) -> None:
+        """Add edges to the fact graph."""
+        for edge in edges:
+            self._warm._graph.add_edge(edge)
+            self._event_log.record_edge_added(edge, "")
+            self._bus.publish(
+                CKFEvent(
+                    CKFEventType.EDGE_ADDED,
+                    {"source_id": edge.source_id, "target_id": edge.target_id},
+                )
+            )
+        self._hnsw_dirty = True
+
+    # ====================================================================
+    # Retrieve — 4-mode merge
+    # ====================================================================
+
+    def retrieve(
+        self,
+        query_embedding: list[float] | None = None,
+        seed_ids: set[str] | None = None,
+        entity_type: str | None = None,
+        relationship_type: str | RelationType | None = None,
+        topic: str | None = None,
+        modes: list[str] | None = None,
+        budget: int = 200,
+    ) -> MergeResult:
+        """Retrieve facts using up to 4 modes, merged and ranked.
+
+        Parameters
+        ----------
+        query_embedding : for semantic mode
+        seed_ids : for graph_walk mode
+        entity_type / relationship_type : for pattern mode
+        topic : for community mode
+        modes : subset of ["graph_walk", "pattern", "semantic", "community"]
+                defaults to all applicable modes
+        budget : max facts to return
+        """
+        if modes is None:
+            modes = self._infer_modes(query_embedding, seed_ids, entity_type, topic)
+
+        graph = self._warm._graph
+        mode_results: dict[str, list[tuple[Fact, float]]] = {}
+
+        # Mode 1: Graph walk
+        if "graph_walk" in modes and seed_ids:
+            gw = graph_walk(graph, seed_ids, max_hops=2, max_results=budget)
+            mode_results["graph_walk"] = [
+                (f, 1.0 / (1.0 + gw.distances.get(f.id, 3))) for f in gw.facts
+            ]
+
+        # Mode 2: Pattern query
+        if "pattern" in modes and (entity_type or relationship_type):
+            pq = pattern_query(graph, entity_type, relationship_type, max_results=budget)
+            mode_results["pattern"] = [(f, f.confidence or 0.5) for f in pq.facts]
+
+        # Mode 3: Semantic fallback
+        if "semantic" in modes and query_embedding:
+            self._ensure_hnsw()
+            sem = semantic_fallback(
+                query_embedding,
+                self._warm._facts,
+                top_k=budget,
+                hnsw_index=self._hnsw,
+            )
+            mode_results["semantic"] = [
+                (sf.fact, sem.scores.get(sf.id, 0.0)) for sf in sem.facts
+            ]
+
+        # Mode 4: Community summary
+        if "community" in modes and topic:
+            comms = self.community_summary(topic)
+            comm_facts: list[tuple[Fact, float]] = []
+            for comm in comms:
+                for fid in comm.fact_ids:
+                    fact = graph.nodes.get(fid)
+                    if fact:
+                        comm_facts.append((fact, fact.confidence or 0.5))
+            mode_results["community"] = comm_facts[:budget]
+
+        # Merge
+        # Validate facts from all modes before merging (§audit G6)
+        for mode_name in list(mode_results):
+            mode_results[mode_name] = [
+                (f, s) for f, s in mode_results[mode_name]
+                if f.id and f.text and len(f.text.strip()) >= 3
+            ]
+
+        fact_to_comm = (
+            self._community_result.fact_to_community
+            if self._community_result
+            else None
+        )
+        return multi_mode_merge(
+            mode_results,
+            fact_to_community=fact_to_comm,
+            max_results=budget,
+        )
+
+    # ====================================================================
+    # Query (pattern shorthand)
+    # ====================================================================
+
+    def query(
+        self,
+        entity_type: str | None = None,
+        relationship_type: str | RelationType | None = None,
+        min_confidence: float = 0.0,
+        max_results: int = 200,
+    ) -> PatternQueryResult:
+        """Convenience: pattern query on the fact graph."""
+        return pattern_query(
+            self._warm._graph, entity_type, relationship_type,
+            min_confidence=min_confidence, max_results=max_results,
+        )
+
+    # ====================================================================
+    # Graph walk
+    # ====================================================================
+
+    def graph_walk(
+        self,
+        seed_ids: set[str],
+        max_hops: int = 2,
+        max_results: int = 200,
+    ) -> GraphWalkResult:
+        """BFS traversal from seed facts."""
+        return graph_walk(self._warm._graph, seed_ids, max_hops, max_results)
+
+    # ====================================================================
+    # Community
+    # ====================================================================
+
+    def community_summary(self, topic: str) -> list[Community]:
+        """Return communities matching *topic*."""
+        if self._community_result is None and self._config.community_detect_enabled:
+            self._community_result = self._community.detect(self._warm._graph)
+            self._bus.publish(
+                CKFEvent(CKFEventType.COMMUNITY_UPDATED, {"community_count": len(self._community_result.communities)})
+            )
+        return self._community.community_summary(self._warm._graph, topic)
+
+    def detect_communities(self) -> CommunityResult:
+        """Force community detection run."""
+        self._community_result = self._community.detect(self._warm._graph)
+        self._bus.publish(
+            CKFEvent(CKFEventType.COMMUNITY_UPDATED, {"community_count": len(self._community_result.communities)})
+        )
+        return self._community_result
+
+    # ====================================================================
+    # Temporal query
+    # ====================================================================
+
+    def temporal_query(
+        self,
+        start_window: str,
+        end_window: str,
+    ) -> list[str]:
+        """Return fact IDs active between two windows."""
+        return self._event_log.facts_between(start_window, end_window)  # type: ignore[return-value]
+
+    # ====================================================================
+    # Persistence
+    # ====================================================================
+
+    def persist(self, path: str | Path) -> None:
+        """Persist full state to cold storage, including community IDs."""
+        community_map = {}
+        if self._community_result:
+            community_map = dict(self._community_result.fact_to_community)
+        persist_to_cold(
+            self._warm, self._event_log, str(path),
+            community_map=community_map,
+        )
+
+    def restore(self, path: str | Path) -> list[str]:
+        """Restore state from cold storage.  Returns warnings."""
+        _header, warnings, community_map = restore_from_cold(
+            self._warm, self._event_log, str(path)
+        )
+        self._hnsw_dirty = True
+        self._community_result = None
+        return warnings
+
+    # ====================================================================
+    # Subscribe (pub/sub)
+    # ====================================================================
+
+    def subscribe(self, event_type: CKFEventType, callback: EventCallback) -> None:
+        """Register a callback for CKF events."""
+        self._bus.subscribe(event_type, callback)
+
+    # ====================================================================
+    # Introspection
+    # ====================================================================
+
+    def fact_count(self) -> int:
+        return len(self._warm._facts)
+
+    def health(self) -> CKFHealth:
+        """Return a health snapshot."""
+        from .community import _check_leiden
+
+        return CKFHealth(
+            fact_count=len(self._warm._facts),
+            edge_count=len(self._warm._graph.edges),
+            community_count=(
+                len(self._community_result.communities)
+                if self._community_result
+                else 0
+            ),
+            event_count=len(self._event_log._events),
+            tombstoned_count=self._gc.tombstone_count(),
+            estimated_bytes=GarbageCollector.estimate_store_bytes(self._warm._facts),
+            hnsw_active=self._hnsw is not None,
+            leiden_available=_check_leiden(),
+        )
+
+    # ====================================================================
+    # GC
+    # ====================================================================
+
+    def run_gc(self, current_window: int = 0) -> GCResult:
+        """Run cross-session garbage collection."""
+        estimated = GarbageCollector.estimate_store_bytes(self._warm._facts)
+        return self._gc.run(self._warm._facts, estimated, current_window)
+
+    def should_gc(self) -> bool:
+        """Check if GC should be triggered."""
+        estimated = GarbageCollector.estimate_store_bytes(self._warm._facts)
+        return self._gc.should_gc(estimated)
+
+    # ====================================================================
+    # Internal helpers
+    # ====================================================================
+
+    def _infer_modes(
+        self,
+        query_embedding: list[float] | None,
+        seed_ids: set[str] | None,
+        entity_type: str | None,
+        topic: str | None,
+    ) -> list[str]:
+        """Auto-detect which modes to activate based on provided inputs."""
+        modes: list[str] = []
+        if seed_ids:
+            modes.append("graph_walk")
+        if entity_type:
+            modes.append("pattern")
+        if query_embedding:
+            modes.append("semantic")
+        if topic:
+            modes.append("community")
+        return modes  # return empty if no inputs match any mode
+
+    def _ensure_hnsw(self) -> None:
+        """Build/rebuild HNSW index if needed and hnswlib is available."""
+        if not self._hnsw_dirty:
+            return
+        if _check_hnswlib() is None:
+            self._hnsw = None
+            self._hnsw_dirty = False
+            return
+        if len(self._warm._facts) < self._config.hnsw_threshold:
+            self._hnsw = None
+            self._hnsw_dirty = False
+            return
+
+        # Determine embedding dimension from first available embedding
+        dim = 0
+        for sf in self._warm._facts.values():
+            if sf.has_embedding() and sf.embedding:
+                dim = len(sf.embedding)
+                break
+        if dim == 0:
+            self._hnsw = None
+            self._hnsw_dirty = False
+            return
+
+        try:
+            idx = HNSWIndex(dim=dim, max_elements=len(self._warm._facts) + 1000)
+            for sf in self._warm._facts.values():
+                emb = sf.embedding
+                if emb is not None:
+                    idx.add(sf.id, emb)
+            self._hnsw = idx
+        except Exception:  # noqa: BLE001
+            logger.warning("Failed to build HNSW index, using brute-force")
+            self._hnsw = None
+        self._hnsw_dirty = False

crp/ckf/gc.py

@@ -0,0 +1,175 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""CKF cross-session garbage collection (§3.8).
+
+gc_score formula determines fact retention priority.
+Tombstone → purge lifecycle.  Budget 500 MB, trigger 80%, target 70%.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+
+from crp.state.fact import StateFact
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Config
+# ---------------------------------------------------------------------------
+
+BUDGET_BYTES = 500 * 1024 * 1024  # 500 MB
+TRIGGER_RATIO = 0.80  # Trigger GC at 80% of budget
+TARGET_RATIO = 0.70  # Reduce to 70% of budget
+TOMBSTONE_AGE_WINDOWS = 50  # Windows before tombstone → purge
+
+
+# ---------------------------------------------------------------------------
+# GC score formula
+# ---------------------------------------------------------------------------
+
+
+def gc_score(fact: StateFact, current_window: int = 0) -> float:
+    """Compute GC retention score for a fact.
+
+    Higher score = more worth keeping.  Components:
+    - Confidence: raw confidence value
+    - Freshness: inverse of age (recently created facts score higher)
+    - Usage: how many envelopes consumed this fact
+    - Graph connectivity: number of edges (well-connected facts are more valuable)
+
+    Formula:
+        gc_score = 0.3 * confidence + 0.3 * freshness + 0.2 * usage + 0.2 * connectivity
+    """
+    confidence = min(1.0, fact.confidence or 0.0)
+
+    # Freshness — decays with age; 1.0 at age 0, approaches 0 for old facts
+    age = fact.age_in_windows
+    freshness = 1.0 / (1.0 + age * 0.1)
+
+    # Usage — normalised seen_count (cap at 20)
+    usage = min(1.0, fact.seen_count / 20.0)
+
+    # Connectivity — normalised edge count (cap at 10)
+    connectivity = min(1.0, len(fact.graph_edges) / 10.0)
+
+    return 0.3 * confidence + 0.3 * freshness + 0.2 * usage + 0.2 * connectivity
+
+
+# ---------------------------------------------------------------------------
+# GC result
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class GCResult:
+    """Result of a GC pass."""
+
+    tombstoned: list[str] = field(default_factory=list)
+    purged: list[str] = field(default_factory=list)
+    bytes_freed_estimate: int = 0
+    facts_before: int = 0
+    facts_after: int = 0
+
+
+# ---------------------------------------------------------------------------
+# GarbageCollector
+# ---------------------------------------------------------------------------
+
+
+class GarbageCollector:
+    """Cross-session GC for the CKF fact store.
+
+    Lifecycle: active → tombstoned → purged.
+    Tombstoned facts are excluded from retrieval but retained for
+    TOMBSTONE_AGE_WINDOWS before final purge.
+    """
+
+    def __init__(
+        self,
+        budget_bytes: int = BUDGET_BYTES,
+        trigger_ratio: float = TRIGGER_RATIO,
+        target_ratio: float = TARGET_RATIO,
+    ) -> None:
+        self._budget = budget_bytes
+        self._trigger = trigger_ratio
+        self._target = target_ratio
+        self._tombstones: dict[str, int] = {}  # fact_id → window when tombstoned
+
+    def should_gc(self, estimated_bytes: int) -> bool:
+        """Return True if GC should run (estimated usage ≥ trigger)."""
+        return estimated_bytes >= int(self._budget * self._trigger)
+
+    def run(
+        self,
+        facts: dict[str, StateFact],
+        estimated_bytes: int,
+        current_window: int = 0,
+    ) -> GCResult:
+        """Execute a GC pass.
+
+        1. Purge old tombstones (aged out).
+        2. If still over target, tombstone lowest-scoring active facts.
+        """
+        result = GCResult(facts_before=len(facts))
+        target_bytes = int(self._budget * self._target)
+
+        # --- Phase 1: Purge old tombstones ---
+        to_purge: list[str] = []
+        for fid, tombstone_window in list(self._tombstones.items()):
+            if current_window - tombstone_window >= TOMBSTONE_AGE_WINDOWS:
+                to_purge.append(fid)
+
+        for fid in to_purge:
+            if fid in facts:
+                result.bytes_freed_estimate += self._estimate_fact_bytes(facts[fid])
+                del facts[fid]
+                result.purged.append(fid)
+            self._tombstones.pop(fid, None)
+
+        # Recalculate
+        estimated_bytes -= result.bytes_freed_estimate
+
+        # --- Phase 2: Tombstone low-value facts if still over target ---
+        if estimated_bytes > target_bytes:
+            # Score all active (non-tombstoned) facts
+            active: list[tuple[float, str]] = []
+            for fid, sf in facts.items():
+                if fid not in self._tombstones and not sf.is_superseded:
+                    active.append((gc_score(sf, current_window), fid))
+
+            # Sort ascending — lowest score first (candidates for tombstoning)
+            active.sort()
+
+            for _score, fid in active:
+                if estimated_bytes <= target_bytes:
+                    break
+                self._tombstones[fid] = current_window
+                result.tombstoned.append(fid)
+                estimated_bytes -= self._estimate_fact_bytes(facts[fid])
+
+        result.facts_after = len(facts)
+        return result
+
+    def is_tombstoned(self, fact_id: str) -> bool:
+        return fact_id in self._tombstones
+
+    def tombstone_count(self) -> int:
+        return len(self._tombstones)
+
+    @staticmethod
+    def _estimate_fact_bytes(sf: StateFact) -> int:
+        """Rough byte estimate for a StateFact."""
+        base = len(sf.text.encode("utf-8")) + 200  # overhead
+        if sf.has_embedding():
+            base += len(sf.embedding or []) * 4  # float32
+        return base
+
+    @staticmethod
+    def estimate_store_bytes(facts: dict[str, StateFact]) -> int:
+        """Estimate total bytes for the fact store."""
+        total = 0
+        for sf in facts.values():
+            total += GarbageCollector._estimate_fact_bytes(sf)
+        return total

crp/ckf/graph_walk.py

@@ -0,0 +1,87 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""CKF Mode 1: Graph walk — BFS traversal from seed facts (§3.8).
+
+``graph_walk(seed_facts, max_hops=2)`` returns a ranked list of facts
+reachable within *max_hops* of the seed set, ordered by proximity.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass, field
+
+from crp.extraction.types import Fact, FactGraph
+
+# ---------------------------------------------------------------------------
+# Result type
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class GraphWalkResult:
+    """Result of a graph walk query."""
+
+    facts: list[Fact] = field(default_factory=list)
+    distances: dict[str, int] = field(default_factory=dict)
+    visited_count: int = 0
+
+
+# ---------------------------------------------------------------------------
+# BFS graph walk
+# ---------------------------------------------------------------------------
+
+
+def graph_walk(
+    graph: FactGraph,
+    seed_ids: set[str],
+    max_hops: int = 2,
+    max_results: int = 200,
+) -> GraphWalkResult:
+    """BFS walk from *seed_ids* up to *max_hops*.
+
+    Returns facts ordered by hop distance (closer first), then by confidence.
+    Seed facts themselves are included at distance 0.
+    """
+    if not seed_ids or not graph.nodes:
+        return GraphWalkResult()
+
+    # Build adjacency index for O(1) neighbour lookup
+    adj: dict[str, set[str]] = {}
+    for edge in graph.edges:
+        adj.setdefault(edge.source_id, set()).add(edge.target_id)
+        adj.setdefault(edge.target_id, set()).add(edge.source_id)
+
+    distances: dict[str, int] = {}
+    queue: deque[tuple[str, int]] = deque()
+
+    # Seed nodes
+    for sid in seed_ids:
+        if sid in graph.nodes:
+            distances[sid] = 0
+            queue.append((sid, 0))
+
+    # BFS
+    while queue:
+        node_id, dist = queue.popleft()
+        if dist >= max_hops:
+            continue
+        for neighbour in adj.get(node_id, set()):
+            if neighbour not in distances and neighbour in graph.nodes:
+                distances[neighbour] = dist + 1
+                queue.append((neighbour, dist + 1))
+
+    # Collect and rank: sort by distance asc, then confidence desc
+    ranked: list[tuple[int, float, str]] = []
+    for fid, dist in distances.items():
+        fact = graph.nodes.get(fid)
+        if fact:
+            ranked.append((dist, -(fact.confidence or 0.0), fid))
+
+    ranked.sort()
+    facts = [graph.nodes[fid] for _, _, fid in ranked[:max_results]]
+    return GraphWalkResult(
+        facts=facts,
+        distances={fid: distances[fid] for _, _, fid in ranked[:max_results]},
+        visited_count=len(distances),
+    )