crprotocol 2.0.0__py3-none-any.whl

crp/ckf/merge.py

@@ -0,0 +1,133 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""CKF multi-mode merge — deduplicate, score, rank, community boost (§3.8).
+
+``multi_mode_merge()`` combines results from all four CKF retrieval modes
+into a single ranked list, applying deduplication and community-coherence boosting.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from crp.extraction.types import Fact
+
+# ---------------------------------------------------------------------------
+# Config
+# ---------------------------------------------------------------------------
+
+# Weight per mode (graph_walk, pattern, semantic, community)
+DEFAULT_MODE_WEIGHTS: dict[str, float] = {
+    "graph_walk": 0.30,
+    "pattern": 0.25,
+    "semantic": 0.30,
+    "community": 0.15,
+}
+
+# Community coherence boost: facts from same community as top fact get +boost
+COMMUNITY_BOOST = 0.10
+
+
+# ---------------------------------------------------------------------------
+# Result type
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class MergedFact:
+    """A fact with a merged score from multiple retrieval modes."""
+
+    fact: Fact
+    score: float = 0.0
+    sources: list[str] = field(default_factory=list)
+    community_boosted: bool = False
+
+
+@dataclass
+class MergeResult:
+    """Result of multi-mode merge."""
+
+    facts: list[MergedFact] = field(default_factory=list)
+    total_candidates: int = 0
+    deduplicated: int = 0
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def multi_mode_merge(
+    mode_results: dict[str, list[tuple[Fact, float]]],
+    fact_to_community: dict[str, int] | None = None,
+    mode_weights: dict[str, float] | None = None,
+    max_results: int = 200,
+) -> MergeResult:
+    """Merge results from multiple CKF retrieval modes.
+
+    Parameters
+    ----------
+    mode_results : dict[str, list[tuple[Fact, float]]]
+        Mapping of mode name → list of (fact, score) tuples.
+        Mode names: "graph_walk", "pattern", "semantic", "community".
+    fact_to_community : dict[str, int] | None
+        Mapping of fact_id → community_id for community boosting.
+    mode_weights : dict[str, float] | None
+        Override default mode weights.
+    max_results : int
+        Maximum number of facts to return.
+    """
+    weights = mode_weights or DEFAULT_MODE_WEIGHTS
+
+    # --- Phase 1: Aggregate scores ---
+    scores: dict[str, float] = {}
+    sources: dict[str, list[str]] = {}
+    fact_by_id: dict[str, Fact] = {}
+    total_candidates = 0
+
+    for mode_name, results in mode_results.items():
+        weight = weights.get(mode_name, 0.2)
+        for fact, raw_score in results:
+            total_candidates += 1
+            fid = fact.id
+            fact_by_id[fid] = fact
+            scores[fid] = scores.get(fid, 0.0) + raw_score * weight
+            sources.setdefault(fid, []).append(mode_name)
+
+    # --- Phase 2: Community boost ---
+    if fact_to_community and scores:
+        # Find the top fact's community
+        top_fid = max(scores, key=lambda k: scores[k])
+        top_comm = fact_to_community.get(top_fid)
+        if top_comm is not None:
+            for fid in scores:
+                if fact_to_community.get(fid) == top_comm and fid != top_fid:
+                    scores[fid] += COMMUNITY_BOOST
+
+    # --- Phase 3: Rank and build result ---
+    ranked = sorted(scores.items(), key=lambda x: -x[1])
+    merged_facts: list[MergedFact] = []
+    for fid, score in ranked[:max_results]:
+        fact = fact_by_id.get(fid)  # type: ignore[assignment]
+        if fact is None:
+            continue
+        boosted = False
+        if fact_to_community:
+            top_fid = ranked[0][0] if ranked else ""
+            top_comm = fact_to_community.get(top_fid)
+            if top_comm is not None and fact_to_community.get(fid) == top_comm and fid != top_fid:
+                boosted = True
+        merged_facts.append(
+            MergedFact(
+                fact=fact,
+                score=score,
+                sources=sources.get(fid, []),
+                community_boosted=boosted,
+            )
+        )
+
+    return MergeResult(
+        facts=merged_facts,
+        total_candidates=total_candidates,
+        deduplicated=total_candidates - len(merged_facts),
+    )

crp/ckf/pattern_query.py

@@ -0,0 +1,122 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""CKF Mode 2: Pattern query — structured matching by entity/relation type (§3.8).
+
+``pattern_query(graph, entity_type, relationship_type)`` filters facts
+and edges by category and relation type.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from crp.extraction.types import Fact, FactEdge, FactGraph, RelationType
+
+# ---------------------------------------------------------------------------
+# Result type
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class PatternQueryResult:
+    """Result of a pattern query."""
+
+    facts: list[Fact] = field(default_factory=list)
+    edges: list[FactEdge] = field(default_factory=list)
+    match_count: int = 0
+
+
+# ---------------------------------------------------------------------------
+# Pattern query
+# ---------------------------------------------------------------------------
+
+
+def pattern_query(
+    graph: FactGraph,
+    entity_type: str | None = None,
+    relationship_type: str | RelationType | None = None,
+    min_confidence: float = 0.0,
+    max_results: int = 200,
+    metadata_filter: dict[str, Any] | None = None,
+) -> PatternQueryResult:
+    """Structured query: filter facts by category and edges by relation type.
+
+    Parameters
+    ----------
+    graph : FactGraph
+    entity_type : str | None
+        Filter facts whose ``category`` matches (case-insensitive).
+    relationship_type : str | RelationType | None
+        Filter edges whose ``relation_type`` matches.
+    min_confidence : float
+        Minimum confidence threshold for both facts and edges.
+    max_results : int
+        Cap on returned facts.
+    metadata_filter : dict | None
+        Key-value pairs that must all appear in ``fact.metadata``.
+    """
+    # Normalise relationship type for comparison
+    rel_str: str | None = None
+    if relationship_type is not None:
+        rel_str = (
+            relationship_type.value
+            if isinstance(relationship_type, RelationType)
+            else str(relationship_type)
+        ).upper()
+
+    ent_upper = entity_type.upper() if entity_type else None
+
+    # --- Filter edges first (to collect connected fact IDs) ---
+    matched_edges: list[FactEdge] = []
+    edge_fact_ids: set[str] = set()
+
+    if rel_str is not None:
+        for edge in graph.edges:
+            edge_rel = edge.relation_type
+            if isinstance(edge_rel, RelationType):
+                edge_rel = edge_rel.value
+            if str(edge_rel).upper() == rel_str and edge.confidence >= min_confidence:
+                matched_edges.append(edge)
+                edge_fact_ids.add(edge.source_id)
+                edge_fact_ids.add(edge.target_id)
+
+    # --- Filter facts ---
+    matched_facts: list[Fact] = []
+    for fact in graph.nodes.values():
+        # Confidence gate
+        if fact.confidence < min_confidence:
+            continue
+        # Skip superseded
+        if fact.superseded_by is not None:
+            continue
+
+        include = True
+
+        # Category filter
+        if ent_upper is not None and fact.category.upper() != ent_upper:
+            include = False
+
+        # Metadata filter
+        if metadata_filter:
+            for k, v in metadata_filter.items():
+                if fact.metadata.get(k) != v:
+                    include = False
+                    break
+
+        # If relationship filter given, fact must be connected by matching edge
+        if rel_str is not None and fact.id not in edge_fact_ids:
+            include = False
+
+        if include:
+            matched_facts.append(fact)
+
+    # Sort by confidence desc
+    matched_facts.sort(key=lambda f: -(f.confidence or 0.0))
+    matched_facts = matched_facts[:max_results]
+
+    return PatternQueryResult(
+        facts=matched_facts,
+        edges=matched_edges,
+        match_count=len(matched_facts),
+    )

crp/ckf/pubsub.py

@@ -0,0 +1,128 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""PubSub event bus for CKF internal notifications (§3.8).
+
+Events: fact_created, fact_superseded, edge_added, community_updated, anomaly_detected.
+Subscribers receive typed payloads asynchronously (fire-and-forget).
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Event types
+# ---------------------------------------------------------------------------
+
+
+class CKFEventType(str, Enum):
+    """Events emitted by the CKF subsystem."""
+
+    FACT_CREATED = "fact_created"
+    FACT_SUPERSEDED = "fact_superseded"
+    EDGE_ADDED = "edge_added"
+    COMMUNITY_UPDATED = "community_updated"
+    ANOMALY_DETECTED = "anomaly_detected"
+
+
+@dataclass
+class CKFEvent:
+    """Payload for a CKF event."""
+
+    event_type: CKFEventType
+    data: dict[str, Any] = field(default_factory=dict)
+    source: str = ""
+
+
+# ---------------------------------------------------------------------------
+# Callback type
+# ---------------------------------------------------------------------------
+
+EventCallback = Callable[[CKFEvent], None]
+
+
+# ---------------------------------------------------------------------------
+# PubSubEventBus
+# ---------------------------------------------------------------------------
+
+
+class PubSubEventBus:
+    """Thread-safe pub/sub for CKF lifecycle events.
+
+    Usage::
+
+        bus = PubSubEventBus()
+        bus.subscribe(CKFEventType.FACT_CREATED, my_handler)
+        bus.publish(CKFEvent(CKFEventType.FACT_CREATED, {"fact_id": "abc"}))
+    """
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._subscribers: dict[CKFEventType, list[EventCallback]] = {
+            et: [] for et in CKFEventType
+        }
+        self._global: list[EventCallback] = []
+
+    # --- Subscribe ---
+
+    def subscribe(
+        self,
+        event_type: CKFEventType,
+        callback: EventCallback,
+    ) -> None:
+        """Register *callback* for events of *event_type*."""
+        with self._lock:
+            self._subscribers[event_type].append(callback)
+
+    def subscribe_all(self, callback: EventCallback) -> None:
+        """Register *callback* for all event types."""
+        with self._lock:
+            self._global.append(callback)
+
+    def unsubscribe(
+        self,
+        event_type: CKFEventType,
+        callback: EventCallback,
+    ) -> None:
+        """Remove *callback* from *event_type* subscribers."""
+        with self._lock:
+            subs = self._subscribers.get(event_type, [])
+            if callback in subs:
+                subs.remove(callback)
+
+    # --- Publish ---
+
+    def publish(self, event: CKFEvent) -> None:
+        """Dispatch *event* to all subscribers.  Fire-and-forget — errors logged."""
+        with self._lock:
+            callbacks = list(self._subscribers.get(event.event_type, []))
+            callbacks += list(self._global)
+
+        for cb in callbacks:
+            try:
+                cb(event)
+            except Exception:  # noqa: BLE001
+                logger.exception("CKF event handler error for %s", event.event_type)
+
+    # --- Introspection ---
+
+    def subscriber_count(self, event_type: CKFEventType | None = None) -> int:
+        """Return number of subscribers, optionally filtered by event type."""
+        with self._lock:
+            if event_type is None:
+                return sum(len(v) for v in self._subscribers.values()) + len(self._global)
+            return len(self._subscribers.get(event_type, [])) + len(self._global)
+
+    def clear(self) -> None:
+        """Remove all subscribers."""
+        with self._lock:
+            for et in self._subscribers:
+                self._subscribers[et] = []
+            self._global = []

crp/ckf/semantic.py

@@ -0,0 +1,207 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""CKF Mode 3: Semantic fallback — ANN similarity retrieval (§3.8).
+
+``semantic_fallback(query, facts, top_k)`` retrieves the *top_k* most
+semantically similar facts to *query*.  Uses HNSW when hnswlib is available,
+otherwise brute-force cosine similarity.
+
+Adaptive top_k: 20 (default), scales up to 200 for large stores.
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+from dataclasses import dataclass, field
+from typing import Any
+
+from crp.state.fact import StateFact
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# ANN backend detection
+# ---------------------------------------------------------------------------
+
+_HNSWLIB: Any = None
+_HNSWLIB_CHECKED = False
+
+
+def _check_hnswlib() -> Any:
+    global _HNSWLIB, _HNSWLIB_CHECKED  # noqa: PLW0603
+    if not _HNSWLIB_CHECKED:
+        try:
+            import hnswlib  # type: ignore[import-untyped]
+
+            _HNSWLIB = hnswlib
+        except ImportError:
+            _HNSWLIB = None
+        _HNSWLIB_CHECKED = True
+    return _HNSWLIB
+
+
+# ---------------------------------------------------------------------------
+# Result type
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class SemanticResult:
+    """Result of a semantic similarity query."""
+
+    facts: list[StateFact] = field(default_factory=list)
+    scores: dict[str, float] = field(default_factory=dict)
+    used_ann: bool = False
+
+
+# ---------------------------------------------------------------------------
+# Cosine similarity
+# ---------------------------------------------------------------------------
+
+
+def _cosine_sim(a: list[float], b: list[float]) -> float:
+    """Compute cosine similarity between two vectors."""
+    dot = sum(x * y for x, y in zip(a, b))
+    norm_a = math.sqrt(sum(x * x for x in a))
+    norm_b = math.sqrt(sum(x * x for x in b))
+    if norm_a < 1e-12 or norm_b < 1e-12:
+        return 0.0
+    return dot / (norm_a * norm_b)
+
+
+# ---------------------------------------------------------------------------
+# Adaptive top_k
+# ---------------------------------------------------------------------------
+
+MIN_TOP_K = 20
+MAX_TOP_K = 200
+ANN_THRESHOLD = 1000
+
+
+def adaptive_top_k(fact_count: int, base_k: int = MIN_TOP_K) -> int:
+    """Scale top_k based on store size: 20 for small, up to 200 for large."""
+    if fact_count <= 100:
+        return base_k
+    # Linear scale between 100 and 10_000 facts
+    scale = min(1.0, (fact_count - 100) / 9900)
+    return min(MAX_TOP_K, base_k + int(scale * (MAX_TOP_K - base_k)))
+
+
+# ---------------------------------------------------------------------------
+# HNSW index wrapper
+# ---------------------------------------------------------------------------
+
+
+class HNSWIndex:
+    """Thin wrapper around hnswlib for ANN queries."""
+
+    def __init__(self, dim: int, max_elements: int = 10_000) -> None:
+        lib = _check_hnswlib()
+        if lib is None:
+            raise RuntimeError("hnswlib not available")
+        self._lib = lib
+        self._dim = dim
+        self._max_elements = max_elements
+        self._index = lib.Index(space="cosine", dim=dim)
+        self._index.init_index(max_elements=max_elements, ef_construction=200, M=16)
+        self._index.set_ef(50)
+        self._id_map: dict[int, str] = {}
+        self._next_int: int = 0
+
+    def _resize_if_needed(self) -> None:
+        """Double capacity when nearing the limit."""
+        if self._next_int >= self._max_elements:
+            self._max_elements *= 2
+            self._index.resize_index(self._max_elements)
+
+    def add(self, fact_id: str, embedding: list[float]) -> None:
+        self._resize_if_needed()
+        int_id = self._next_int
+        self._id_map[int_id] = fact_id
+        self._index.add_items([embedding], [int_id])
+        self._next_int += 1
+
+    def query(self, embedding: list[float], k: int) -> list[tuple[str, float]]:
+        """Return [(fact_id, distance), ...] for top-k nearest."""
+        k = min(k, self._next_int)
+        if k == 0:
+            return []
+        labels, distances = self._index.knn_query([embedding], k=k)
+        results = []
+        for label, dist in zip(labels[0], distances[0]):
+            fid = self._id_map.get(int(label))
+            if fid:
+                # hnswlib cosine returns 1 - cos_sim
+                results.append((fid, 1.0 - float(dist)))
+        return results
+
+    @property
+    def count(self) -> int:
+        return self._next_int
+
+
+# ---------------------------------------------------------------------------
+# Brute-force fallback
+# ---------------------------------------------------------------------------
+
+
+def _brute_force_query(
+    query_embedding: list[float],
+    facts: dict[str, StateFact],
+    top_k: int,
+) -> list[tuple[str, float]]:
+    """Brute-force cosine similarity search."""
+    scored: list[tuple[str, float]] = []
+    for fid, sf in facts.items():
+        emb = sf.embedding
+        if emb is not None:
+            sim = _cosine_sim(query_embedding, emb)
+            scored.append((fid, sim))
+    scored.sort(key=lambda x: -x[1])
+    return scored[:top_k]
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def semantic_fallback(
+    query_embedding: list[float],
+    facts: dict[str, StateFact],
+    top_k: int | None = None,
+    hnsw_index: HNSWIndex | None = None,
+) -> SemanticResult:
+    """Retrieve the *top_k* most similar facts to *query_embedding*.
+
+    Uses HNSW index if provided and store is large enough; otherwise brute-force.
+    Adaptive top_k: 20→200 based on store size.
+    """
+    if not query_embedding or not facts:
+        return SemanticResult()
+
+    k = top_k if top_k is not None else adaptive_top_k(len(facts))
+    used_ann = False
+
+    # Try HNSW if index provided and large enough
+    if hnsw_index is not None and hnsw_index.count >= ANN_THRESHOLD:
+        try:
+            raw = hnsw_index.query(query_embedding, k)
+            used_ann = True
+        except Exception:  # noqa: BLE001
+            logger.warning("HNSW query failed, falling back to brute-force")
+            raw = _brute_force_query(query_embedding, facts, k)
+    else:
+        raw = _brute_force_query(query_embedding, facts, k)
+
+    # Build result
+    result_facts: list[StateFact] = []
+    scores: dict[str, float] = {}
+    for fid, sim in raw:
+        sf = facts.get(fid)
+        if sf and not sf.is_superseded:
+            result_facts.append(sf)
+            scores[fid] = sim
+
+    return SemanticResult(facts=result_facts, scores=scores, used_ann=used_ann)

crp/cli/__init__.py

@@ -0,0 +1,7 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""CRP CLI — crp init, dispatch, ingest, status, preview, serve."""
+
+from crp.cli.startup import StartupResult, StartupStep, run_startup
+
+__all__ = ["StartupResult", "StartupStep", "run_startup"]