crprotocol 2.0.0__py3-none-any.whl

crp/state/cold_storage.py

@@ -0,0 +1,258 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Cold storage — Tier 3 cross-session persistence (§3.6).
+
+Persists facts, graph structure, community IDs, and metadata to disk.
+PersistedStateHeader provides schema versioning + integrity checksums.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from .event_log import FactEventLog
+from .warm_store import WarmStateStore
+
+# ---------------------------------------------------------------------------
+# Persisted state header
+# ---------------------------------------------------------------------------
+
+COLD_SCHEMA_VERSION = 1
+
+
+@dataclass
+class PersistedStateHeader:
+    """Schema-versioned header for cold storage files (§3.6)."""
+
+    schema_version: int = COLD_SCHEMA_VERSION
+    created_at: float = field(default_factory=time.time)
+    session_id: str = ""
+    fact_count: int = 0
+    edge_count: int = 0
+    window_count: int = 0
+    checksum: str = ""  # BLAKE3 or SHA-256 of payload
+    hmac_signature: str = ""  # HMAC chain signature (§4G.3)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "created_at": self.created_at,
+            "session_id": self.session_id,
+            "fact_count": self.fact_count,
+            "edge_count": self.edge_count,
+            "window_count": self.window_count,
+            "checksum": self.checksum,
+            "hmac_signature": self.hmac_signature,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> PersistedStateHeader:
+        return cls(
+            schema_version=data.get("schema_version", COLD_SCHEMA_VERSION),
+            created_at=data.get("created_at", 0.0),
+            session_id=data.get("session_id", ""),
+            fact_count=data.get("fact_count", 0),
+            edge_count=data.get("edge_count", 0),
+            window_count=data.get("window_count", 0),
+            checksum=data.get("checksum", ""),
+            hmac_signature=data.get("hmac_signature", ""),
+        )
+
+
+# ---------------------------------------------------------------------------
+# Cold storage operations
+# ---------------------------------------------------------------------------
+
+
+def _compute_checksum(payload: str) -> str:
+    """Compute integrity checksum (BLAKE3 preferred, SHA-256 fallback)."""
+    try:
+        import blake3  # type: ignore[import-untyped]
+
+        return blake3.blake3(payload.encode()).hexdigest()
+    except ImportError:
+        return hashlib.sha256(payload.encode()).hexdigest()
+
+
+def persist_to_cold(
+    warm_store: WarmStateStore,
+    event_log: FactEventLog,
+    path: str | Path,
+    session_id: str = "",
+    community_map: dict[str, int] | None = None,
+    hmac_key: bytes | None = None,
+) -> PersistedStateHeader:
+    """Persist warm store + event log to cold storage.
+
+    File format: JSON with header + payload.
+    Includes community IDs (§4E.1) and HMAC chain (§4G.3) when provided.
+    Uses atomic writes to prevent partial-write corruption (§4G.4).
+    """
+    store_data = warm_store.to_dict()
+    event_data = event_log.to_list()
+
+    payload: dict[str, Any] = {
+        "warm_store": store_data,
+        "event_log": event_data,
+    }
+
+    # Include community IDs if available (§4E.1)
+    if community_map:
+        payload["community_map"] = community_map
+
+    payload_str = json.dumps(payload, sort_keys=True, default=str)
+
+    header = PersistedStateHeader(
+        session_id=session_id,
+        fact_count=warm_store.fact_count,
+        edge_count=len(warm_store.graph.edges),
+        window_count=warm_store.window_count,
+        checksum=_compute_checksum(payload_str),
+    )
+
+    # Compute HMAC chain signature if key provided (§4G.3)
+    if hmac_key:
+        import hmac as _hmac
+        header.hmac_signature = _hmac.new(
+            hmac_key, payload_str.encode(), hashlib.sha256
+        ).hexdigest()
+
+    cold_data = {
+        "header": header.to_dict(),
+        "payload": payload,
+    }
+
+    p = Path(path)
+    p.parent.mkdir(parents=True, exist_ok=True)
+
+    # Atomic write: write to temp, then rename (§4G.4)
+    tmp = p.with_suffix(".tmp")
+    try:
+        tmp.write_text(json.dumps(cold_data, default=str), encoding="utf-8")
+        # os.replace is atomic on POSIX; best-effort on Windows
+        os.replace(str(tmp), str(p))
+    except BaseException:
+        # Clean up partial temp file on failure
+        tmp.unlink(missing_ok=True)
+        raise
+
+    return header
+
+
+def restore_from_cold(
+    warm_store: WarmStateStore,
+    event_log: FactEventLog,
+    path: str | Path,
+    hmac_key: bytes | None = None,
+) -> tuple[PersistedStateHeader, list[str], dict[str, int]]:
+    """Load cold storage into warm store + event log.
+
+    Returns (header, warnings, community_map) where:
+    - warnings list integrity issues
+    - community_map has restored community assignments (§4E.1)
+
+    Performs HMAC chain verification when key provided (§4G.3).
+    Handles partial-write corruption (§4G.4).
+    Applies schema migrations (§4G.2).
+    """
+    p = Path(path)
+    warnings: list[str] = []
+    community_map: dict[str, int] = {}
+
+    if not p.exists():
+        # Check for temp file from interrupted atomic write (§4G.4)
+        tmp = p.with_suffix(".tmp")
+        if tmp.exists():
+            warnings.append("Found interrupted write (.tmp file); attempting recovery")
+            try:
+                cold_data = json.loads(tmp.read_text(encoding="utf-8"))
+                warnings.append("Recovered from .tmp file")
+            except (json.JSONDecodeError, OSError):
+                warnings.append(f"Cold storage file not found and .tmp is corrupt: {path}")
+                return PersistedStateHeader(), warnings, community_map
+        else:
+            warnings.append(f"Cold storage file not found: {path}")
+            return PersistedStateHeader(), warnings, community_map
+    else:
+        try:
+            cold_data = json.loads(p.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError) as exc:
+            warnings.append(f"Failed to read cold storage: {exc}")
+            return PersistedStateHeader(), warnings, community_map
+
+    header = PersistedStateHeader.from_dict(cold_data.get("header", {}))
+    payload = cold_data.get("payload", {})
+
+    # Schema migration (§4G.2)
+    if header.schema_version < COLD_SCHEMA_VERSION:
+        warnings.append(
+            f"Migrating schema v{header.schema_version} → v{COLD_SCHEMA_VERSION}"
+        )
+        payload = _migrate_payload(payload, header.schema_version, COLD_SCHEMA_VERSION)
+    elif header.schema_version > COLD_SCHEMA_VERSION:
+        warnings.append(
+            f"Schema version {header.schema_version} is newer than current "
+            f"{COLD_SCHEMA_VERSION} — forward compat may lose data"
+        )
+
+    # Verify checksum
+    payload_str = json.dumps(payload, sort_keys=True, default=str)
+    expected_checksum = _compute_checksum(payload_str)
+    integrity_failed = False
+    if header.checksum and header.checksum != expected_checksum:
+        warnings.append("Checksum verification FAILED — data may be corrupted")
+        integrity_failed = True
+
+    # Verify HMAC chain (§4G.3)
+    if hmac_key and header.hmac_signature:
+        import hmac as _hmac
+        expected_hmac = _hmac.new(
+            hmac_key, payload_str.encode(), hashlib.sha256
+        ).hexdigest()
+        if not _hmac.compare_digest(header.hmac_signature, expected_hmac):
+            warnings.append("HMAC chain verification FAILED — data may be tampered")
+            integrity_failed = True
+    elif hmac_key and not header.hmac_signature:
+        warnings.append("HMAC key provided but no HMAC signature in stored data")
+
+    # SECURITY: fail-closed — reject corrupted/tampered data (§audit5 SEC-H1)
+    if integrity_failed:
+        warnings.append("REJECTED: data not loaded due to integrity failure")
+        return header, warnings, community_map
+
+    # Load warm store
+    warm_store.load_from_dict(payload.get("warm_store", {}))
+
+    # Load event log
+    event_log.load_from_list(payload.get("event_log", []))
+
+    # Restore community IDs (§4E.1)
+    if "community_map" in payload:
+        community_map = payload["community_map"]
+
+    # Verify counts
+    if header.fact_count != warm_store.fact_count:
+        warnings.append(
+            f"Fact count mismatch: header={header.fact_count}, "
+            f"loaded={warm_store.fact_count}"
+        )
+
+    return header, warnings, community_map
+
+
+def _migrate_payload(
+    payload: dict[str, Any],
+    from_version: int,
+    to_version: int,
+) -> dict[str, Any]:
+    """Apply schema migrations sequentially (§4G.2)."""
+    # Currently only v1 exists. Future migrations will be added here:
+    # if from_version < 2:
+    #     payload = _migrate_v1_to_v2(payload)
+    return payload

crp/state/compaction.py

@@ -0,0 +1,263 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Compaction engine — deduplicate and summarize warm state (§3.6).
+
+Trigger: fact_count > 5000 OR envelope_latency > 500ms.
+Algorithm:
+  1. Archive superseded facts
+  2. Cluster remaining by cosine similarity (>0.80 threshold)
+  3. TextRank summarize clusters
+  4. Rebuild ANN index
+  5. Compact graph
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+
+from crp.extraction.types import Fact
+
+from .event_log import FactEventLog
+from .fact import StateFact
+from .warm_store import WarmStateStore
+
+# ---------------------------------------------------------------------------
+# Compaction config
+# ---------------------------------------------------------------------------
+
+COMPACTION_FACT_THRESHOLD = 5000
+COMPACTION_LATENCY_THRESHOLD_MS = 500.0
+CLUSTER_SIMILARITY_THRESHOLD = 0.80
+
+
+@dataclass
+class CompactionConfig:
+    """Tuneable compaction parameters."""
+
+    fact_threshold: int = COMPACTION_FACT_THRESHOLD
+    latency_threshold_ms: float = COMPACTION_LATENCY_THRESHOLD_MS
+    cluster_sim_threshold: float = CLUSTER_SIMILARITY_THRESHOLD
+    max_cluster_size: int = 20
+
+
+@dataclass
+class CompactionResult:
+    """Result of a compaction pass."""
+
+    facts_before: int = 0
+    facts_after: int = 0
+    superseded_archived: int = 0
+    clusters_found: int = 0
+    summaries_created: int = 0
+    elapsed_ms: float = 0.0
+
+
+# ---------------------------------------------------------------------------
+# Simple cosine similarity (no ML deps)
+# ---------------------------------------------------------------------------
+
+
+def _cosine_sim_texts(a: str, b: str) -> float:
+    """Word-overlap cosine similarity between two texts."""
+    words_a = set(a.lower().split())
+    words_b = set(b.lower().split())
+    if not words_a or not words_b:
+        return 0.0
+    intersection = words_a & words_b
+    return len(intersection) / (math.sqrt(len(words_a)) * math.sqrt(len(words_b)))
+
+
+# ---------------------------------------------------------------------------
+# Clustering
+# ---------------------------------------------------------------------------
+
+
+def _cluster_facts(
+    facts: list[StateFact],
+    threshold: float,
+    max_cluster_size: int,
+) -> list[list[StateFact]]:
+    """Simple greedy clustering by pairwise cosine similarity.
+
+    When ML embeddings are available, uses them.  Fallback: word overlap.
+    """
+    clusters: list[list[StateFact]] = []
+    assigned: set[str] = set()
+
+    for sf in facts:
+        if sf.id in assigned:
+            continue
+        cluster = [sf]
+        assigned.add(sf.id)
+
+        for other in facts:
+            if other.id in assigned:
+                continue
+            if len(cluster) >= max_cluster_size:
+                break
+
+            # Use embeddings if both have them
+            if sf.has_embedding() and other.has_embedding():
+                emb_a = sf.embedding
+                emb_b = other.embedding
+                if emb_a and emb_b:
+                    dot = sum(x * y for x, y in zip(emb_a, emb_b))
+                    na = math.sqrt(sum(x * x for x in emb_a)) or 1e-12
+                    nb = math.sqrt(sum(x * x for x in emb_b)) or 1e-12
+                    sim = dot / (na * nb)
+                else:
+                    sim = _cosine_sim_texts(sf.text, other.text)
+            else:
+                sim = _cosine_sim_texts(sf.text, other.text)
+
+            if sim >= threshold:
+                cluster.append(other)
+                assigned.add(other.id)
+
+        clusters.append(cluster)
+
+    return clusters
+
+
+# ---------------------------------------------------------------------------
+# TextRank-inspired summarization (single-fact reduction)
+# ---------------------------------------------------------------------------
+
+
+def _summarize_cluster(cluster: list[StateFact]) -> str:
+    """Pick the most representative fact from a cluster.
+
+    Uses a simple centrality heuristic: the fact with the highest average
+    similarity to all others.  When ML embeddings are unavailable, uses the
+    highest-confidence fact.
+    """
+    if len(cluster) == 1:
+        return cluster[0].text
+
+    # Try embedding-based centrality
+    has_all_emb = all(sf.has_embedding() for sf in cluster)
+    if has_all_emb:
+        best_idx = 0
+        best_score = -1.0
+        for i, sf in enumerate(cluster):
+            emb_i = sf.embedding
+            if not emb_i:
+                continue
+            total_sim = 0.0
+            for j, other in enumerate(cluster):
+                if i == j:
+                    continue
+                emb_j = other.embedding
+                if emb_j:
+                    dot = sum(x * y for x, y in zip(emb_i, emb_j))
+                    na = math.sqrt(sum(x * x for x in emb_i)) or 1e-12
+                    nb = math.sqrt(sum(x * x for x in emb_j)) or 1e-12
+                    total_sim += dot / (na * nb)
+            if total_sim > best_score:
+                best_score = total_sim
+                best_idx = i
+        return cluster[best_idx].text
+
+    # Fallback: highest confidence
+    return max(cluster, key=lambda sf: sf.confidence).text
+
+
+# ---------------------------------------------------------------------------
+# Compaction engine
+# ---------------------------------------------------------------------------
+
+
+def should_compact(
+    store: WarmStateStore,
+    last_envelope_latency_ms: float = 0.0,
+    config: CompactionConfig | None = None,
+) -> bool:
+    """Check if compaction should run."""
+    cfg = config or CompactionConfig()
+    if store.fact_count > cfg.fact_threshold:
+        return True
+    return last_envelope_latency_ms > cfg.latency_threshold_ms
+
+
+def compact(
+    store: WarmStateStore,
+    event_log: FactEventLog,
+    window_id: str,
+    config: CompactionConfig | None = None,
+) -> CompactionResult:
+    """Run a compaction pass on the warm store.
+
+    1. Archive superseded facts → cold (emit ARCHIVED events)
+    2. Cluster remaining active facts by similarity
+    3. Summarize multi-fact clusters → single representative fact
+    4. Update graph with summary facts
+    """
+    import time as _time
+    import uuid
+
+    cfg = config or CompactionConfig()
+    t0 = _time.perf_counter()
+    result = CompactionResult(facts_before=store.fact_count)
+
+    # Step 1: Archive superseded
+    all_facts = store.get_facts(include_superseded=True)
+    superseded = [sf for sf in all_facts if sf.is_superseded]
+    for sf in superseded:
+        store.remove_fact(sf.id)
+        event_log.record_archived(sf.id, window_id)
+        result.superseded_archived += 1
+
+    # Step 2: Cluster active facts
+    active = store.get_facts()
+    if len(active) <= cfg.fact_threshold // 2:
+        # Not enough to bother clustering
+        result.facts_after = store.fact_count
+        result.elapsed_ms = (_time.perf_counter() - t0) * 1000
+        return result
+
+    clusters = _cluster_facts(active, cfg.cluster_sim_threshold, cfg.max_cluster_size)
+    result.clusters_found = len(clusters)
+
+    # Step 3: Summarize multi-fact clusters
+    for cluster in clusters:
+        if len(cluster) <= 1:
+            continue
+        summary_text = _summarize_cluster(cluster)
+        summary_fact = Fact(
+            id=str(uuid.uuid4()),
+            text=summary_text,
+            category="compacted_summary",
+            source_window_id=window_id,
+            confidence=max(sf.confidence for sf in cluster),
+            extraction_stage=0,
+        )
+
+        # Remove cluster members, add summary
+        for sf in cluster:
+            store.remove_fact(sf.id)
+            event_log.record_compaction(sf.id, window_id, summary_id=summary_fact.id)
+
+        store.add_facts([summary_fact])
+        event_log.record_fact_created(summary_fact, window_id)
+        result.summaries_created += 1
+
+    # Step 4: Clean up stale graph edges (§4D.5)
+    _cleanup_stale_edges(store)
+
+    result.facts_after = store.fact_count
+    result.elapsed_ms = (_time.perf_counter() - t0) * 1000
+    return result
+
+
+def _cleanup_stale_edges(store: WarmStateStore) -> int:
+    """Remove graph edges that reference non-existent facts (§4D.5)."""
+    active_ids = {sf.id for sf in store.get_facts(include_superseded=True)}
+    graph = store.graph
+    stale = [
+        e for e in graph.edges
+        if e.source_id not in active_ids or e.target_id not in active_ids
+    ]
+    for edge in stale:
+        graph.edges.remove(edge)
+    return len(stale)

crp/state/critical_state.py

@@ -0,0 +1,104 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Critical & structural state — always-included envelope sections (§3.1).
+
+CriticalState: goal, phase, blockers, constraints (Tier 0 — never evicted).
+StructuralState: continuation tracking for document position (§04 §3.5.2).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class CriticalState:
+    """Tier-0 critical state — ALWAYS included in every envelope (§3.1).
+
+    Tracks the task's fundamental parameters that must survive every window.
+    """
+
+    goal: str = ""
+    phase: str = ""
+    blockers: list[str] = field(default_factory=list)
+    constraints: list[str] = field(default_factory=list)
+    window_id: str = ""  # last window that updated this
+
+    def to_sections(self) -> dict[str, str]:
+        """Convert to envelope section dict for the formatter."""
+        sections: dict[str, str] = {}
+        if self.goal:
+            sections["GOAL"] = self.goal
+        if self.phase:
+            sections["PHASE"] = self.phase
+        if self.blockers:
+            sections["BLOCKER"] = "\n".join(f"- {b}" for b in self.blockers)
+        if self.constraints:
+            sections["CONSTRAINT"] = "\n".join(f"- {c}" for c in self.constraints)
+        return sections
+
+    def update(self, **kwargs: Any) -> None:
+        """Partial update of critical state fields."""
+        for key, value in kwargs.items():
+            if hasattr(self, key):
+                setattr(self, key, value)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "goal": self.goal,
+            "phase": self.phase,
+            "blockers": self.blockers,
+            "constraints": self.constraints,
+            "window_id": self.window_id,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> CriticalState:
+        return cls(
+            goal=data.get("goal", ""),
+            phase=data.get("phase", ""),
+            blockers=data.get("blockers", []),
+            constraints=data.get("constraints", []),
+            window_id=data.get("window_id", ""),
+        )
+
+
+@dataclass
+class StructuralState:
+    """Document structure tracking for continuation stitching (§04 §3.5.2).
+
+    Tracks where the LLM is in its output so continuation windows can resume
+    from the correct position.
+    """
+
+    current_section: str = ""
+    list_position: int = 0
+    open_blocks: list[str] = field(default_factory=list)  # e.g. ["```python", "- item"]
+    markdown_depth: int = 0  # heading level (1-6)
+    last_heading: str = ""
+    code_block_open: bool = False
+    code_language: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "current_section": self.current_section,
+            "list_position": self.list_position,
+            "open_blocks": self.open_blocks,
+            "markdown_depth": self.markdown_depth,
+            "last_heading": self.last_heading,
+            "code_block_open": self.code_block_open,
+            "code_language": self.code_language,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> StructuralState:
+        return cls(
+            current_section=data.get("current_section", ""),
+            list_position=data.get("list_position", 0),
+            open_blocks=data.get("open_blocks", []),
+            markdown_depth=data.get("markdown_depth", 0),
+            last_heading=data.get("last_heading", ""),
+            code_block_open=data.get("code_block_open", False),
+            code_language=data.get("code_language", ""),
+        )