crprotocol 2.0.0__py3-none-any.whl

crp/advanced/curator.py

@@ -0,0 +1,265 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""LLM context curation — progressive understanding synthesis (§18).
+
+Periodically dispatches curation windows to build an evolving synthesis
+of findings, relationships, and gaps. Injected into envelopes as
+Section 1.5 between CRITICAL STATE and DISCOVERIES.
+"""
+
+from __future__ import annotations
+
+import time
+import uuid
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+# Curation interval by tier: {tier: (interval, max_tokens)}
+TIER_CONFIG: dict[str, tuple[int, int]] = {
+    "A": (5, 500),
+    "B": (5, 800),
+    "C": (10, 1000),
+    "D": (20, 1500),
+}
+
+
+# ---------------------------------------------------------------------------
+# Data types
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class LLMSynthesis:
+    """Curated synthesis from LLM review of accumulated facts."""
+
+    synthesis_id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    text: str = ""
+    window_index: int = 0
+    supersedes: str | None = None
+    evolution_count: int = 1
+    critical_findings: list[str] | None = None
+    key_relationships: list[str] | None = None
+    gaps: list[str] | None = None
+    confidence: float = 1.0
+    created_at: float = field(default_factory=time.time)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "synthesis_id": self.synthesis_id,
+            "text": self.text,
+            "window_index": self.window_index,
+            "supersedes": self.supersedes,
+            "evolution_count": self.evolution_count,
+            "critical_findings": self.critical_findings,
+            "key_relationships": self.key_relationships,
+            "gaps": self.gaps,
+            "confidence": self.confidence,
+            "created_at": self.created_at,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> LLMSynthesis:
+        return cls(
+            synthesis_id=data.get("synthesis_id", str(uuid.uuid4())),
+            text=data.get("text", ""),
+            window_index=data.get("window_index", 0),
+            supersedes=data.get("supersedes"),
+            evolution_count=data.get("evolution_count", 1),
+            critical_findings=data.get("critical_findings"),
+            key_relationships=data.get("key_relationships"),
+            gaps=data.get("gaps"),
+            confidence=data.get("confidence", 1.0),
+            created_at=data.get("created_at", 0.0),
+        )
+
+
+@dataclass
+class CurationConfig:
+    """Configuration for LLM curation."""
+
+    enabled: bool = True
+    curation_interval: int = 5
+    max_synthesis_tokens: int = 1500
+    progressive: bool = True
+    quality_tier: str = "B"
+
+
+# ---------------------------------------------------------------------------
+# LLMContextCurator
+# ---------------------------------------------------------------------------
+
+
+class LLMContextCurator:
+    """LLM-driven context curation with progressive understanding."""
+
+    def __init__(
+        self,
+        dispatch_fn: Callable[[str, str], tuple[str, Any]] | None = None,
+        config: CurationConfig | None = None,
+    ) -> None:
+        self._dispatch_fn = dispatch_fn
+        self.config = config or CurationConfig()
+        self._current_synthesis: LLMSynthesis | None = None
+        self._synthesis_history: list[LLMSynthesis] = []
+
+    @property
+    def current_synthesis(self) -> LLMSynthesis | None:
+        return self._current_synthesis
+
+    @property
+    def evolution_count(self) -> int:
+        if self._current_synthesis:
+            return self._current_synthesis.evolution_count
+        return 0
+
+    def should_curate(self, window_index: int) -> bool:
+        """Check if curation should run at this window."""
+        if not self.config.enabled:
+            return False
+        interval, _ = TIER_CONFIG.get(
+            self.config.quality_tier,
+            (self.config.curation_interval, self.config.max_synthesis_tokens),
+        )
+        return window_index > 0 and window_index % interval == 0
+
+    def curate(
+        self,
+        window_index: int,
+        top_facts: list[str],
+        recent_output_summary: str = "",
+    ) -> LLMSynthesis | None:
+        """Run curation (initial or progressive).
+
+        Returns new synthesis or None if dispatch unavailable.
+        """
+        if not self._dispatch_fn:
+            return None
+
+        if self.config.progressive and self._current_synthesis:
+            return self._progressive_curation(
+                window_index, top_facts, recent_output_summary,
+            )
+        return self._initial_curation(window_index, top_facts, recent_output_summary)
+
+    def _initial_curation(
+        self,
+        window_index: int,
+        top_facts: list[str],
+        recent_output_summary: str,
+    ) -> LLMSynthesis:
+        """First curation — no prior synthesis to build on."""
+        facts_text = "\n".join(f"- {f}" for f in top_facts[:40])
+        prompt = (
+            "Analyze the extracted facts and provide:\n"
+            "1. 5 most critical findings\n"
+            "2. 3 key relationships between findings\n"
+            "3. Current assessment\n"
+            "4. What's missing / gaps\n\n"
+            f"Recent output:\n{recent_output_summary[:1000]}\n\n"
+            "Be concise."
+        )
+
+        output, _ = self._dispatch_fn(prompt, facts_text)  # type: ignore[misc]
+        synthesis = self._parse_synthesis(output, window_index)
+        self._current_synthesis = synthesis
+        self._synthesis_history.append(synthesis)
+        return synthesis
+
+    def _progressive_curation(
+        self,
+        window_index: int,
+        top_facts: list[str],
+        recent_output_summary: str,
+    ) -> LLMSynthesis:
+        """Progressive curation — revise previous synthesis."""
+        prev = self._current_synthesis
+        facts_text = "\n".join(f"- {f}" for f in top_facts[:40])
+        prompt = (
+            "Revise your previous synthesis based on new facts.\n"
+            f"Previous synthesis:\n{prev.text[:1500] if prev else ''}\n\n"
+            "Update:\n"
+            "1. Revised critical findings\n"
+            "2. Updated relationships\n"
+            "3. Updated assessment\n"
+            "4. New gaps identified\n\n"
+            f"New facts since last synthesis:\n{facts_text}\n\n"
+            f"Recent output:\n{recent_output_summary[:500]}\n\n"
+            "Be concise."
+        )
+
+        output, _ = self._dispatch_fn(prompt, "")  # type: ignore[misc]
+        synthesis = self._parse_synthesis(output, window_index)
+        synthesis.supersedes = prev.synthesis_id if prev else None
+        synthesis.evolution_count = (prev.evolution_count + 1) if prev else 1
+        self._current_synthesis = synthesis
+        self._synthesis_history.append(synthesis)
+        return synthesis
+
+    def _parse_synthesis(self, output: str, window_index: int) -> LLMSynthesis:
+        """Parse curation output into structured synthesis."""
+        findings: list[str] = []
+        relationships: list[str] = []
+        gaps: list[str] = []
+
+        section = ""
+        for line in output.split("\n"):
+            line_lower = line.lower().strip()
+            if "finding" in line_lower or "critical" in line_lower:
+                section = "findings"
+            elif "relationship" in line_lower:
+                section = "relationships"
+            elif "gap" in line_lower or "missing" in line_lower:
+                section = "gaps"
+            elif "assessment" in line_lower:
+                section = "assessment"
+
+            if line.strip().startswith("-") or line.strip().startswith("•"):
+                item = line.strip().lstrip("-•").strip()
+                if section == "findings":
+                    findings.append(item)
+                elif section == "relationships":
+                    relationships.append(item)
+                elif section == "gaps":
+                    gaps.append(item)
+
+        return LLMSynthesis(
+            text=output,
+            window_index=window_index,
+            critical_findings=findings or None,
+            key_relationships=relationships or None,
+            gaps=gaps or None,
+        )
+
+    def format_for_envelope(self) -> str:
+        """Format current synthesis for envelope injection (Section 1.5)."""
+        if not self._current_synthesis:
+            return ""
+        s = self._current_synthesis
+        parts = [
+            f"[LLM_SYNTHESIS (Window {s.window_index}, evolution {s.evolution_count})]",
+        ]
+        if s.critical_findings:
+            parts.append("CRITICAL FINDINGS: " + "; ".join(s.critical_findings))
+        if s.key_relationships:
+            parts.append("KEY RELATIONSHIPS: " + "; ".join(s.key_relationships))
+        if s.gaps:
+            parts.append("GAPS: " + "; ".join(s.gaps))
+        return "\n".join(parts)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "current": self._current_synthesis.to_dict() if self._current_synthesis else None,
+            "history": [s.to_dict() for s in self._synthesis_history],
+            "config": {
+                "enabled": self.config.enabled,
+                "curation_interval": self.config.curation_interval,
+                "max_synthesis_tokens": self.config.max_synthesis_tokens,
+                "progressive": self.config.progressive,
+                "quality_tier": self.config.quality_tier,
+            },
+        }

crp/advanced/feedback.py

@@ -0,0 +1,146 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Human-in-the-loop feedback — fact override, confidence adjustment (§18, MAY)."""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class FeedbackEntry:
+    """Single human feedback action."""
+
+    feedback_id: str = ""
+    fact_id: str = ""
+    action: str = ""  # "override" | "boost" | "penalize" | "reject"
+    original_text: str = ""
+    corrected_text: str | None = None
+    confidence_delta: float = 0.0
+    reason: str = ""
+    timestamp: float = field(default_factory=time.time)
+    applied: bool = False
+
+
+class FeedbackLoop:
+    """Human-in-the-loop corrections for facts in warm state."""
+
+    def __init__(self) -> None:
+        self._entries: list[FeedbackEntry] = []
+        self._fact_adjustments: dict[str, float] = {}  # fact_id → cumulative delta
+
+    @property
+    def entry_count(self) -> int:
+        return len(self._entries)
+
+    def override_fact(
+        self,
+        fact_id: str,
+        corrected_text: str,
+        reason: str = "",
+    ) -> FeedbackEntry:
+        """Replace fact text with human-provided correction."""
+        entry = FeedbackEntry(
+            feedback_id=f"fb-{len(self._entries)}",
+            fact_id=fact_id,
+            action="override",
+            corrected_text=corrected_text,
+            reason=reason,
+            applied=True,
+        )
+        self._entries.append(entry)
+        return entry
+
+    def boost_confidence(
+        self,
+        fact_id: str,
+        delta: float = 0.1,
+        reason: str = "",
+    ) -> FeedbackEntry:
+        """Increase fact confidence based on human validation."""
+        self._fact_adjustments[fact_id] = (
+            self._fact_adjustments.get(fact_id, 0.0) + delta
+        )
+        entry = FeedbackEntry(
+            feedback_id=f"fb-{len(self._entries)}",
+            fact_id=fact_id,
+            action="boost",
+            confidence_delta=delta,
+            reason=reason,
+            applied=True,
+        )
+        self._entries.append(entry)
+        return entry
+
+    def penalize_confidence(
+        self,
+        fact_id: str,
+        delta: float = -0.2,
+        reason: str = "",
+    ) -> FeedbackEntry:
+        """Decrease fact confidence based on human rejection."""
+        self._fact_adjustments[fact_id] = (
+            self._fact_adjustments.get(fact_id, 0.0) + delta
+        )
+        entry = FeedbackEntry(
+            feedback_id=f"fb-{len(self._entries)}",
+            fact_id=fact_id,
+            action="penalize",
+            confidence_delta=delta,
+            reason=reason,
+            applied=True,
+        )
+        self._entries.append(entry)
+        return entry
+
+    def reject_fact(
+        self,
+        fact_id: str,
+        reason: str = "",
+    ) -> FeedbackEntry:
+        """Mark fact as rejected (confidence → 0)."""
+        self._fact_adjustments[fact_id] = -1.0  # Signal full rejection
+        entry = FeedbackEntry(
+            feedback_id=f"fb-{len(self._entries)}",
+            fact_id=fact_id,
+            action="reject",
+            confidence_delta=-1.0,
+            reason=reason,
+            applied=True,
+        )
+        self._entries.append(entry)
+        return entry
+
+    def get_adjusted_confidence(
+        self,
+        fact_id: str,
+        base_confidence: float,
+    ) -> float:
+        """Get confidence after applying all feedback adjustments."""
+        delta = self._fact_adjustments.get(fact_id, 0.0)
+        if delta <= -1.0:
+            return 0.0
+        return max(0.0, min(1.0, base_confidence + delta))
+
+    def get_entries_for_fact(self, fact_id: str) -> list[FeedbackEntry]:
+        return [e for e in self._entries if e.fact_id == fact_id]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "entries": [
+                {
+                    "feedback_id": e.feedback_id,
+                    "fact_id": e.fact_id,
+                    "action": e.action,
+                    "corrected_text": e.corrected_text,
+                    "confidence_delta": e.confidence_delta,
+                    "reason": e.reason,
+                    "timestamp": e.timestamp,
+                    "applied": e.applied,
+                }
+                for e in self._entries
+            ],
+            "adjustments": dict(self._fact_adjustments),
+        }

crp/advanced/hierarchical.py

@@ -0,0 +1,211 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Hierarchical processing — map-reduce-validate for Tier C/D inputs (§4.5, §11).
+
+Splits massive inputs into segments, processes each independently,
+reduces iteratively, and validates cross-window consistency.
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+DEFAULT_SEGMENT_SIZE_MULTIPLIER = 100  # segment_size = 100 × context_window
+DEFAULT_FAN_IN = 50
+
+
+# ---------------------------------------------------------------------------
+# Data types
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class HierarchicalPlan:
+    """Plan for hierarchical processing."""
+
+    total_tokens: int = 0
+    segment_count: int = 0
+    segment_size: int = 0
+    fan_in: int = DEFAULT_FAN_IN
+    hierarchy_levels: int = 1
+    estimated_degradation: float = 0.0
+    processing_mode: str = "hierarchical"
+
+
+@dataclass
+class HierarchicalConfig:
+    """Configuration for hierarchical processing."""
+
+    segment_size: int | None = None
+    fan_in: int | None = None
+    context_window: int = 128_000
+
+
+@dataclass
+class SegmentResult:
+    """Output of processing one segment."""
+
+    segment_index: int = 0
+    synthesis: str = ""
+    facts_extracted: int = 0
+    token_count: int = 0
+
+
+# ---------------------------------------------------------------------------
+# Degradation model
+# ---------------------------------------------------------------------------
+
+
+def chain_degradation(levels: int, per_level: float = 0.03) -> float:
+    """Compute effective degradation after N hierarchy levels.
+
+    d_chain(L) = 1 - (1 - per_level)^L
+    """
+    return 1.0 - (1.0 - per_level) ** levels
+
+
+def effective_context(
+    context_window: int, levels: int, per_level: float = 0.03,
+) -> float:
+    """Effective context capacity after hierarchical degradation.
+
+    EffCtx_hier(N) = C × (1 - d_chain(⌈log_k(N)⌉))
+    """
+    d = chain_degradation(levels, per_level)
+    return context_window * (1.0 - d)
+
+
+# ---------------------------------------------------------------------------
+# HierarchicalProcessor
+# ---------------------------------------------------------------------------
+
+
+class HierarchicalProcessor:
+    """Map-reduce-validate pattern for oversized inputs."""
+
+    def __init__(
+        self,
+        dispatch_fn: Callable[[str, str], tuple[str, Any]] | None = None,
+        count_tokens: Callable[[str], int] | None = None,
+        context_window: int = 128_000,
+    ) -> None:
+        self._dispatch_fn = dispatch_fn
+        self._count_tokens = count_tokens or (lambda t: len(t) // 4)
+        self._context_window = context_window
+
+    def plan(
+        self, total_tokens: int, config: HierarchicalConfig | None = None,
+    ) -> HierarchicalPlan:
+        """Create a hierarchical processing plan."""
+        cfg = config or HierarchicalConfig()
+        seg_size = cfg.segment_size or (DEFAULT_SEGMENT_SIZE_MULTIPLIER * self._context_window)
+        fan_in = cfg.fan_in or DEFAULT_FAN_IN
+
+        segment_count = max(1, math.ceil(total_tokens / seg_size))
+        levels = max(1, math.ceil(math.log(max(segment_count, 2)) / math.log(max(fan_in, 2))))
+        degradation = chain_degradation(levels)
+
+        mode = "hierarchical"
+        if total_tokens > 1000 * self._context_window:
+            mode = "hierarchical_multi_level"
+
+        return HierarchicalPlan(
+            total_tokens=total_tokens,
+            segment_count=segment_count,
+            segment_size=seg_size,
+            fan_in=fan_in,
+            hierarchy_levels=levels,
+            estimated_degradation=degradation,
+            processing_mode=mode,
+        )
+
+    def map_phase(
+        self,
+        segments: list[str],
+        task_intent: str,
+    ) -> list[SegmentResult]:
+        """MAP: Process each segment independently."""
+        results: list[SegmentResult] = []
+        for i, segment in enumerate(segments):
+            if self._dispatch_fn:
+                prompt = (
+                    "Summarize and extract ALL key facts from the following "
+                    f"segment ({i + 1}/{len(segments)}) for: {task_intent}"
+                )
+                output, _ = self._dispatch_fn(prompt, segment)
+            else:
+                # Fallback: take first 500 chars as summary
+                output = segment[:500]
+
+            results.append(SegmentResult(
+                segment_index=i,
+                synthesis=output,
+                facts_extracted=0,
+                token_count=self._count_tokens(output),
+            ))
+        return results
+
+    def reduce_phase(
+        self,
+        syntheses: list[str],
+        task_intent: str,
+        fan_in: int = DEFAULT_FAN_IN,
+    ) -> list[str]:
+        """REDUCE: Iteratively merge syntheses until ≤ fan_in remain."""
+        current = syntheses
+        while len(current) > fan_in:
+            batches: list[list[str]] = []
+            for i in range(0, len(current), fan_in):
+                batches.append(current[i:i + fan_in])
+
+            next_level: list[str] = []
+            for batch in batches:
+                joined = "\n\n---\n\n".join(batch)
+                if self._dispatch_fn:
+                    prompt = (
+                        f"Synthesize these {len(batch)} segment summaries "
+                        f"into a coherent overview for: {task_intent}"
+                    )
+                    output, _ = self._dispatch_fn(prompt, joined)
+                else:
+                    output = joined[:1000]
+                next_level.append(output)
+            current = next_level
+        return current
+
+    def hierarchical_dispatch(
+        self,
+        task_intent: str,
+        large_input: str,
+        config: HierarchicalConfig | None = None,
+    ) -> tuple[list[str], HierarchicalPlan]:
+        """Full map-reduce-validate pipeline for oversized input.
+
+        Returns (final_syntheses, plan).
+        """
+        total_tokens = self._count_tokens(large_input)
+        plan = self.plan(total_tokens, config)
+
+        # Segment the input
+        seg_char_size = len(large_input) // max(plan.segment_count, 1)
+        segments: list[str] = []
+        for i in range(plan.segment_count):
+            start = i * seg_char_size
+            end = start + seg_char_size if i < plan.segment_count - 1 else len(large_input)
+            segments.append(large_input[start:end])
+
+        # MAP
+        map_results = self.map_phase(segments, task_intent)
+
+        # REDUCE
+        syntheses = [r.synthesis for r in map_results]
+        reduced = self.reduce_phase(syntheses, task_intent, plan.fan_in)
+
+        return reduced, plan