crprotocol 2.0.0__py3-none-any.whl

crp/advanced/review_cycle.py

@@ -0,0 +1,329 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Review cycle management — active LLM review patterns (§14).
+
+Three interaction patterns:
+  1. Pre-generation planning (predict chain > 5 windows)
+  2. Checkpoint review (periodic, Tier 3 models only)
+  3. Post-generation self-assessment (quality scoring + targeted re-gen)
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Data types
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class ReviewGuidance:
+    """Output from a checkpoint review."""
+
+    on_track: bool = True
+    contradictions: list[str] = field(default_factory=list)
+    priorities: list[str] = field(default_factory=list)
+    new_gaps: list[str] = field(default_factory=list)
+    raw_output: str = ""
+
+
+@dataclass
+class AssessmentResult:
+    """Output from post-generation self-assessment."""
+
+    score: float = 0.0  # 0-10
+    issues: list[str] = field(default_factory=list)
+    needs_correction: bool = False
+    corrections_applied: int = 0
+    raw_output: str = ""
+
+
+@dataclass
+class PlannedSection:
+    """One section in the generation plan."""
+
+    title: str = ""
+    key_points: list[str] = field(default_factory=list)
+    dependencies: list[str] = field(default_factory=list)
+    estimated_tokens: int = 0
+
+
+@dataclass
+class DocumentPlan:
+    """Full generation plan from pre-generation planning."""
+
+    sections: list[PlannedSection] = field(default_factory=list)
+    total_estimated_tokens: int = 0
+    estimated_windows: int = 0
+
+
+# ---------------------------------------------------------------------------
+# ReviewCycleManager
+# ---------------------------------------------------------------------------
+
+
+class ReviewCycleManager:
+    """Active LLM review cycles — planning, checkpoint, assessment."""
+
+    def __init__(
+        self,
+        dispatch_fn: Callable[[str, str], tuple[str, Any]] | None = None,
+        model_review_capability: int = 1,
+        correction_mode: str = "flag",
+        max_correction_windows: int = 3,
+    ) -> None:
+        self._dispatch_fn = dispatch_fn
+        self._model_capability = model_review_capability
+        self._correction_mode = correction_mode
+        self._max_corrections = max_correction_windows
+
+    # ------------------------------------------------------------------
+    # 1. Pre-generation planning
+    # ------------------------------------------------------------------
+
+    def pre_generation_plan(
+        self,
+        task_intent: str,
+        predicted_chain_length: int = 0,
+    ) -> DocumentPlan | None:
+        """Generate document plan when chain > 5 windows.
+
+        Returns None if chain is short or no dispatch_fn.
+        """
+        if predicted_chain_length <= 5:
+            return None
+        if not self._dispatch_fn:
+            return None
+
+        prompt = (
+            "Create an outline for the following task. For each section provide:\n"
+            "- Section title\n"
+            "- 2-3 key points to cover\n"
+            "- Dependencies on other sections\n\n"
+            f"Task: {task_intent}\n"
+            f"Estimated length: {predicted_chain_length} windows\n\n"
+            "Format: numbered sections."
+        )
+
+        try:
+            output, _ = self._dispatch_fn(prompt, "")
+        except Exception:
+            return None
+
+        return self._parse_plan(output, predicted_chain_length)
+
+    def _parse_plan(self, output: str, windows: int) -> DocumentPlan:
+        """Parse LLM output into a DocumentPlan."""
+        sections: list[PlannedSection] = []
+        current_title = ""
+        current_points: list[str] = []
+
+        for line in output.split("\n"):
+            line = line.strip()
+            if not line:
+                continue
+            # New section: starts with number
+            if re.match(r"\d+[\.\)]\s", line):
+                if current_title:
+                    sections.append(PlannedSection(
+                        title=current_title,
+                        key_points=current_points,
+                    ))
+                current_title = re.sub(r"^\d+[\.\)]\s*", "", line)
+                current_points = []
+            elif line.startswith("-") or line.startswith("•"):
+                current_points.append(line.lstrip("-•").strip())
+
+        if current_title:
+            sections.append(PlannedSection(
+                title=current_title,
+                key_points=current_points,
+            ))
+
+        return DocumentPlan(
+            sections=sections,
+            estimated_windows=windows,
+        )
+
+    # ------------------------------------------------------------------
+    # 2. Checkpoint review
+    # ------------------------------------------------------------------
+
+    def checkpoint_review(
+        self,
+        window_index: int,
+        review_interval: int = 20,
+        task_intent: str = "",
+        top_facts: list[str] | None = None,
+        gap_summary: str = "",
+    ) -> ReviewGuidance | None:
+        """Periodic review at checkpoint windows.
+
+        Gate: model_capability < 3 → None
+        Gate: window_index not at interval → None
+        """
+        if self._model_capability < 3:
+            return None
+        if window_index % review_interval != 0:
+            return None
+        if not self._dispatch_fn:
+            return None
+
+        facts_section = ""
+        if top_facts:
+            facts_section = "\n".join(f"- {f}" for f in top_facts[:30])
+
+        prompt = (
+            "Review checkpoint. Assess the following:\n"
+            "1. Are we on track for the task?\n"
+            "2. Any contradictions in the findings?\n"
+            "3. What should be prioritized next?\n"
+            "4. Any new gaps identified?\n\n"
+            f"Task: {task_intent}\n\n"
+            f"Key facts so far:\n{facts_section}\n\n"
+            f"Gap summary: {gap_summary}\n\n"
+            "Be concise."
+        )
+
+        try:
+            output, _ = self._dispatch_fn(prompt, "")
+        except Exception:
+            return None
+
+        return self._parse_review(output)
+
+    def _parse_review(self, output: str) -> ReviewGuidance:
+        """Parse checkpoint review output."""
+        guidance = ReviewGuidance(raw_output=output)
+        lines = output.split("\n")
+        for line in lines:
+            line_lower = line.lower().strip()
+            if "not on track" in line_lower or "off track" in line_lower:
+                guidance.on_track = False
+            if "contradict" in line_lower:
+                guidance.contradictions.append(line.strip())
+            if "priorit" in line_lower or "next" in line_lower:
+                guidance.priorities.append(line.strip())
+            if "gap" in line_lower or "missing" in line_lower:
+                guidance.new_gaps.append(line.strip())
+        return guidance
+
+    # ------------------------------------------------------------------
+    # 3. Post-generation self-assessment
+    # ------------------------------------------------------------------
+
+    def post_generation_assessment(
+        self,
+        accumulated_output: str,
+        task_intent: str,
+    ) -> AssessmentResult:
+        """Score output quality and flag issues.
+
+        Weak model → basic heuristic scoring.
+        Strong model → full LLM self-assessment.
+        """
+        if self._model_capability < 3 or not self._dispatch_fn:
+            return self._heuristic_assessment(accumulated_output, task_intent)
+
+        prompt = (
+            "Score the following output on a scale of 1-10 for completeness, "
+            "accuracy, and coherence. Start your response with 'SCORE: X/10'. "
+            "Then list any issues as numbered items.\n\n"
+            f"Task: {task_intent}"
+        )
+
+        try:
+            output, _ = self._dispatch_fn(prompt, accumulated_output[:5000])
+        except Exception:
+            return self._heuristic_assessment(accumulated_output, task_intent)
+
+        return self._parse_assessment(output)
+
+    def _parse_assessment(self, output: str) -> AssessmentResult:
+        """Parse LLM assessment output."""
+        result = AssessmentResult(raw_output=output)
+
+        # Extract score
+        score_match = re.search(r"SCORE:\s*(\d+(?:\.\d+)?)\s*/\s*10", output, re.IGNORECASE)
+        if score_match:
+            result.score = float(score_match.group(1))
+        else:
+            result.score = 5.0  # Default if can't parse
+
+        # Extract issues
+        for line in output.split("\n"):
+            line = line.strip()
+            if re.match(r"\d+\.", line):
+                result.issues.append(re.sub(r"^\d+\.\s*", "", line))
+
+        result.needs_correction = (
+            result.score < 6
+            and self._correction_mode == "correct"
+        )
+
+        return result
+
+    def _heuristic_assessment(
+        self, output: str, task_intent: str,
+    ) -> AssessmentResult:
+        """Basic quality scoring without LLM."""
+        score = 5.0
+        issues: list[str] = []
+
+        # Length heuristic
+        words = len(output.split())
+        if words < 50:
+            score -= 2
+            issues.append("Output is very short")
+        elif words > 200:
+            score += 1
+
+        # Check for task keyword coverage
+        task_words = set(task_intent.lower().split())
+        output_words = set(output.lower().split())
+        coverage = len(task_words & output_words) / max(len(task_words), 1)
+        if coverage < 0.3:
+            score -= 1
+            issues.append("Low task keyword coverage")
+        elif coverage > 0.7:
+            score += 1
+
+        score = max(1.0, min(10.0, score))
+
+        return AssessmentResult(
+            score=score,
+            issues=issues,
+            needs_correction=score < 6 and self._correction_mode == "correct",
+        )
+
+    # ------------------------------------------------------------------
+    # Targeted re-generation
+    # ------------------------------------------------------------------
+
+    def targeted_regeneration(
+        self,
+        issues: list[str],
+        task_intent: str,
+    ) -> list[str]:
+        """Re-generate targeted fixes for each issue (capped at max_corrections)."""
+        if not self._dispatch_fn:
+            return []
+
+        corrections: list[str] = []
+        for issue in issues[:self._max_corrections]:
+            prompt = (
+                f"Fix this specific issue in the output: {issue}\n"
+                f"Original task: {task_intent}\n"
+                "Provide only the corrected section."
+            )
+            try:
+                output, _ = self._dispatch_fn(prompt, "")
+                corrections.append(output)
+            except Exception:
+                corrections.append(f"[correction failed for: {issue}]")
+
+        return corrections

crp/advanced/scale_mode.py

@@ -0,0 +1,129 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Scale-mode selector — auto-configure session by quality tier (§8.3, §15).
+
+Classifies input into quality tiers S/A/B/C/D based on token ratio,
+then configures processing mode, validation tiers, review cycles, etc.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import IntEnum
+
+# ---------------------------------------------------------------------------
+# Quality tiers (§10.2)
+# ---------------------------------------------------------------------------
+
+
+class QualityTier(IntEnum):
+    """Quality tiers — S (single window) through D (>1000 windows at 128K ctx)."""
+
+    S = 0  # ≤ C
+    A = 1  # C–10C
+    B = 2  # 10C–100C
+    C = 3  # 100C–1000C
+    D = 4  # >1000C
+
+
+def classify_quality_tier(
+    estimated_tokens: int, context_window: int,
+) -> QualityTier:
+    """Classify input into quality tier based on token-to-context ratio."""
+    if context_window <= 0:
+        return QualityTier.S
+    ratio = estimated_tokens / context_window
+    if ratio <= 1:
+        return QualityTier.S
+    if ratio <= 10:
+        return QualityTier.A
+    if ratio <= 100:
+        return QualityTier.B
+    if ratio <= 1000:
+        return QualityTier.C
+    return QualityTier.D
+
+
+# ---------------------------------------------------------------------------
+# Processing mode
+# ---------------------------------------------------------------------------
+
+
+def select_processing_mode(
+    estimated_tokens: int, context_window: int,
+) -> str:
+    """Select processing mode based on windows needed."""
+    if context_window <= 0:
+        return "SERIAL"
+    windows = estimated_tokens / context_window
+    if windows <= 10:
+        return "SERIAL"
+    if windows <= 100:
+        return "SERIAL_WITH_REGROUNDING"
+    if windows <= 1000:
+        return "HIERARCHICAL"
+    return "HIERARCHICAL_MULTI_LEVEL"
+
+
+# ---------------------------------------------------------------------------
+# Session configuration
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class SessionConfig:
+    """Auto-configured session parameters."""
+
+    quality_tier: QualityTier = QualityTier.S
+    processing_mode: str = "SERIAL"
+    cqs_enabled: bool = False
+    validation_tiers: int = 1
+    review_cycles_enabled: bool = False
+    planning_window: bool = False
+    hierarchical: bool = False
+    re_grounding: bool = False
+    model_review_capability: int = 1
+
+
+class ScaleModeSelector:
+    """Auto-configure session based on quality tier and model capability."""
+
+    def __init__(
+        self,
+        context_window: int = 128_000,
+    ) -> None:
+        self._context_window = context_window
+
+    def configure_session(
+        self,
+        estimated_tokens: int,
+        model_capability: int = 1,
+    ) -> SessionConfig:
+        """Auto-configure session based on input size and model capability.
+
+        Args:
+            estimated_tokens: Total estimated input tokens.
+            model_capability: Assessed model capability (1, 2, or 3).
+
+        Returns:
+            SessionConfig with all parameters set.
+        """
+        tier = classify_quality_tier(estimated_tokens, self._context_window)
+        mode = select_processing_mode(estimated_tokens, self._context_window)
+
+        return SessionConfig(
+            quality_tier=tier,
+            processing_mode=mode,
+            cqs_enabled=tier >= QualityTier.A,
+            validation_tiers=min(
+                model_capability,
+                3 if tier >= QualityTier.C else 2 if tier >= QualityTier.B else 1,
+            ),
+            review_cycles_enabled=(
+                tier >= QualityTier.B and model_capability >= 3
+            ),
+            planning_window=tier >= QualityTier.B,
+            hierarchical=tier >= QualityTier.C,
+            re_grounding=tier >= QualityTier.B,
+            model_review_capability=model_capability,
+        )

crp/advanced/source_grounding.py

@@ -0,0 +1,207 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Source grounding — store/retrieve verbatim source passages (§17).
+
+Stores passages for facts with confidence ≥ 0.8. Integrates passages
+into envelopes with tier-based budget allocation.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Constants (§17)
+# ---------------------------------------------------------------------------
+
+HIGH_CONFIDENCE_THRESHOLD = 0.8
+
+# Budget allocation by quality tier (fact_pct, source_pct)
+BUDGET_BY_TIER: dict[str, tuple[float, float]] = {
+    "S": (1.0, 0.0),    # No envelope needed
+    "A": (0.90, 0.10),   # Low drift risk
+    "B": (0.70, 0.30),   # Highest drift risk
+    "C": (0.70, 0.30),   # Hierarchy adds abstraction
+    "D": (0.75, 0.25),   # Space premium
+}
+
+HIGH_RELEVANCE_THRESHOLD = 0.7
+
+
+# ---------------------------------------------------------------------------
+# Data types
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class SourcePassage:
+    """Verbatim passage from the original input linked to facts."""
+
+    passage_id: str = ""
+    text: str = ""
+    source_window: int = 0
+    token_offset_start: int = 0
+    token_offset_end: int = 0
+    linked_fact_ids: list[str] = field(default_factory=list)
+    token_count: int = 0
+    relevance_score: float = 0.0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "passage_id": self.passage_id,
+            "text": self.text,
+            "source_window": self.source_window,
+            "token_offset_start": self.token_offset_start,
+            "token_offset_end": self.token_offset_end,
+            "linked_fact_ids": list(self.linked_fact_ids),
+            "token_count": self.token_count,
+            "relevance_score": self.relevance_score,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> SourcePassage:
+        return cls(
+            passage_id=data.get("passage_id", ""),
+            text=data.get("text", ""),
+            source_window=data.get("source_window", 0),
+            token_offset_start=data.get("token_offset_start", 0),
+            token_offset_end=data.get("token_offset_end", 0),
+            linked_fact_ids=data.get("linked_fact_ids", []),
+            token_count=data.get("token_count", 0),
+            relevance_score=data.get("relevance_score", 0.0),
+        )
+
+
+# ---------------------------------------------------------------------------
+# SourceGroundingEngine
+# ---------------------------------------------------------------------------
+
+
+class SourceGroundingEngine:
+    """Store and retrieve verbatim source passages for high-confidence facts."""
+
+    def __init__(
+        self,
+        count_tokens: Callable[[str], int] | None = None,
+    ) -> None:
+        self._count_tokens = count_tokens or (lambda t: len(t) // 4)
+        self._passages: dict[str, SourcePassage] = {}      # passage_id → passage
+        self._fact_to_passages: dict[str, list[str]] = {}  # fact_id → [passage_ids]
+
+    @property
+    def passage_count(self) -> int:
+        return len(self._passages)
+
+    def store_passage(
+        self,
+        passage: SourcePassage,
+        fact_confidence: float = 0.0,
+    ) -> bool:
+        """Store a passage if linked fact has confidence ≥ threshold.
+
+        Returns True if stored, False if below threshold.
+        """
+        if fact_confidence < HIGH_CONFIDENCE_THRESHOLD:
+            return False
+
+        passage.token_count = self._count_tokens(passage.text)
+        self._passages[passage.passage_id] = passage
+        for fid in passage.linked_fact_ids:
+            self._fact_to_passages.setdefault(fid, []).append(passage.passage_id)
+        return True
+
+    def get_passages_for_fact(self, fact_id: str) -> list[SourcePassage]:
+        """Retrieve all source passages linked to a fact."""
+        pids = self._fact_to_passages.get(fact_id, [])
+        return [self._passages[pid] for pid in pids if pid in self._passages]
+
+    def build_source_grounded_envelope(
+        self,
+        scored_facts: list[dict[str, Any]],
+        budget_tokens: int,
+        quality_tier: str = "B",
+    ) -> tuple[list[dict[str, Any]], list[SourcePassage]]:
+        """Build envelope with source passages allocated by tier.
+
+        Args:
+            scored_facts: Sorted list of {"id", "text", "score", ...}
+            budget_tokens: Total envelope budget in tokens.
+            quality_tier: S/A/B/C/D
+
+        Returns:
+            (packed_facts, included_passages)
+        """
+        fact_pct, source_pct = BUDGET_BY_TIER.get(quality_tier, (0.70, 0.30))
+        fact_budget = int(budget_tokens * fact_pct)
+        source_budget = int(budget_tokens * source_pct)
+
+        # Phase 1: Pack facts within fact budget
+        packed_facts: list[dict[str, Any]] = []
+        tokens_used = 0
+        for fact in scored_facts:
+            t = self._count_tokens(fact.get("text", ""))
+            if tokens_used + t > fact_budget:
+                break
+            packed_facts.append(fact)
+            tokens_used += t
+
+        # Phase 2: Allocate source passages for high-relevance packed facts
+        included_passages: list[SourcePassage] = []
+        source_tokens_used = 0
+
+        for fact in packed_facts:
+            fid = fact.get("id", "")
+            score = fact.get("score", 0.0)
+            if score < HIGH_RELEVANCE_THRESHOLD:
+                continue
+
+            for passage in self.get_passages_for_fact(fid):
+                if source_tokens_used + passage.token_count > source_budget:
+                    break
+                included_passages.append(passage)
+                source_tokens_used += passage.token_count
+
+        return packed_facts, included_passages
+
+    def format_envelope_section(
+        self,
+        fact: dict[str, Any],
+        passages: list[SourcePassage],
+    ) -> str:
+        """Format a fact with its source passages for envelope inclusion.
+
+        Format:
+        - {fact text} — Window N
+          ↳ [SOURCE: Window N, tokens X-Y]
+            "{verbatim original text}"
+        """
+        lines = [f"- {fact.get('text', '')} — Window {fact.get('window', '?')}"]
+        for p in passages:
+            lines.append(
+                f"  ↳ [SOURCE: Window {p.source_window}, "
+                f"tokens {p.token_offset_start}-{p.token_offset_end}]"
+            )
+            lines.append(f'    "{p.text}"')
+        return "\n".join(lines)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize for persistence."""
+        return {
+            "passages": {pid: p.to_dict() for pid, p in self._passages.items()},
+            "fact_to_passages": dict(self._fact_to_passages),
+        }
+
+    @classmethod
+    def from_dict(
+        cls,
+        data: dict[str, Any],
+        count_tokens: Callable[[str], int] | None = None,
+    ) -> SourceGroundingEngine:
+        """Restore from serialized state."""
+        engine = cls(count_tokens=count_tokens)
+        for pid, pdata in data.get("passages", {}).items():
+            engine._passages[pid] = SourcePassage.from_dict(pdata)
+        engine._fact_to_passages = data.get("fact_to_passages", {})
+        return engine

crp/ckf/__init__.py

@@ -0,0 +1,35 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Contextual Knowledge Fabric — 4-mode retrieval, community detection, pub/sub."""
+
+from .community import Community, CommunityDetector, CommunityResult
+from .fabric import CKFConfig, CKFHealth, ContextualKnowledgeFabric
+from .gc import GarbageCollector, GCResult
+from .graph_walk import GraphWalkResult, graph_walk
+from .merge import MergedFact, MergeResult, multi_mode_merge
+from .pattern_query import PatternQueryResult, pattern_query
+from .pubsub import CKFEvent, CKFEventType, PubSubEventBus
+from .semantic import SemanticResult, semantic_fallback
+
+__all__ = [
+    "CKFConfig",
+    "CKFEvent",
+    "CKFEventType",
+    "CKFHealth",
+    "Community",
+    "CommunityDetector",
+    "CommunityResult",
+    "ContextualKnowledgeFabric",
+    "GCResult",
+    "GarbageCollector",
+    "GraphWalkResult",
+    "MergeResult",
+    "MergedFact",
+    "PatternQueryResult",
+    "PubSubEventBus",
+    "SemanticResult",
+    "graph_walk",
+    "multi_mode_merge",
+    "pattern_query",
+    "semantic_fallback",
+]