crprotocol 2.0.0__py3-none-any.whl

crp/envelope/__init__.py

@@ -0,0 +1,30 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Envelope construction — 6-phase algorithm, scoring, packing, formatting."""
+
+from .builder import EnvelopeResult, EnvelopeState, compute_envelope_budget, construct
+from .decomposer import DecompositionResult, decompose_task_aspects
+from .formatter import EnvelopeSection, format_envelope
+from .packer import PackedFact, PackingResult, estimate_tokens, pack_facts
+from .reranker import CrossEncoderCache, rerank
+from .scoring import ScoredFact, ScoringConfig, score_facts
+
+__all__ = [
+    "CrossEncoderCache",
+    "DecompositionResult",
+    "EnvelopeResult",
+    "EnvelopeSection",
+    "EnvelopeState",
+    "PackedFact",
+    "PackingResult",
+    "ScoredFact",
+    "ScoringConfig",
+    "compute_envelope_budget",
+    "construct",
+    "decompose_task_aspects",
+    "estimate_tokens",
+    "format_envelope",
+    "pack_facts",
+    "rerank",
+    "score_facts",
+]

crp/envelope/builder.py

@@ -0,0 +1,288 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Envelope builder — top-level 6-phase construction orchestrator (§3.2).
+
+``construct(task_intent, budget, state)`` returns the final envelope text.
+
+6-Phase algorithm:
+  Phase 1: Multi-aspect task decomposition       → decomposer.py
+  Phase 2: Bi-encoder scoring                    → scoring.py
+  Phase 3: Cross-encoder reranking               → reranker.py
+  Phase 4: Graph-aware packing                   → packer.py
+  Phase 5: Bookend strategy                      → packer.py
+  Phase 6: CKF retrieval gate                    → this module
+
+Envelope budget formula (02_CORE §2.1):
+  E_max = C − S − T − G
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+logger = logging.getLogger("crp.envelope.builder")
+
+from crp.core.task_intent import TaskIntent
+from crp.extraction.types import Fact, FactGraph
+
+from .decomposer import DecompositionResult, decompose_task_aspects
+from .formatter import format_envelope
+from .packer import PackingResult, estimate_tokens, pack_facts
+from .reranker import CrossEncoderCache, rerank
+from .scoring import ScoringConfig, score_facts
+
+# ---------------------------------------------------------------------------
+# Envelope state — holds facts, graph, and per-session metadata
+# ---------------------------------------------------------------------------
+
+CKF_GATE_TOKENS = 120  # min remaining tokens before pulling from CKF
+CKF_RESERVE_RATIO = 0.15  # fraction of fact_budget reserved for CKF when retriever is available
+
+
+@dataclass
+class EnvelopeState:
+    """Session-scoped state passed to the builder."""
+
+    # Fact store (warm tier)
+    facts: list[Fact] = field(default_factory=list)
+    graph: FactGraph = field(default_factory=FactGraph)
+
+    # Metadata for scoring
+    current_window_index: int = 0
+    seen_counts: dict[str, int] = field(default_factory=dict)
+    fact_window_indices: dict[str, int] = field(default_factory=dict)
+
+    # Critical state sections (pre-populated by state management)
+    sections: dict[str, str] = field(default_factory=dict)
+
+    # CKF callback — if registered, called when budget allows
+    ckf_retriever: Callable[[str, int], list[Fact]] | None = None
+
+    # Cross-encoder cache (shared across windows)
+    ce_cache: CrossEncoderCache = field(default_factory=CrossEncoderCache)
+
+    # Scoring config override
+    scoring_config: ScoringConfig | None = None
+
+
+# ---------------------------------------------------------------------------
+# Build result
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class EnvelopeResult:
+    """Output of ``construct()``."""
+
+    envelope_text: str = ""
+    envelope_tokens: int = 0
+    budget_tokens: int = 0
+    saturation: float = 0.0  # envelope_tokens / budget_tokens
+    facts_included: int = 0
+    facts_available: int = 0
+    bookend_count: int = 0
+    compressed_count: int = 0
+    ckf_facts_added: int = 0
+    latency_ms: float = 0.0
+    decomposition: DecompositionResult | None = None
+    packing: PackingResult | None = None
+
+
+# ---------------------------------------------------------------------------
+# Envelope budget computation
+# ---------------------------------------------------------------------------
+
+
+def compute_envelope_budget(
+    context_window: int,
+    system_tokens: int,
+    task_tokens: int,
+    generation_reserve: int | None = None,
+    max_output_tokens: int | None = None,
+) -> int:
+    """E_max = C − S − T − G.
+
+    Generation reserve precedence:
+      1. User-specified max_output_tokens
+      2. Explicit generation_reserve
+      3. Default: min(C // 4, 16384)
+    """
+    if max_output_tokens is not None:
+        g = max_output_tokens
+    elif generation_reserve is not None:
+        g = generation_reserve
+    else:
+        g = min(context_window // 4, 16384)
+    return max(0, context_window - system_tokens - task_tokens - g)
+
+
+# ---------------------------------------------------------------------------
+# Main entry point
+# ---------------------------------------------------------------------------
+
+
+def construct(
+    task_intent: TaskIntent,
+    budget_tokens: int,
+    state: EnvelopeState,
+    *,
+    count_tokens: Callable[[str], int] | None = None,
+    chars_per_token: float = 3.3,
+) -> EnvelopeResult:
+    """Build an envelope for *task_intent* within *budget_tokens*.
+
+    This is the top-level orchestrator implementing the 6-phase algorithm.
+
+    Parameters
+    ----------
+    task_intent : TaskIntent
+        The task to assemble context for.
+    budget_tokens : int
+        Maximum envelope tokens (E_max from budget formula).
+    state : EnvelopeState
+        Session-scoped state containing facts, graph, sections, etc.
+    count_tokens : callable | None
+        Model tokenizer function ``(str) → int``.  Falls back to estimator.
+    chars_per_token : float
+        Calibrated chars/token ratio for fallback estimation.
+    """
+    t0 = time.perf_counter()
+
+    def _count(text: str) -> int:
+        if count_tokens is not None:
+            return count_tokens(text)
+        return estimate_tokens(text, chars_per_token)
+
+    result = EnvelopeResult(
+        budget_tokens=budget_tokens,
+        facts_available=len(state.facts),
+    )
+
+    if budget_tokens <= 0:
+        result.latency_ms = (time.perf_counter() - t0) * 1000
+        return result
+
+    # -----------------------------------------------------------------------
+    # Phase 1: Multi-aspect task decomposition
+    # -----------------------------------------------------------------------
+    decomposition = decompose_task_aspects(task_intent)
+    result.decomposition = decomposition
+
+    # -----------------------------------------------------------------------
+    # Phase 2: Bi-encoder scoring
+    # -----------------------------------------------------------------------
+    scored = score_facts(
+        state.facts,
+        decomposition,
+        state.graph,
+        current_window_index=state.current_window_index,
+        seen_counts=state.seen_counts,
+        fact_window_indices=state.fact_window_indices,
+        config=state.scoring_config,
+    )
+
+    # -----------------------------------------------------------------------
+    # Phase 3: Cross-encoder reranking
+    # -----------------------------------------------------------------------
+    scored = rerank(
+        scored,
+        task_intent,
+        cache=state.ce_cache,
+        current_window=state.current_window_index,
+    )
+
+    # -----------------------------------------------------------------------
+    # Compute section overhead (tier 1-2 sections that always appear)
+    # -----------------------------------------------------------------------
+    section_text_parts: dict[str, str] = {}
+    section_overhead = 0
+    for name, content in state.sections.items():
+        if content and content.strip():
+            header = f"[{name.upper()}]\n{content.strip()}"
+            section_overhead += _count(header) + 2  # +2 for "\n\n"
+            section_text_parts[name] = content.strip()
+
+    # Budget remaining for facts
+    fact_budget = max(0, budget_tokens - section_overhead)
+
+    # Reserve budget for CKF when retriever is available (§audit fix G1)
+    ckf_reserve = 0
+    if state.ckf_retriever is not None and fact_budget > CKF_GATE_TOKENS * 2:
+        ckf_reserve = max(CKF_GATE_TOKENS, int(fact_budget * CKF_RESERVE_RATIO))
+    warm_budget = fact_budget - ckf_reserve
+
+    # -----------------------------------------------------------------------
+    # Phase 4+5: Graph-aware packing + bookend strategy
+    # -----------------------------------------------------------------------
+    packing = pack_facts(
+        scored,
+        state.graph,
+        warm_budget,
+        count_tokens=count_tokens,
+        chars_per_token=chars_per_token,
+    )
+    result.packing = packing
+    result.facts_included = packing.facts_packed
+    result.bookend_count = packing.bookend_count
+    result.compressed_count = packing.compressed_count
+
+    # -----------------------------------------------------------------------
+    # Phase 6: CKF retrieval gate
+    # -----------------------------------------------------------------------
+    # CKF gets reserved budget + any unused warm store budget
+    remaining_for_ckf = ckf_reserve + (warm_budget - packing.total_tokens)
+    if (
+        remaining_for_ckf > CKF_GATE_TOKENS
+        and state.ckf_retriever is not None
+        and decomposition.aspects
+    ):
+        query = " ".join(decomposition.aspects[:3])
+        try:
+            ckf_facts = state.ckf_retriever(query, remaining_for_ckf)
+            if ckf_facts:
+                # Score and pack CKF facts into remaining budget
+                ckf_scored = score_facts(
+                    ckf_facts,
+                    decomposition,
+                    state.graph,
+                    current_window_index=state.current_window_index,
+                    config=state.scoring_config,
+                )
+                ckf_packed = pack_facts(
+                    ckf_scored,
+                    state.graph,
+                    remaining_for_ckf,
+                    count_tokens=count_tokens,
+                    chars_per_token=chars_per_token,
+                )
+                # Merge CKF results
+                packing.packed_facts.extend(ckf_packed.packed_facts)
+                packing.total_tokens += ckf_packed.total_tokens
+                result.ckf_facts_added = ckf_packed.facts_packed
+        except Exception:  # noqa: BLE001
+            logger.warning("CKF lookup failed during envelope build", exc_info=True)
+
+    # -----------------------------------------------------------------------
+    # Serialization
+    # -----------------------------------------------------------------------
+    envelope_text = format_envelope(section_text_parts, packing.packed_facts)
+
+    result.envelope_text = envelope_text
+    result.envelope_tokens = _count(envelope_text)
+
+    # Clamp envelope to budget if it overflows (§audit M9)
+    if budget_tokens > 0 and result.envelope_tokens > budget_tokens:
+        # Truncate envelope text to fit within budget
+        target_chars = int(budget_tokens * chars_per_token)
+        if len(envelope_text) > target_chars:
+            result.envelope_text = envelope_text[:target_chars]
+            result.envelope_tokens = _count(result.envelope_text)
+
+    if budget_tokens > 0:
+        result.saturation = min(1.0, result.envelope_tokens / budget_tokens)
+    result.latency_ms = (time.perf_counter() - t0) * 1000
+
+    return result

crp/envelope/decomposer.py

@@ -0,0 +1,236 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Phase 1 — Multi-aspect task decomposition (§3.2).
+
+Decomposes TaskIntent into explicit + implicit aspects, each with an embedding
+vector.  Falls back to lightweight word-overlap vectors when sentence-transformers
+is unavailable.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+
+from crp.core.task_intent import TaskIntent
+
+# ---------------------------------------------------------------------------
+# Embedding abstraction — lightweight fallback when ML deps absent
+# ---------------------------------------------------------------------------
+
+# Sentinel that indicates sentence-transformers is available
+_ST_AVAILABLE: bool = False
+_EMBED_MODEL: object | None = None
+_EMBED_DIM: int = 384  # all-MiniLM-L6-v2 output dim
+
+
+def _try_load_sentence_transformer() -> bool:
+    """Attempt lazy load of sentence-transformers model.  Returns True on success."""
+    global _ST_AVAILABLE, _EMBED_MODEL  # noqa: PLW0603
+    if _ST_AVAILABLE:
+        return True
+    try:
+        from sentence_transformers import SentenceTransformer  # type: ignore[import-untyped]
+
+        _EMBED_MODEL = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+        _ST_AVAILABLE = True
+    except Exception:  # noqa: BLE001
+        pass
+    return _ST_AVAILABLE
+
+
+def get_embedding_fn() -> "Callable[[str], list[float]] | None":
+    """Return a text→embedding function using the loaded model, or None.
+
+    This is the canonical embedding function for the CRP pipeline.
+    Used by: set_embedding_function(), gap_analysis(), CKF semantic mode.
+    """
+    if not _try_load_sentence_transformer() or _EMBED_MODEL is None:
+        return None
+    model = _EMBED_MODEL
+
+    def _embed(text: str) -> list[float]:
+        return model.encode([text], show_progress_bar=False).tolist()[0]  # type: ignore[attr-defined]
+
+    return _embed
+
+
+# -- lightweight fallback vectors (word-overlap, normalised) -----------------
+
+_STOP = frozenset(
+    ["a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", "of", "is", "it", "its", "are", "was", "were", "be", "been", "being", "have", "has", "had", "do", "does", "did", "will", "would", "shall", "should", "may", "might", "can", "could", "this", "that", "these", "those", "i", "we", "you", "he", "she", "they", "me", "us", "him", "her", "them", "my", "our", "your", "his", "its", "their", "with", "from", "by", "as", "not", "no"]
+)
+
+
+def _tokenize(text: str) -> list[str]:
+    """Lower-case word tokeniser (alphanum only)."""
+    return [w for w in re.findall(r"[a-z0-9]+", text.lower()) if w not in _STOP]
+
+
+def _bag_vector(tokens: Sequence[str], vocab: dict[str, int], dim: int) -> list[float]:
+    """Sparse bag-of-words vector projected to *dim* via hashing."""
+    vec = [0.0] * dim
+    for tok in tokens:
+        idx = vocab.get(tok)
+        if idx is None:
+            idx = int(hashlib.sha256(tok.encode()).hexdigest(), 16) % dim
+        vec[idx] += 1.0
+    # L2 normalise
+    norm = math.sqrt(sum(v * v for v in vec)) or 1.0
+    return [v / norm for v in vec]
+
+
+def _build_vocab(all_tokens: list[str]) -> dict[str, int]:
+    """Deterministic vocabulary mapping (ordered by first occurrence)."""
+    vocab: dict[str, int] = {}
+    for tok in all_tokens:
+        if tok not in vocab:
+            vocab[tok] = len(vocab) % _EMBED_DIM
+    return vocab
+
+
+# ---------------------------------------------------------------------------
+# Public result dataclass
+# ---------------------------------------------------------------------------
+
+@dataclass
+class DecompositionResult:
+    """Output of ``decompose_task_aspects``."""
+
+    aspects: list[str] = field(default_factory=list)
+    aspect_embeddings: list[list[float]] = field(default_factory=list)
+    full_embedding: list[float] = field(default_factory=list)
+    used_ml_model: bool = False
+
+
+# ---------------------------------------------------------------------------
+# Noun-phrase extraction (lightweight, no ML)
+# ---------------------------------------------------------------------------
+
+# Simple regex pattern for noun-phrase-like chunks: sequences of adjectives/nouns
+_NP_PATTERN = re.compile(
+    r"\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)\b"  # Capitalised phrases
+    r"|"
+    r"\b([a-z]+(?:[-_][a-z]+)+)\b"  # Hyphenated/underscored compound terms
+)
+
+
+def _extract_noun_phrases(text: str) -> list[str]:
+    """Extract candidate noun phrases from *text*."""
+    phrases: list[str] = []
+    for m in _NP_PATTERN.finditer(text):
+        phrase = (m.group(1) or m.group(2) or "").strip()
+        if phrase and len(phrase) > 2:
+            phrases.append(phrase)
+    # Also grab quoted strings as explicit aspects
+    for m in re.finditer(r'"([^"]{3,80})"', text):
+        phrases.append(m.group(1))
+    # Deduplicate preserving order
+    seen: set[str] = set()
+    result: list[str] = []
+    for p in phrases:
+        key = p.lower()
+        if key not in seen:
+            seen.add(key)
+            result.append(p)
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Implicit aspect expansion (word-based when no embeddings)
+# ---------------------------------------------------------------------------
+
+def _expand_aspects_implicit(aspects: list[str], full_text: str) -> list[str]:
+    """Add implicit aspects derived from the full text that aren't in *aspects*.
+
+    When ML embeddings are unavailable, uses word co-occurrence: finds bigrams
+    from the full text that share a word with an existing aspect.
+    """
+    if not aspects:
+        # Fallback: use first N significant tokens as aspects
+        tokens = _tokenize(full_text)
+        # Pick unique tokens in order, up to 5
+        seen: set[str] = set()
+        result: list[str] = []
+        for t in tokens:
+            if t not in seen and len(t) > 2:
+                seen.add(t)
+                result.append(t)
+                if len(result) >= 5:
+                    break
+        return result
+
+    aspect_words = set()
+    for a in aspects:
+        aspect_words.update(_tokenize(a))
+
+    words = _tokenize(full_text)
+    new_aspects: list[str] = []
+    seen_lower = {a.lower() for a in aspects}
+    for i in range(len(words) - 1):
+        bigram = f"{words[i]} {words[i + 1]}"
+        if bigram.lower() not in seen_lower:
+            # At least one word overlaps with existing aspects
+            if words[i] in aspect_words or words[i + 1] in aspect_words:
+                seen_lower.add(bigram.lower())
+                new_aspects.append(bigram)
+                if len(new_aspects) >= 3:  # cap expansion
+                    break
+    return aspects + new_aspects
+
+
+# ---------------------------------------------------------------------------
+# Main entry point
+# ---------------------------------------------------------------------------
+
+def decompose_task_aspects(task_intent: TaskIntent) -> DecompositionResult:
+    """Decompose *task_intent* into aspects with embedding vectors.
+
+    Algorithm (§3.2 Phase 1):
+    1. Concatenate system_prompt + task_input
+    2. Extract noun phrases as explicit aspects
+    3. Expand with implicit aspects via dependency analysis
+    4. Compute embedding for each aspect + full text
+    """
+    system = task_intent.system_prompt or ""
+    task = task_intent.task_input or ""
+    full_text = f"{system} {task}".strip()
+
+    if not full_text:
+        return DecompositionResult()
+
+    # Step 1-2: extract explicit aspects from noun phrases
+    explicit = _extract_noun_phrases(full_text)
+
+    # Step 3: implicit expansion
+    expanded = _expand_aspects_implicit(explicit, full_text)
+    if not expanded:
+        expanded = [full_text[:200]]  # absolute fallback
+
+    # Step 4: compute embeddings
+    use_ml = _try_load_sentence_transformer()
+
+    if use_ml and _EMBED_MODEL is not None:
+        model = _EMBED_MODEL
+        texts_to_embed = expanded + [full_text]
+        embeddings = model.encode(texts_to_embed, show_progress_bar=False).tolist()  # type: ignore[attr-defined]
+        aspect_embs = embeddings[:-1]
+        full_emb = embeddings[-1]
+    else:
+        # Fallback: bag-of-words vectors
+        all_tokens = _tokenize(full_text)
+        for a in expanded:
+            all_tokens.extend(_tokenize(a))
+        vocab = _build_vocab(all_tokens)
+        aspect_embs = [_bag_vector(_tokenize(a), vocab, _EMBED_DIM) for a in expanded]
+        full_emb = _bag_vector(_tokenize(full_text), vocab, _EMBED_DIM)
+
+    return DecompositionResult(
+        aspects=expanded,
+        aspect_embeddings=aspect_embs,
+        full_embedding=full_emb,
+        used_ml_model=use_ml,
+    )