crprotocol 2.0.0__py3-none-any.whl

crp/envelope/formatter.py

@@ -0,0 +1,168 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Phase 3G — Envelope serialization (§2.2).
+
+Formats the envelope as plain text with [BRACKETED_CAPS] section markers.
+Section priority tiers (03_ENVELOPE §2.2):
+
+  Tier 1 (always): [GOAL], [PHASE], [BLOCKER], [CONSTRAINT], [WINDOW]
+  Tier 2 (include when available): [LLM_SYNTHESIS], [TASK], [OUTPUT_FORMAT]
+  Tier 3 (adaptive): [DISCOVERIES], [SOURCE], [DECISIONS], [ERROR_LOG],
+      [TOOL_HISTORY], [EXPANDED: {label}], [KNOWLEDGE: {query}],
+      [KNOWLEDGE_GRAPH: {seed}], [KNOWLEDGE_COMMUNITY: {name}]
+  Tier 4 (weak models only): [REASONING APPROACH], [SIMILAR SOLVED EXAMPLES]
+
+Fact format:
+  - {fact text}: {detail} — {source window/evidence}
+    ↳ [{RELATION_TYPE}] {target_fact_text}
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .packer import PackedFact
+
+# ---------------------------------------------------------------------------
+# Section definitions
+# ---------------------------------------------------------------------------
+
+# Priority tiers — lower number = higher priority
+TIER_1_SECTIONS = ("GOAL", "PHASE", "BLOCKER", "CONSTRAINT", "WINDOW")
+TIER_2_SECTIONS = ("LLM_SYNTHESIS", "TASK", "OUTPUT_FORMAT")
+TIER_3_SECTIONS = (
+    "DISCOVERIES",
+    "SOURCE",
+    "DECISIONS",
+    "ERROR_LOG",
+    "TOOL_HISTORY",
+)
+# Tier 3 dynamic: EXPANDED, KNOWLEDGE, KNOWLEDGE_GRAPH, KNOWLEDGE_COMMUNITY
+# Tier 4: REASONING APPROACH, SIMILAR SOLVED EXAMPLES
+
+
+@dataclass
+class EnvelopeSection:
+    """One section of the formatted envelope."""
+
+    name: str = ""  # e.g. "GOAL", "EXPANDED: MySource"
+    content: str = ""
+    tier: int = 1
+    tokens: int = 0
+
+
+# ---------------------------------------------------------------------------
+# Section header formatting
+# ---------------------------------------------------------------------------
+
+
+def _section_header(name: str) -> str:
+    """Format a [BRACKETED_CAPS] section header."""
+    return f"[{name.upper()}]"
+
+
+def _classify_tier(name: str) -> int:
+    """Determine the tier of a section by its name."""
+    upper = name.upper()
+    if upper in TIER_1_SECTIONS:
+        return 1
+    if upper in TIER_2_SECTIONS:
+        return 2
+    if upper in TIER_3_SECTIONS:
+        return 3
+    if upper.startswith(("EXPANDED:", "KNOWLEDGE:", "KNOWLEDGE_GRAPH:", "KNOWLEDGE_COMMUNITY:")):
+        return 3
+    if upper in ("REASONING APPROACH", "SIMILAR SOLVED EXAMPLES"):
+        return 4
+    return 3  # default to tier 3 for unknown sections
+
+
+# ---------------------------------------------------------------------------
+# Format facts into [DISCOVERIES] section
+# ---------------------------------------------------------------------------
+
+
+def format_facts_section(packed_facts: list[PackedFact]) -> str:
+    """Format packed facts into the body of the [DISCOVERIES] section.
+
+    Separates bookend facts with a marker comment.
+    """
+    main_facts: list[str] = []
+    bookend_facts: list[str] = []
+
+    for pf in packed_facts:
+        if pf.is_bookend:
+            bookend_facts.append(pf.text)
+        else:
+            main_facts.append(pf.text)
+
+    parts: list[str] = []
+    if main_facts:
+        parts.extend(main_facts)
+    if bookend_facts:
+        parts.append("")  # blank line before bookend
+        parts.append("--- Key facts (reinforced) ---")
+        parts.extend(bookend_facts)
+
+    return "\n".join(parts)
+
+
+# ---------------------------------------------------------------------------
+# Main formatting function
+# ---------------------------------------------------------------------------
+
+
+def format_envelope(
+    sections: dict[str, str],
+    packed_facts: list[PackedFact] | None = None,
+) -> str:
+    """Format *sections* into the final envelope text.
+
+    Parameters
+    ----------
+    sections : dict[str, str]
+        Section name → content text.  Names should be upper-case
+        (e.g. ``{"GOAL": "Analyse CVE...", "PHASE": "scanning"}``).
+    packed_facts : list[PackedFact] | None
+        If provided, formats and inserts as the [DISCOVERIES] section.
+
+    Returns
+    -------
+    str
+        Plain-text envelope: ``[SECTION_NAME]\\ncontent\\n\\n...``
+    """
+    # Build section objects with tier classification
+    envelope_sections: list[EnvelopeSection] = []
+
+    for name, content in sections.items():
+        if content and content.strip():
+            envelope_sections.append(
+                EnvelopeSection(
+                    name=name,
+                    content=content.strip(),
+                    tier=_classify_tier(name),
+                )
+            )
+
+    # Add packed facts as DISCOVERIES if not already provided
+    if packed_facts and "DISCOVERIES" not in sections:
+        facts_text = format_facts_section(packed_facts)
+        if facts_text.strip():
+            envelope_sections.append(
+                EnvelopeSection(
+                    name="DISCOVERIES",
+                    content=facts_text,
+                    tier=3,
+                )
+            )
+
+    # Sort by tier (lower = higher priority), then preserve insertion order within tier
+    envelope_sections.sort(key=lambda s: s.tier)
+
+    # Serialize
+    parts: list[str] = []
+    for sec in envelope_sections:
+        header = _section_header(sec.name)
+        parts.append(f"{header}\n{sec.content}")
+
+    return "\n\n".join(parts)

crp/envelope/packer.py

@@ -0,0 +1,211 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Phase 4+5 — Graph-aware packing & bookend strategy (§3.2).
+
+Greedy packing: sort by score, pack while token budget remains.
+Graph neighbour pulling: 2-hop BFS, indented sub-lines.
+Compressed fact fallback: if >50 tokens remain but no full fact fits.
+Bookend strategy: duplicate top-3 scored facts at END of envelope.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from crp.extraction.types import FactGraph, RelationType
+
+from .scoring import ScoredFact
+
+# ---------------------------------------------------------------------------
+# Token estimation
+# ---------------------------------------------------------------------------
+
+# Default chars-per-token ratio for English BPE models.
+# The spec PROHIBITS hardcoded //4 but ALLOWS a calibrated estimator.
+# This default (3.3 chars/token) is conservative for most English BPE models.
+_DEFAULT_CHARS_PER_TOKEN = 3.3
+
+
+def estimate_tokens(text: str, chars_per_token: float = _DEFAULT_CHARS_PER_TOKEN) -> int:
+    """Estimate token count from text length using calibrated ratio.
+
+    This is used as a fallback when no model tokenizer is available.
+    When an actual tokenizer is provided via ``count_tokens``, use that instead.
+    """
+    if not text:
+        return 0
+    return max(1, int(len(text) / chars_per_token + 0.5))
+
+
+# ---------------------------------------------------------------------------
+# Packing result
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class PackedFact:
+    """A fact selected for the envelope, with its formatted text."""
+
+    fact_id: str = ""
+    text: str = ""
+    score: float = 0.0
+    tokens: int = 0
+    is_neighbour: bool = False
+    is_compressed: bool = False
+    is_bookend: bool = False
+
+
+@dataclass
+class PackingResult:
+    """Output of the packing phase."""
+
+    packed_facts: list[PackedFact] = field(default_factory=list)
+    total_tokens: int = 0
+    budget: int = 0
+    facts_considered: int = 0
+    facts_packed: int = 0
+    compressed_count: int = 0
+    bookend_count: int = 0
+
+
+# ---------------------------------------------------------------------------
+# Graph neighbour formatting
+# ---------------------------------------------------------------------------
+
+def _format_neighbours(
+    fact_id: str,
+    graph: FactGraph,
+    max_hops: int = 2,
+) -> list[str]:
+    """BFS neighbours up to *max_hops*, formatted as indented sub-lines."""
+    sub = graph.subgraph_for({fact_id}, max_hops=max_hops)
+    lines: list[str] = []
+    for edge in sub.edges:
+        if edge.source_id == fact_id:
+            target = sub.nodes.get(edge.target_id)
+            if target:
+                rel = edge.relation_type
+                if isinstance(rel, RelationType):
+                    rel = rel.value
+                lines.append(f"  ↳ [{rel}] {target.text}")
+    return lines
+
+
+# ---------------------------------------------------------------------------
+# Compressed fact (truncation)
+# ---------------------------------------------------------------------------
+
+def _compress_fact(text: str, max_tokens: int, chars_per_token: float) -> str:
+    """Truncate *text* to fit within *max_tokens*."""
+    max_chars = int(max_tokens * chars_per_token)
+    if len(text) <= max_chars:
+        return text
+    return text[: max_chars - 3] + "..."
+
+
+# ---------------------------------------------------------------------------
+# Main packing function
+# ---------------------------------------------------------------------------
+
+BOOKEND_COUNT = 3
+MIN_REMAINING_FOR_COMPRESSED = 50
+
+
+def pack_facts(
+    scored_facts: list[ScoredFact],
+    graph: FactGraph,
+    budget_tokens: int,
+    *,
+    count_tokens: Any = None,
+    chars_per_token: float = _DEFAULT_CHARS_PER_TOKEN,
+) -> PackingResult:
+    """Greedily pack *scored_facts* into the envelope token budget.
+
+    Parameters
+    ----------
+    scored_facts : list[ScoredFact]
+        Pre-sorted by composite score (descending).
+    graph : FactGraph
+        Fact graph for 2-hop neighbour pulling.
+    budget_tokens : int
+        Maximum tokens available for the facts section.
+    count_tokens : callable | None
+        Actual tokenizer function ``(str) → int``.  If None, uses estimate.
+    chars_per_token : float
+        Calibrated chars/token for the estimator fallback.
+    """
+    if not scored_facts or budget_tokens <= 0:
+        return PackingResult(budget=budget_tokens, facts_considered=len(scored_facts))
+
+    def _count(text: str) -> int:
+        if count_tokens is not None:
+            return count_tokens(text)
+        return estimate_tokens(text, chars_per_token)
+
+    result = PackingResult(budget=budget_tokens, facts_considered=len(scored_facts))
+    used_tokens = 0
+    packed_ids: set[str] = set()
+
+    # Phase 4: Greedy packing with graph-neighbour pulling
+    for sf in scored_facts:
+        fact_line = f"- {sf.fact.text}"
+        neighbour_lines = _format_neighbours(sf.fact.id, graph)
+        full_text = fact_line
+        if neighbour_lines:
+            full_text = fact_line + "\n" + "\n".join(neighbour_lines)
+
+        fact_tokens = _count(full_text)
+
+        if used_tokens + fact_tokens <= budget_tokens:
+            pf = PackedFact(
+                fact_id=sf.fact.id,
+                text=full_text,
+                score=sf.composite_score,
+                tokens=fact_tokens,
+            )
+            result.packed_facts.append(pf)
+            packed_ids.add(sf.fact.id)
+            used_tokens += fact_tokens
+            continue
+
+        # Compressed fallback: if >50 tokens remain, truncate this fact to fit
+        remaining = budget_tokens - used_tokens
+        if remaining >= MIN_REMAINING_FOR_COMPRESSED:
+            compressed_text = _compress_fact(fact_line, remaining, chars_per_token)
+            ct = _count(compressed_text)
+            if ct <= remaining:
+                pf = PackedFact(
+                    fact_id=sf.fact.id,
+                    text=compressed_text,
+                    score=sf.composite_score,
+                    tokens=ct,
+                    is_compressed=True,
+                )
+                result.packed_facts.append(pf)
+                packed_ids.add(sf.fact.id)
+                used_tokens += ct
+                result.compressed_count += 1
+            break  # budget exhausted after compressed fit attempt
+        else:
+            break  # not enough room even for compression
+
+    # Phase 5: Bookend strategy — duplicate top-3 facts at END
+    bookend_candidates = [pf for pf in result.packed_facts if not pf.is_compressed][:BOOKEND_COUNT]
+    for pf in bookend_candidates:
+        bookend_tokens = _count(pf.text)
+        if used_tokens + bookend_tokens <= budget_tokens:
+            bpf = PackedFact(
+                fact_id=pf.fact_id,
+                text=pf.text,
+                score=pf.score,
+                tokens=bookend_tokens,
+                is_bookend=True,
+            )
+            result.packed_facts.append(bpf)
+            used_tokens += bookend_tokens
+            result.bookend_count += 1
+
+    result.total_tokens = used_tokens
+    result.facts_packed = len(packed_ids)
+    return result

crp/envelope/reranker.py

@@ -0,0 +1,209 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Phase 3 — Cross-encoder reranking (§3.2).
+
+Reranks the top-K bi-encoder scored facts using a cross-encoder for higher
+accuracy.  Falls back to bi-encoder scores when the cross-encoder model is
+unavailable.
+
+Blended score: 0.6 × CE_score + 0.4 × bi_encoder_score
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+
+from crp.core.task_intent import TaskIntent
+
+from .scoring import ScoredFact
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+TOP_K_RERANK = 200
+MIN_FACTS_FOR_RERANK = 10
+CE_WEIGHT = 0.6
+BI_WEIGHT = 0.4
+IDLE_UNLOAD_WINDOWS = 10
+TASK_SIM_INVALIDATION_THRESHOLD = 0.9
+
+# ---------------------------------------------------------------------------
+# Cross-encoder model management (lazy load, idle unload)
+# ---------------------------------------------------------------------------
+
+_CE_MODEL: object | None = None
+_CE_AVAILABLE: bool | None = None  # None = not yet attempted
+_CE_LAST_USED_WINDOW: int = 0
+
+
+def _try_load_cross_encoder() -> bool:
+    """Attempt lazy load of cross-encoder model.  Returns True on success."""
+    global _CE_MODEL, _CE_AVAILABLE  # noqa: PLW0603
+    if _CE_AVAILABLE is True:
+        return True
+    if _CE_AVAILABLE is False:
+        return False
+    try:
+        from sentence_transformers import CrossEncoder  # type: ignore[import-untyped]
+
+        _CE_MODEL = CrossEncoder("cross-encoder/ms-marco-MiniLM-L6-v2")
+        _CE_AVAILABLE = True
+    except Exception:  # noqa: BLE001
+        _CE_AVAILABLE = False
+    return _CE_AVAILABLE or False
+
+
+def maybe_unload_cross_encoder(current_window: int) -> None:
+    """Unload cross-encoder if idle for > IDLE_UNLOAD_WINDOWS."""
+    global _CE_MODEL, _CE_AVAILABLE, _CE_LAST_USED_WINDOW  # noqa: PLW0603
+    if _CE_MODEL is not None and (current_window - _CE_LAST_USED_WINDOW) > IDLE_UNLOAD_WINDOWS:
+        _CE_MODEL = None
+        _CE_AVAILABLE = None
+
+
+def preload_cross_encoder() -> bool:
+    """Eagerly load the cross-encoder model at startup to avoid cold-start delay.
+
+    Returns True if the model was loaded successfully, False otherwise.
+    Called from CRPOrchestrator.__init__() to pay the ~5s load cost once
+    at protocol start rather than during the first rerank operation.
+    """
+    return _try_load_cross_encoder()
+
+
+# ---------------------------------------------------------------------------
+# CrossEncoderCache
+# ---------------------------------------------------------------------------
+
+
+def _task_hash(task_intent: TaskIntent) -> str:
+    """Deterministic hash of the task for cache keying."""
+    parts = (task_intent.system_prompt or "") + "|" + (task_intent.task_input or "")
+    return hashlib.sha256(parts.encode()).hexdigest()[:16]
+
+
+@dataclass
+class CrossEncoderCache:
+    """Per-session cache for cross-encoder scores.
+
+    Key: (task_hash, fact_id).  Invalidation rules:
+    - Full clear on compaction.
+    - Remove entry on fact supersession.
+    - Full clear if task similarity < 0.9 (compared via hash change).
+    """
+
+    _cache: dict[tuple[str, str], float] = field(default_factory=dict)
+    _last_task_hash: str = ""
+
+    def get(self, t_hash: str, fact_id: str) -> float | None:
+        return self._cache.get((t_hash, fact_id))
+
+    def put(self, t_hash: str, fact_id: str, score: float) -> None:
+        self._cache[(t_hash, fact_id)] = score
+
+    def invalidate_fact(self, fact_id: str) -> None:
+        """Remove all entries for a superseded fact."""
+        keys = [k for k in self._cache if k[1] == fact_id]
+        for k in keys:
+            del self._cache[k]
+
+    def invalidate_all(self) -> None:
+        """Full cache clear (compaction or task change)."""
+        self._cache.clear()
+        self._last_task_hash = ""
+
+    def check_task_change(self, new_task_hash: str) -> None:
+        """If task hash changed, invalidate entire cache."""
+        if self._last_task_hash and self._last_task_hash != new_task_hash:
+            self.invalidate_all()
+        self._last_task_hash = new_task_hash
+
+    @property
+    def size(self) -> int:
+        return len(self._cache)
+
+
+# ---------------------------------------------------------------------------
+# Reranking
+# ---------------------------------------------------------------------------
+
+
+def rerank(
+    scored_facts: list[ScoredFact],
+    task_intent: TaskIntent,
+    *,
+    cache: CrossEncoderCache | None = None,
+    current_window: int = 0,
+) -> list[ScoredFact]:
+    """Rerank *scored_facts* using the cross-encoder.
+
+    - Only top TOP_K_RERANK facts are reranked.
+    - If fewer than MIN_FACTS_FOR_RERANK facts, skip reranking entirely.
+    - Falls back to bi-encoder scores if the model is unavailable.
+
+    Returns all facts sorted by blended score descending.
+    """
+    global _CE_LAST_USED_WINDOW  # noqa: PLW0603
+
+    if not scored_facts:
+        return []
+
+    # If too few facts, skip reranking — bi-encoder is sufficient
+    if len(scored_facts) < MIN_FACTS_FOR_RERANK:
+        return scored_facts  # already sorted by bi-encoder score
+
+    # Select candidates for reranking
+    candidates = scored_facts[:TOP_K_RERANK]
+    remainder = scored_facts[TOP_K_RERANK:]
+
+    # Attempt cross-encoder
+    if not _try_load_cross_encoder() or _CE_MODEL is None:
+        return scored_facts  # fallback: bi-encoder order preserved
+
+    _CE_LAST_USED_WINDOW = current_window
+
+    t_hash = _task_hash(task_intent)
+    if cache is not None:
+        cache.check_task_change(t_hash)
+
+    # Build query text
+    query = (task_intent.system_prompt or "") + " " + (task_intent.task_input or "")
+    query = query.strip()
+
+    # Score pairs (using cache where available)
+    pairs_to_score: list[tuple[int, str]] = []  # (index_in_candidates, fact_text)
+    ce_scores: dict[int, float] = {}
+
+    for i, sf in enumerate(candidates):
+        cached = cache.get(t_hash, sf.fact.id) if cache else None
+        if cached is not None:
+            ce_scores[i] = cached
+        else:
+            pairs_to_score.append((i, sf.fact.text))
+
+    # Batch predict uncached pairs
+    if pairs_to_score:
+        input_pairs = [(query, text) for _, text in pairs_to_score]
+        try:
+            raw_scores = _CE_MODEL.predict(input_pairs)  # type: ignore[attr-defined]
+            for (idx, _), raw in zip(pairs_to_score, raw_scores):
+                score_val = float(raw)
+                ce_scores[idx] = score_val
+                if cache is not None:
+                    cache.put(t_hash, candidates[idx].fact.id, score_val)
+        except Exception:  # noqa: BLE001
+            # Model failure — fall back to bi-encoder scores
+            return scored_facts
+
+    # Compute blended scores
+    for i, sf in enumerate(candidates):
+        ce_score = ce_scores.get(i, 0.0)
+        sf.composite_score = CE_WEIGHT * ce_score + BI_WEIGHT * sf.composite_score
+
+    # Re-sort candidates by blended score
+    candidates.sort(key=lambda s: s.composite_score, reverse=True)
+
+    # Append remainder (already below top-K, keep original order)
+    return candidates + remainder