context-engineering-core 0.12.0__py3-none-any.whl

context_engine/layers/__init__.py

@@ -0,0 +1,18 @@
+"""Five independently callable context layers. No inference is performed."""
+
+from .cap import CapLayer
+from .pin import PinLayer
+from .retrieve import RetrieveLayer
+from .summarize import FrozenSummaryPolicy, PrefixSummaryPolicy, SummaryLayer, SummarySnapshot
+from .window import WindowLayer
+
+__all__ = [
+    "CapLayer",
+    "PinLayer",
+    "RetrieveLayer",
+    "WindowLayer",
+    "SummaryLayer",
+    "FrozenSummaryPolicy",
+    "PrefixSummaryPolicy",
+    "SummarySnapshot",
+]

context_engine/layers/cap.py

@@ -0,0 +1,99 @@
+"""Deterministic code-point-safe head/tail compression of working messages."""
+
+from dataclasses import dataclass, replace
+
+from ..config import CapConfig
+from ..context import ContextCarrier, LayerDiagnostic
+from ..errors import CapBudgetError, LayerStateError
+from ..tokens import TokenCounter
+from ..work import Cancellation, raise_if_cancelled
+
+
+def bounded_piece(
+    text: str,
+    allowance: int,
+    counter: TokenCounter,
+    *,
+    tail: bool = False,
+    cancellation: Cancellation | None = None,
+) -> str:
+    """Find a fitting prefix/suffix; never assume BPE counts are strictly monotone."""
+    low, high = 0, len(text)
+    best = ""
+    while low <= high:
+        raise_if_cancelled(cancellation)
+        middle = (low + high) // 2
+        candidate = text[len(text) - middle :] if tail else text[:middle]
+        if counter.count_text(candidate) <= allowance:
+            best = candidate
+            low = middle + 1
+        else:
+            high = middle - 1
+    return best
+
+
+@dataclass(frozen=True)
+class CapLayer:
+    counter: TokenCounter
+    config: CapConfig = CapConfig()
+
+    def cap_text(self, text: str, *, cancellation: Cancellation | None = None) -> str:
+        raise_if_cancelled(cancellation)
+        if self.counter.count_text(text) <= self.config.max_tokens:
+            return text
+        head = bounded_piece(text, self.config.head_tokens, self.counter, cancellation=cancellation)
+        tail = bounded_piece(
+            text, self.config.tail_tokens, self.counter, tail=True, cancellation=cancellation
+        )
+        # At least one original code point at both ends is required for a capped message.
+        head, tail = head or text[:1], tail or text[-1:]
+        while head and tail:
+            raise_if_cancelled(cancellation)
+            end = len(text) - len(tail)
+            omitted = text[len(head) : end]
+            marker = f"\n[... {self.counter.count_text(omitted)} tokens elided ...]\n"
+            candidate = head + marker + tail
+            if omitted and self.counter.count_text(candidate) <= self.config.max_tokens:
+                return candidate
+            if len(head) >= len(tail) and len(head) > 1:
+                head = head[:-1]
+            elif len(tail) > 1:
+                tail = tail[1:]
+            else:
+                break
+        raise CapBudgetError(
+            "CAP allocation cannot fit a head, elision marker and tail",
+            max_tokens=self.config.max_tokens,
+        )
+
+    def apply(self, context: ContextCarrier) -> ContextCarrier:
+        if context.plan is not None:
+            raise LayerStateError("CAP must run before budget planning")
+        turns, changed = [], []
+        before = after = 0
+        for turn in context.original_turns:
+            raise_if_cancelled(context.cancellation)
+            messages = []
+            for message in turn.messages:
+                content = self.cap_text(message.content, cancellation=context.cancellation)
+                before += self.counter.count_text(message.content)
+                after += self.counter.count_text(content)
+                messages.append(replace(message, content=content))
+            working = replace(turn, messages=tuple(messages))
+            turns.append(working)
+            if working != turn:
+                changed.append(turn.turn_id)
+        return replace(
+            context,
+            working_turns=tuple(turns),
+            diagnostics=context.diagnostics
+            + (
+                LayerDiagnostic(
+                    "CAP",
+                    "messages_capped" if changed else "unchanged",
+                    tuple(changed),
+                    before,
+                    after,
+                ),
+            ),
+        )

context_engine/layers/common.py

@@ -0,0 +1,162 @@
+"""Shared rendering/accounting primitives used by layers and the public pipeline."""
+
+import hashlib
+from dataclasses import asdict, replace
+
+from ..context import ContextCarrier, LayerDiagnostic
+from ..errors import LayerStateError
+from ..models import (
+    BlockKind,
+    ChatRequest,
+    ContextBlock,
+    Message,
+    Role,
+    ToolDefinition,
+    Turn,
+    canonical_json,
+)
+from ..tokens import TokenCounter
+from ..work import raise_if_cancelled
+
+ORDER = tuple(BlockKind)
+OPTIONAL = (BlockKind.SUMMARY, BlockKind.WINDOW, BlockKind.RETRIEVED)
+
+
+def history_digest(turns: tuple[Turn, ...]) -> str:
+    values = [
+        (asdict(t.scope), t.turn_id, t.revision, t.timestamp.isoformat(), t.content_hash)
+        for t in turns
+    ]
+    return hashlib.sha256(canonical_json(values).encode()).hexdigest()
+
+
+def input_digest(context: ContextCarrier) -> str:
+    pins = []
+    for pin in context.pins.pins:
+        value = asdict(pin)
+        value["effective_at"] = pin.effective_at.isoformat()
+        value["expires_at"] = pin.expires_at.isoformat() if pin.expires_at else None
+        pins.append(value)
+    values = {
+        "scope": asdict(context.scope),
+        "original": history_digest(context.original_turns),
+        "working": history_digest(context.working_turns),
+        "pins": pins,
+        "question": context.question.to_wire(),
+        "question_id": context.question.message_id,
+    }
+    return hashlib.sha256(canonical_json(values).encode()).hexdigest()
+
+
+def render_request(
+    blocks: tuple[ContextBlock, ...], question: Message, tools: tuple[ToolDefinition, ...] = ()
+) -> ChatRequest:
+    """Quote all historical/evidence blocks as user data, including historical tool calls."""
+    by_kind = {block.kind: block for block in blocks}
+    messages = []
+    for kind in ORDER:
+        if kind == BlockKind.QUESTION:
+            messages.append(Message("ctx:question", Role.USER, question.content))
+            continue
+        block = by_kind.get(kind)
+        if block is None or not block.content:
+            continue
+        role = Role.SYSTEM if kind == BlockKind.SYSTEM else Role.USER
+        content = (
+            block.content
+            if kind == BlockKind.SYSTEM
+            else f"[{kind.value.upper()} DATA]\n{block.content}"
+        )
+        messages.append(Message(f"ctx:{kind.value}", role, content))
+    return ChatRequest(tuple(messages), tools)
+
+
+def required_blocks(context: ContextCarrier) -> tuple[ContextBlock, ...]:
+    return tuple(b for b in context.blocks if b.kind not in OPTIONAL)
+
+
+def require_prepared(context: ContextCarrier, counter: TokenCounter) -> None:
+    if context.plan is None or context.required_request is None:
+        raise LayerStateError("PIN must reserve mandatory context first")
+    if context.prepared_fingerprint != input_digest(context):
+        raise LayerStateError("History, question or pins changed after budget planning; rerun PIN")
+    request = render_request(
+        required_blocks(context), context.question, context.required_request.tools
+    )
+    if (
+        request != context.required_request
+        or counter.count_request(request) != context.plan.required_estimate
+    ):
+        raise LayerStateError("Mandatory blocks or counting profile changed after planning")
+
+
+def set_block(
+    context: ContextCarrier, block: ContextBlock, diagnostic: LayerDiagnostic
+) -> ContextCarrier:
+    blocks = tuple(b for b in context.blocks if b.kind != block.kind) + (block,)
+    return replace(
+        context,
+        blocks=tuple(sorted(blocks, key=lambda b: ORDER.index(b.kind))),
+        diagnostics=context.diagnostics + (diagnostic,),
+    )
+
+
+def window_block(
+    context: ContextCarrier, ids: tuple[str, ...], counter: TokenCounter
+) -> ContextBlock:
+    selected = [t for t in context.working_turns if t.turn_id in ids]
+    content = (
+        canonical_json(
+            [{"turn": t.turn_id, "messages": [m.to_wire() for m in t.messages]} for t in selected]
+        )
+        if selected
+        else ""
+    )
+    return ContextBlock(BlockKind.WINDOW, content, text_tokens=counter.count_text(content))
+
+
+def incremental_cost(context: ContextCarrier, block: ContextBlock, counter: TokenCounter) -> int:
+    request = render_request(
+        required_blocks(context) + (block,), context.question, context.required_request.tools
+    )
+    return max(
+        0,
+        counter.count_request(request).estimated_tokens
+        - context.plan.required_estimate.estimated_tokens,
+    )
+
+
+def fits_block(
+    context: ContextCarrier, block: ContextBlock, allowance: int, counter: TokenCounter
+) -> bool:
+    if incremental_cost(context, block, counter) > allowance:
+        return False
+    blocks = tuple(b for b in context.blocks if b.kind != block.kind) + (block,)
+    # RETRIEVE runs first but must leave the exact planned WINDOW viable.
+    if context.window_planned and block.kind != BlockKind.WINDOW:
+        blocks = tuple(b for b in blocks if b.kind != BlockKind.WINDOW) + (
+            window_block(context, context.plan.window_turn_ids, counter),
+        )
+    request = render_request(blocks, context.question, context.required_request.tools)
+    return counter.count_request(request).estimated_tokens <= context.plan.input_allowance
+
+
+def plan_window(context: ContextCarrier, counter: TokenCounter) -> ContextCarrier:
+    require_prepared(context, counter)
+    selected: tuple[str, ...] = ()
+    for turn in reversed(context.working_turns):
+        raise_if_cancelled(context.cancellation)
+        candidate = (turn.turn_id,) + selected
+        if (
+            incremental_cost(context, window_block(context, candidate, counter), counter)
+            > context.plan.window_tokens
+        ):
+            break
+        selected = candidate
+    if context.window_planned:
+        if selected != context.plan.window_turn_ids:
+            raise LayerStateError("Stored WINDOW membership differs from shared planner")
+        return context
+    return replace(
+        context, plan=replace(context.plan, window_turn_ids=selected), window_planned=True
+    )

context_engine/layers/pin.py

@@ -0,0 +1,68 @@
+"""Reserve supplied instructions, active keyed facts, the question and tool schemas."""
+
+from dataclasses import dataclass, replace
+from datetime import datetime
+
+from ..budget import plan_budget
+from ..config import BudgetConfig
+from ..context import ContextCarrier, LayerDiagnostic
+from ..models import BlockKind, ContextBlock, ToolDefinition, aware_time, canonical_json, text_value
+from ..tokens import TokenCounter
+from .common import input_digest, render_request
+
+
+@dataclass(frozen=True)
+class PinLayer:
+    counter: TokenCounter
+    budget: BudgetConfig
+    system: str
+    at: datetime
+    tools: tuple[ToolDefinition, ...] = ()
+
+    def __post_init__(self) -> None:
+        text_value("system", self.system)
+        aware_time("at", self.at)
+        object.__setattr__(self, "tools", tuple(self.tools))
+
+    def apply(self, context: ContextCarrier) -> ContextCarrier:
+        active = tuple(pin for pin in context.pins.pins if pin.is_active(self.at))
+        content = (
+            canonical_json([{"key": pin.key, "value": pin.value} for pin in active])
+            if active
+            else ""
+        )
+        blocks = (
+            ContextBlock(
+                BlockKind.SYSTEM,
+                self.system,
+                prefix_stable=True,
+                text_tokens=self.counter.count_text(self.system),
+            ),
+            ContextBlock(
+                BlockKind.PINNED,
+                content,
+                sources=tuple(pin.source for pin in active if pin.source is not None),
+                prefix_stable=True,
+                text_tokens=self.counter.count_text(content),
+            ),
+        )
+        request = render_request(blocks, context.question, self.tools)
+        plan = plan_budget(required_request=request, config=self.budget, counter=self.counter)
+        # Re-running PIN deliberately invalidates all query-dependent layer outputs.
+        return replace(
+            context,
+            plan=plan,
+            blocks=blocks,
+            required_request=request,
+            prepared_fingerprint=input_digest(context),
+            window_planned=False,
+            retrieved_chunks=(),
+            diagnostics=context.diagnostics
+            + (
+                LayerDiagnostic(
+                    "PIN",
+                    "required_context_reserved",
+                    output_text_tokens=sum(b.text_tokens for b in blocks),
+                ),
+            ),
+        )

context_engine/layers/retrieve.py

@@ -0,0 +1,233 @@
+"""BM25+ over originals; optional recovery of missing chunks in capped WINDOW turns."""
+
+import hashlib
+import re
+import unicodedata
+from dataclasses import asdict, dataclass, replace
+
+from rank_bm25 import BM25Plus
+
+from ..config import RetrievalConfig
+from ..context import ContextCarrier, LayerDiagnostic
+from ..errors import MemoryIntegrityError
+from ..models import BlockKind, Chunk, ContextBlock, SourceRef, Turn, canonical_json
+from ..tokens import TokenCounter
+from ..work import Cancellation, raise_if_cancelled, validate_cancellation
+from .common import fits_block, history_digest, plan_window, set_block
+
+
+@dataclass(frozen=True)
+class ChunkIndex:
+    history_fingerprint: str
+    config_fingerprint: str
+    chunks: tuple[Chunk, ...]
+
+    def validate(self, history, config):
+        expected = hashlib.sha256(canonical_json(asdict(config)).encode()).hexdigest()
+        if (
+            self.history_fingerprint != history_digest(history)
+            or self.config_fingerprint != expected
+        ):
+            raise MemoryIntegrityError("Chunk index history or configuration is stale")
+        originals = {t.turn_id: t for t in history}
+        if (
+            not isinstance(self.chunks, tuple)
+            or any(not isinstance(c, Chunk) for c in self.chunks)
+            or len({c.chunk_id for c in self.chunks}) != len(self.chunks)
+        ):
+            raise MemoryIntegrityError("Invalid chunk index entries")
+        for chunk in self.chunks:
+            if not isinstance(chunk, Chunk) or chunk.source.turn_id not in originals:
+                raise MemoryIntegrityError("Chunk index contains unknown source")
+            chunk.validate_source(originals[chunk.source.turn_id])
+
+
+def lexical_tokens(text: str) -> list[str]:
+    normalized = unicodedata.normalize("NFKC", text).casefold()
+    return re.findall(r"\w+(?:[-.]\w+)*", normalized)
+
+
+def chunk_turns(
+    turns: tuple[Turn, ...], config: RetrievalConfig, *, cancellation: Cancellation | None = None
+) -> tuple[Chunk, ...]:
+    validate_cancellation(cancellation)
+    raise_if_cancelled(cancellation)
+    chunks = []
+    version = f"characters-v1:{config.chunk_characters}:{config.overlap_characters}"
+    stride = config.chunk_characters - config.overlap_characters
+    for turn in turns:
+        raise_if_cancelled(cancellation)
+        for message in turn.messages:
+            raise_if_cancelled(cancellation)
+            message_hash = message.content_hash
+            for start in range(0, len(message.content), stride):
+                raise_if_cancelled(cancellation)
+                end = min(len(message.content), start + config.chunk_characters)
+                content = message.content[start:end]
+                if content.strip():
+                    source = SourceRef(
+                        turn.scope,
+                        turn.turn_id,
+                        message.message_id,
+                        turn.revision,
+                        start,
+                        end,
+                        message_hash,
+                    )
+                    identifier = hashlib.sha256(
+                        canonical_json([asdict(source), version]).encode()
+                    ).hexdigest()
+                    chunks.append(Chunk(identifier, source, content, version))
+                if end == len(message.content):
+                    break
+    return tuple(chunks)
+
+
+def evidence_block(chunks: tuple[Chunk, ...], counter: TokenCounter) -> ContextBlock:
+    content = (
+        canonical_json(
+            [
+                {
+                    "turn": c.source.turn_id,
+                    "message": c.source.message_id,
+                    "start": c.source.start,
+                    "end": c.source.end,
+                    "text": c.content,
+                }
+                for c in chunks
+            ]
+        )
+        if chunks
+        else ""
+    )
+    return ContextBlock(
+        BlockKind.RETRIEVED,
+        content,
+        sources=tuple(c.source for c in chunks),
+        text_tokens=counter.count_text(content),
+    )
+
+
+def overlaps(left: Chunk, right: Chunk) -> bool:
+    a, b = left.source, right.source
+    return (a.scope, a.turn_id, a.message_id, a.revision) == (
+        b.scope,
+        b.turn_id,
+        b.message_id,
+        b.revision,
+    ) and max(a.start, b.start) < min(a.end, b.end)
+
+
+def window_messages(context: ContextCarrier) -> dict[tuple[str, str], str]:
+    """Actual working content, not full-original provenance ranges on WINDOW blocks."""
+    return {
+        (turn.turn_id, message.message_id): message.content
+        for turn in context.working_turns
+        if turn.turn_id in context.plan.window_turn_ids
+        for message in turn.messages
+    }
+
+
+def represented_in_window(chunk: Chunk, visible: dict[tuple[str, str], str]) -> bool:
+    text = visible.get((chunk.source.turn_id, chunk.source.message_id))
+    return text is not None and chunk.content in text
+
+
+@dataclass(frozen=True)
+class RetrieveLayer:
+    counter: TokenCounter
+    config: RetrievalConfig = RetrievalConfig()
+    chunk_index: ChunkIndex | None = None
+
+    def apply(self, context: ContextCarrier) -> ContextCarrier:
+        if self.chunk_index is not None:
+            if not isinstance(self.chunk_index, ChunkIndex):
+                raise MemoryIntegrityError("Expected a validated chunk index")
+            self.chunk_index.validate(context.original_turns, self.config)
+        planned = plan_window(context, self.counter)
+        visible = window_messages(planned)
+        old = tuple(
+            t
+            for t in planned.original_turns
+            if t.turn_id not in planned.plan.window_turn_ids
+            or (
+                self.config.recover_capped_window
+                and any(m.content != visible[(t.turn_id, m.message_id)] for m in t.messages)
+            )
+        )
+        query = sorted(set(lexical_tokens(planned.question.content)))
+        selected: tuple[Chunk, ...] = ()
+        reason = "no_match"
+        if not query:
+            reason = "empty_query"
+        elif not old:
+            reason = "no_old_history"
+        elif planned.plan.retrieval_tokens == 0:
+            reason = "zero_reserve"
+        else:
+            if self.chunk_index is None:
+                # Preserve the exact 0.8.0 call form when no token is supplied.
+                chunks = (
+                    chunk_turns(old, self.config)
+                    if context.cancellation is None
+                    else chunk_turns(old, self.config, cancellation=context.cancellation)
+                )
+            else:
+                ids = {t.turn_id for t in old}
+                chunks = tuple(c for c in self.chunk_index.chunks if c.source.turn_id in ids)
+            searchable = []
+            for chunk in chunks:
+                raise_if_cancelled(context.cancellation)
+                if represented_in_window(chunk, visible):
+                    continue
+                terms = lexical_tokens(chunk.content)
+                if terms:
+                    searchable.append((chunk, terms))
+            if searchable:
+                # BM25+ has positive IDF for one-document/small corpora. Its delta
+                # also scores nonmatches: explicitly require lexical intersection.
+                ranker = BM25Plus(
+                    [terms for _, terms in searchable],
+                    k1=self.config.k1,
+                    b=self.config.b,
+                    delta=1.0,
+                )
+                scores = ranker.get_scores(query)
+                raise_if_cancelled(context.cancellation)
+                ranked = [
+                    (float(score), c)
+                    for (c, terms), score in zip(searchable, scores, strict=True)
+                    if set(query).intersection(terms) and score > self.config.minimum_score
+                ]
+                ranked.sort(key=lambda item: (-item[0], item[1].chunk_id))
+                raise_if_cancelled(context.cancellation)
+                if ranked:
+                    reason = "evidence_over_budget"
+                for _, chunk in ranked:
+                    raise_if_cancelled(context.cancellation)
+                    if any(overlaps(chunk, existing) for existing in selected):
+                        continue
+                    candidate = selected + (chunk,)
+                    if fits_block(
+                        planned,
+                        evidence_block(candidate, self.counter),
+                        planned.plan.retrieval_tokens,
+                        self.counter,
+                    ):
+                        selected = candidate
+                    if len(selected) >= self.config.top_k:
+                        break
+        if selected:
+            reason = "evidence_retrieved"
+        block = evidence_block(selected, self.counter)
+        result = set_block(
+            planned,
+            block,
+            LayerDiagnostic(
+                "RETRIEVE",
+                reason,
+                tuple(dict.fromkeys(c.source.turn_id for c in selected)),
+                output_text_tokens=block.text_tokens,
+            ),
+        )
+        return replace(result, retrieved_chunks=selected)