token-runtime 0.1.0a2__py3-none-any.whl

token_runtime/anthropic_conformance.py

@@ -0,0 +1,336 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, replace
+
+from .anthropic_adapter import AnthropicMessagesAdapter
+from .capabilities import (
+    CapabilityDetector,
+    CapabilityKey,
+    CapabilityProfile,
+    CapabilityRegistry,
+)
+from .compatibility import (
+    CompatibilityRecord,
+    CompatibilityRegistry,
+    CompatibilityState,
+)
+from .conformance import ConformanceResult, require_conformance, run_conformance
+
+
+ANTHROPIC_MESSAGES_KEY = CapabilityKey(
+    "generic-anthropic",
+    "anthropic_messages",
+    "anthropic",
+)
+EVIDENCE_ID = "token-anthropic-adapter-1:offline-request-boundary-v1"
+
+
+@dataclass(frozen=True, slots=True)
+class AnthropicConformanceBundle:
+    profile: CapabilityProfile
+    record: CompatibilityRecord
+    detector: CapabilityDetector
+
+
+def build_anthropic_conformance() -> AnthropicConformanceBundle:
+    profile = CapabilityProfile(
+        key=ANTHROPIC_MESSAGES_KEY,
+        prompt_cache="request-markers-preserved",
+        tool_calls=True,
+        reasoning_state="exact-replay-required",
+        exact_byte_preservation=True,
+        evidence_id=EVIDENCE_ID,
+    )
+    record = CompatibilityRecord(
+        key=ANTHROPIC_MESSAGES_KEY,
+        state=CompatibilityState.PASSTHROUGH_ONLY,
+        evidence_id=EVIDENCE_ID,
+        reason="offline_conformance_only",
+    )
+    capabilities = CapabilityRegistry()
+    capabilities.register(profile)
+    compatibility = CompatibilityRegistry()
+    compatibility.register(record)
+    return AnthropicConformanceBundle(
+        profile=profile,
+        record=record,
+        detector=CapabilityDetector(capabilities, compatibility),
+    )
+
+
+def _supported_fixture() -> dict[str, object]:
+    return {
+        "model": "opaque-model",
+        "max_tokens": 256,
+        "system": [
+            {
+                "type": "text",
+                "text": "stable policy",
+                "cache_control": {"type": "ephemeral"},
+            }
+        ],
+        "messages": [
+            {"role": "user", "content": "current task"},
+            {
+                "role": "assistant",
+                "content": [
+                    {
+                        "type": "tool_use",
+                        "id": "toolu_1",
+                        "name": "query",
+                        "input": {"q": "status"},
+                    }
+                ],
+            },
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "tool_result",
+                        "tool_use_id": "toolu_1",
+                        "content": "status=green",
+                    }
+                ],
+            },
+        ],
+        "tools": [
+            {
+                "name": "query",
+                "description": "fixture",
+                "input_schema": {"type": "object"},
+                "cache_control": {"type": "ephemeral"},
+            }
+        ],
+        "stream": True,
+        "future_field": {"preserve": True},
+    }
+
+
+def _supported_roundtrip() -> bool:
+    payload = _supported_fixture()
+    adapter = AnthropicMessagesAdapter()
+    envelope = adapter.parse(payload)
+    return envelope.wire_safe and adapter.serialize(envelope) == payload
+
+
+def _cache_markers_preserved() -> bool:
+    payload = _supported_fixture()
+    adapter = AnthropicMessagesAdapter()
+    envelope = adapter.parse(payload)
+    serialized = adapter.serialize(envelope)
+    return (
+        serialized == payload
+        and serialized["system"][0]["cache_control"] == {"type": "ephemeral"}
+        and serialized["tools"][0]["cache_control"] == {"type": "ephemeral"}
+    )
+
+
+
+def _text_path_mutation_preserves_protocol() -> bool:
+    payload = _supported_fixture()
+    adapter = AnthropicMessagesAdapter()
+    envelope = adapter.parse(payload)
+    target = next(block for block in envelope.blocks if block.text == "current task")
+    changed = replace(
+        envelope,
+        blocks=tuple(
+            replace(block, text="updated task") if block is target else block
+            for block in envelope.blocks
+        ),
+    )
+    serialized = adapter.serialize(changed)
+    return (
+        serialized["messages"][0]["content"] == "updated task"
+        and serialized["messages"][1:] == payload["messages"][1:]
+        and serialized["system"] == payload["system"]
+        and serialized["tools"] == payload["tools"]
+        and serialized["future_field"] == payload["future_field"]
+    )
+
+
+def _tool_classification() -> bool:
+    adapter = AnthropicMessagesAdapter()
+    envelope = adapter.parse(_supported_fixture())
+    tool_call = next(block for block in envelope.blocks if block.kind == "tool_call")
+    tool_output = next(block for block in envelope.blocks if block.kind == "tool_output")
+    return (
+        envelope.wire_safe
+        and "path" not in tool_call.metadata
+        and tool_output.role == "tool"
+        and tool_output.metadata.get("path")
+        == ("messages", 2, "content", 0, "content")
+    )
+
+
+def _stream_preserved() -> bool:
+    payload = _supported_fixture()
+    adapter = AnthropicMessagesAdapter()
+    envelope = adapter.parse(payload)
+    serialized = adapter.serialize(envelope)
+    return envelope.wire_safe and serialized.get("stream") is True
+
+
+def _near_match_non_inheritance() -> bool:
+    bundle = build_anthropic_conformance()
+    near_matches = (
+        CapabilityKey(
+            "generic-anthropic",
+            "anthropic_messages",
+            "anthropic",
+            model_family="claude-future",
+        ),
+        CapabilityKey("generic-anthropic", "anthropic_messages", "anthropic-v2"),
+    )
+    for key in near_matches:
+        detected = bundle.detector.detect(key)
+        if detected.profile is not None:
+            return False
+        if detected.compatibility.state is not CompatibilityState.PASSTHROUGH_ONLY:
+            return False
+        if detected.compatibility.reason != "unknown_capability":
+            return False
+    return True
+
+
+def _cited_text_exact_passthrough() -> bool:
+    payload = {
+        "model": "opaque-model",
+        "max_tokens": 128,
+        "system": [
+            {
+                "type": "text",
+                "text": "cited system",
+                "citations": [{"type": "char_location", "cited_text": "cited"}],
+            }
+        ],
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": "cited message",
+                        "citations": [
+                            {"type": "char_location", "cited_text": "cited"}
+                        ],
+                    }
+                ],
+            },
+            {
+                "role": "assistant",
+                "content": [
+                    {
+                        "type": "tool_use",
+                        "id": "toolu_cited",
+                        "name": "query",
+                        "input": {},
+                    }
+                ],
+            },
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "tool_result",
+                        "tool_use_id": "toolu_cited",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "cited tool result",
+                                "citations": [
+                                    {
+                                        "type": "char_location",
+                                        "cited_text": "cited",
+                                    }
+                                ],
+                            }
+                        ],
+                    }
+                ],
+            },
+        ],
+    }
+    adapter = AnthropicMessagesAdapter()
+    envelope = adapter.parse(payload)
+    cited_blocks = [block for block in envelope.blocks if block.text.startswith("cited")]
+    return (
+        not envelope.wire_safe
+        and len(cited_blocks) == 3
+        and all("path" not in block.metadata for block in cited_blocks)
+        and adapter.serialize(envelope) == payload
+    )
+
+def _thinking_exact_passthrough() -> bool:
+    payload = {
+        "model": "opaque-model",
+        "max_tokens": 64,
+        "messages": [
+            {
+                "role": "assistant",
+                "content": [
+                    {"type": "thinking", "thinking": "opaque", "signature": "sig"}
+                ],
+            },
+            {"role": "user", "content": "continue"},
+        ],
+    }
+    adapter = AnthropicMessagesAdapter()
+    envelope = adapter.parse(payload)
+    return not envelope.wire_safe and adapter.serialize(envelope) == payload
+
+
+def _unknown_multimodal_passthrough() -> bool:
+    payload = {
+        "model": "opaque-model",
+        "max_tokens": 64,
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image",
+                        "source": {
+                            "type": "base64",
+                            "media_type": "image/png",
+                            "data": "fixture",
+                        },
+                    }
+                ],
+            }
+        ],
+        "future_field": {"preserve": True},
+    }
+    adapter = AnthropicMessagesAdapter()
+    envelope = adapter.parse(payload)
+    return not envelope.wire_safe and adapter.serialize(envelope) == payload
+
+
+def _certification_state_passthrough_only() -> bool:
+    bundle = build_anthropic_conformance()
+    return (
+        bundle.record.state is CompatibilityState.PASSTHROUGH_ONLY
+        and bundle.record.reason == "offline_conformance_only"
+        and bundle.profile.evidence_id == bundle.record.evidence_id
+    )
+
+
+def anthropic_conformance_cases() -> Mapping[str, Callable[[], bool]]:
+    return {
+        "cache_markers_preserved": _cache_markers_preserved,
+        "certification_state_passthrough_only": _certification_state_passthrough_only,
+        "cited_text_exact_passthrough": _cited_text_exact_passthrough,
+        "near_match_non_inheritance": _near_match_non_inheritance,
+        "stream_preserved": _stream_preserved,
+        "supported_roundtrip": _supported_roundtrip,
+        "text_path_mutation_preserves_protocol": _text_path_mutation_preserves_protocol,
+        "thinking_exact_passthrough": _thinking_exact_passthrough,
+        "tool_classification": _tool_classification,
+        "unknown_multimodal_passthrough": _unknown_multimodal_passthrough,
+    }
+
+
+def run_anthropic_conformance() -> tuple[ConformanceResult, ...]:
+    results = run_conformance(anthropic_conformance_cases())
+    require_conformance(results)
+    return results

token_runtime/anthropic_preserved_thinking_cert.py

@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, fields
+from decimal import Decimal, InvalidOperation
+from hashlib import sha256
+import json
+
+from .capabilities import CapabilityKey
+from .compat_cert import BenchmarkGeneration
+from .compatibility import CompatibilityRecord, CompatibilityState
+
+
+MODEL_ID = "claude-fable-5-1"
+BINDING_BETA = "thinking-binding-controls-2026-08-01"
+BINDING_MODE = "drop_block"
+GENERATION_ID = "token-anthropic-preserved-thinking-cert-1:pt-v1"
+CORPUS_VERSION = "anthropic-preserved-thinking-v1"
+EVALUATOR_VERSION = "anthropic-pt-eval-v2"
+POLICY_VERSION = "compat-policy-v1"
+ACCOUNTING_STATES = frozenset({
+    "unavailable",
+    "estimated_from_response_usage",
+    "usage_reconciled",
+    "billed_workspace_delta_reconciled",
+})
+
+ANTHROPIC_PRESERVED_THINKING_KEY = CapabilityKey(
+    "generic-anthropic", "anthropic_messages", "anthropic", model_family=MODEL_ID
+)
+
+
+def _require_evidence_id(value: str) -> None:
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError("evidence_id must be non-blank")
+
+
+def _require_bool_fields(instance: object, names: tuple[str, ...]) -> None:
+    values = {item.name: getattr(instance, item.name) for item in fields(instance)}
+    for name in names:
+        if type(values[name]) is not bool:
+            raise TypeError(f"{name} must be bool")
+
+
+def _require_nonnegative_finite_decimal(value: str, name: str) -> Decimal:
+    if not isinstance(value, str):
+        raise ValueError(f"{name} must be a decimal string")
+    try:
+        parsed = Decimal(value)
+    except (InvalidOperation, ValueError) as exc:
+        raise ValueError(f"{name} must be a decimal string") from exc
+    if not parsed.is_finite() or parsed < 0:
+        raise ValueError(f"{name} must be finite and non-negative")
+    return parsed
+
+
+@dataclass(frozen=True, slots=True)
+class AnthropicPreservedThinkingOfflineEvidence:
+    evidence_id: str
+    inventory_complete: bool
+    model_scope_exact: bool
+    hard_bypass_complete: bool
+    thinking_stripped_control_reducible: bool
+    protected_prefix_exact: bool
+    token_off_on_wire_equivalent: bool
+    tool_history_fidelity: bool
+    cache_metadata_preserved: bool
+    resume_continuity: bool
+    unknown_native_passthrough: bool
+
+    def __post_init__(self) -> None:
+        _require_evidence_id(self.evidence_id)
+        _require_bool_fields(
+            self,
+            (
+                "inventory_complete",
+                "model_scope_exact",
+                "hard_bypass_complete",
+                "thinking_stripped_control_reducible",
+                "protected_prefix_exact",
+                "token_off_on_wire_equivalent",
+                "tool_history_fidelity",
+                "cache_metadata_preserved",
+                "resume_continuity",
+                "unknown_native_passthrough",
+            ),
+        )
+
+    @property
+    def complete(self) -> bool:
+        return all(
+            (
+                self.inventory_complete,
+                self.model_scope_exact,
+                self.hard_bypass_complete,
+                self.thinking_stripped_control_reducible,
+                self.protected_prefix_exact,
+                self.token_off_on_wire_equivalent,
+                self.tool_history_fidelity,
+                self.cache_metadata_preserved,
+                self.resume_continuity,
+                self.unknown_native_passthrough,
+            )
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class AnthropicPreservedThinkingLiveEvidence:
+    evidence_id: str
+    model_id: str
+    binding_beta: str
+    prefix_mismatch_behavior: str
+    cache_hit_observed: bool
+    tool_history_fidelity: bool
+    resume_continuity: bool
+    prefix_binding_observed: bool
+    input_transformations_clear: bool
+    usage_complete: bool
+    estimated_cost_usd: str
+    accounting_status: str
+
+    def __post_init__(self) -> None:
+        _require_evidence_id(self.evidence_id)
+        if not isinstance(self.model_id, str) or not self.model_id:
+            raise ValueError("model_id must be non-blank")
+        if self.binding_beta != BINDING_BETA:
+            raise ValueError("unsupported binding beta")
+        if self.prefix_mismatch_behavior != BINDING_MODE:
+            raise ValueError("unsupported prefix mismatch behavior")
+        _require_bool_fields(
+            self,
+            (
+                "cache_hit_observed",
+                "tool_history_fidelity",
+                "resume_continuity",
+                "prefix_binding_observed",
+                "input_transformations_clear",
+                "usage_complete",
+            ),
+        )
+        _require_nonnegative_finite_decimal(self.estimated_cost_usd, "estimated_cost_usd")
+        if self.accounting_status not in ACCOUNTING_STATES:
+            raise ValueError("unsupported accounting_status")
+
+    @property
+    def complete(self) -> bool:
+        return (
+            self.model_id == MODEL_ID
+            and self.cache_hit_observed
+            and self.tool_history_fidelity
+            and self.resume_continuity
+            and self.prefix_binding_observed
+            and self.input_transformations_clear
+            and self.usage_complete
+        )
+
+
+def _combined_evidence_id(*evidence_ids: str) -> str:
+    normalized = tuple(sorted(set(evidence_ids)))
+    encoded = json.dumps(list(normalized), separators=(",", ":")).encode("utf-8")
+    return f"anthropic-pt:{sha256(encoded).hexdigest()}"
+
+
+def build_anthropic_preserved_thinking_generation(
+    evidence_ids: tuple[str, ...],
+) -> BenchmarkGeneration:
+    normalized = tuple(sorted(set(evidence_ids)))
+    if not normalized or any(not isinstance(item, str) or not item.strip() for item in normalized):
+        raise ValueError("evidence_ids must contain non-blank values")
+    encoded = json.dumps(list(normalized), separators=(",", ":")).encode("utf-8")
+    return BenchmarkGeneration(
+        generation_id=GENERATION_ID,
+        schema_version=1,
+        corpus_version=CORPUS_VERSION,
+        evaluator_version=EVALUATOR_VERSION,
+        policy_version=POLICY_VERSION,
+        evidence_digest=sha256(encoded).hexdigest(),
+        evidence_ids=normalized,
+    )
+
+
+def classify_anthropic_preserved_thinking(
+    offline: AnthropicPreservedThinkingOfflineEvidence,
+    live: AnthropicPreservedThinkingLiveEvidence | None,
+) -> CompatibilityRecord:
+    if not offline.complete:
+        return CompatibilityRecord(
+            key=ANTHROPIC_PRESERVED_THINKING_KEY,
+            state=CompatibilityState.PASSTHROUGH_ONLY,
+            evidence_id=offline.evidence_id,
+            reason="offline_contract_incomplete",
+        )
+    if live is None:
+        return CompatibilityRecord(
+            key=ANTHROPIC_PRESERVED_THINKING_KEY,
+            state=CompatibilityState.PASSTHROUGH_ONLY,
+            evidence_id=offline.evidence_id,
+            reason="offline_conformance_only",
+        )
+
+    evidence_id = _combined_evidence_id(offline.evidence_id, live.evidence_id)
+    if not live.complete:
+        return CompatibilityRecord(
+            key=ANTHROPIC_PRESERVED_THINKING_KEY,
+            state=CompatibilityState.PASSTHROUGH_ONLY,
+            evidence_id=evidence_id,
+            reason="live_contract_incomplete",
+        )
+    return CompatibilityRecord(
+        key=ANTHROPIC_PRESERVED_THINKING_KEY,
+        state=CompatibilityState.PASSTHROUGH_ONLY,
+        evidence_id=evidence_id,
+        reason="live_canary_candidate",
+    )

token_runtime/benchmark.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .model import OptimizationDecision, RequestEnvelope
+
+
+@dataclass(frozen=True, slots=True)
+class BenchmarkCase:
+    name: str
+    envelope: RequestEnvelope
+
+
+@dataclass(frozen=True, slots=True)
+class BenchmarkCaseResult:
+    name: str
+    decision: str
+    before_estimated_tokens: int
+    after_estimated_tokens: int
+    saved_pct: float
+    reasons: tuple[str, ...]
+    protected_byte_fidelity: float
+    recoverable: bool
+
+
+@dataclass(frozen=True, slots=True)
+class BenchmarkReport:
+    cases: tuple[BenchmarkCaseResult, ...]
+    baseline_estimated_tokens: int
+    optimized_estimated_tokens: int
+    total_saved_pct: float
+    optimized_count: int
+    bypass_count: int
+    eligible_saved_pct: float
+
+
+def _fidelity(original: RequestEnvelope, optimized: RequestEnvelope, protected_ids) -> float:
+    original_by_id = {block.id: block.text for block in original.blocks}
+    optimized_by_id = {block.id: block.text for block in optimized.blocks}
+    ids = tuple(protected_ids)
+    if not ids:
+        return 1.0
+    intact = sum(
+        1 for block_id in ids
+        if optimized_by_id.get(block_id) == original_by_id.get(block_id)
+    )
+    return round(intact / len(ids), 6)
+
+
+def _recoverable(engine, refs: tuple[str, ...]) -> bool:
+    try:
+        for ref in refs:
+            engine.store.get(ref)
+    except Exception:
+        return False
+    return True
+
+
+def benchmark_cases(cases, engine) -> BenchmarkReport:
+    rows: list[BenchmarkCaseResult] = []
+    baseline_total = optimized_total = 0
+    eligible_before = eligible_after = 0
+    optimized_count = 0
+
+    for case in cases:
+        plan = engine.planner.plan(case.envelope)
+        result = engine.optimize(case.envelope)
+        before = result.before_estimated_tokens
+        after = result.after_estimated_tokens
+        baseline_total += before
+        optimized_total += after
+        saved_pct = round((before - after) * 100 / before, 2) if before else 0.0
+        if result.decision is OptimizationDecision.OPTIMIZE:
+            optimized_count += 1
+            eligible_before += before
+            eligible_after += after
+        rows.append(BenchmarkCaseResult(
+            name=case.name,
+            decision=result.decision.value,
+            before_estimated_tokens=before,
+            after_estimated_tokens=after,
+            saved_pct=saved_pct,
+            reasons=result.reasons,
+            protected_byte_fidelity=_fidelity(
+                case.envelope, result.envelope, plan.protected_ids
+            ),
+            recoverable=_recoverable(engine, result.recovery_refs),
+        ))
+
+    total_saved_pct = (
+        round((baseline_total - optimized_total) * 100 / baseline_total, 2)
+        if baseline_total else 0.0
+    )
+    eligible_saved_pct = (
+        round((eligible_before - eligible_after) * 100 / eligible_before, 2)
+        if eligible_before else 0.0
+    )
+    return BenchmarkReport(
+        cases=tuple(rows),
+        baseline_estimated_tokens=baseline_total,
+        optimized_estimated_tokens=optimized_total,
+        total_saved_pct=total_saved_pct,
+        optimized_count=optimized_count,
+        bypass_count=len(rows) - optimized_count,
+        eligible_saved_pct=eligible_saved_pct,
+    )