token-runtime 0.1.0a2__py3-none-any.whl

token_runtime/__init__.py

@@ -0,0 +1 @@
+"""TOKEN adaptive context optimization runtime."""

token_runtime/adapters.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+from copy import deepcopy
+import json
+from typing import Any, Mapping, MutableMapping, Sequence
+
+from .model import ContextBlock, RequestEnvelope
+
+
+def _set_path(root: Any, path: Sequence[Any], value: str) -> None:
+    target = root
+    for key in path[:-1]:
+        target = target[key]
+    target[path[-1]] = value
+
+
+def _kind_for_role(role: str) -> str:
+    if role in {"system", "developer", "user"}:
+        return role
+    if role == "tool":
+        return "tool_output"
+    return "assistant"
+
+
+def _block(block_id: str, kind: str, text: str, *, role=None, turn=0, path=None) -> ContextBlock:
+    metadata = {} if path is None else {"path": tuple(path)}
+    return ContextBlock(block_id, kind, text, role=role, turn_index=turn, metadata=metadata)
+
+
+class _AdapterBase:
+    name = "base"
+
+    def serialize(self, envelope: RequestEnvelope) -> dict[str, Any]:
+        payload = deepcopy(envelope.opaque["original_payload"])
+        for block in envelope.blocks:
+            path = block.metadata.get("path")
+            if path:
+                _set_path(payload, path, block.text)
+        return payload
+
+    @staticmethod
+    def _tool_blocks(tools: Any) -> list[ContextBlock]:
+        if not isinstance(tools, list):
+            return []
+        return [
+            _block(f"tool-schema-{i}", "tool_schema", json.dumps(tool, sort_keys=True, separators=(",", ":")))
+            for i, tool in enumerate(tools)
+        ]
+
+
+class ResponsesAdapter(_AdapterBase):
+    name = "responses"
+
+    def parse(self, payload: Mapping[str, Any]) -> RequestEnvelope:
+        original = deepcopy(dict(payload))
+        blocks: list[ContextBlock] = []
+        safe = True
+        instructions = payload.get("instructions")
+        if isinstance(instructions, str):
+            blocks.append(_block("instructions", "system", instructions, role="system", path=("instructions",)))
+        elif instructions is not None:
+            safe = False
+
+        blocks.extend(self._tool_blocks(payload.get("tools")))
+        input_value = payload.get("input")
+        if isinstance(input_value, str):
+            blocks.append(_block("input", "user", input_value, role="user", path=("input",)))
+        elif isinstance(input_value, list):
+            for i, item in enumerate(input_value):
+                item_blocks, item_safe = self._parse_input_item(i, item)
+                blocks.extend(item_blocks)
+                safe = safe and item_safe
+        elif input_value is not None:
+            safe = False
+
+        return RequestEnvelope(
+            blocks=tuple(blocks),
+            opaque={"original_payload": original},
+            adapter=self.name,
+            wire_safe=safe,
+        )
+
+    def _parse_input_item(self, index: int, item: Any) -> tuple[list[ContextBlock], bool]:
+        if not isinstance(item, Mapping):
+            return [], False
+        item_type = item.get("type")
+        if item_type == "function_call_output":
+            output = item.get("output")
+            if not isinstance(output, str):
+                return [], False
+            return [_block(f"input-{index}", "tool_output", output, role="tool", turn=index, path=("input", index, "output"))], True
+        if item_type == "function_call":
+            return [_block(f"input-{index}", "tool_call", json.dumps(dict(item), sort_keys=True))], True
+
+        role = item.get("role")
+        if not isinstance(role, str):
+            return [], False
+        content = item.get("content")
+        kind = _kind_for_role(role)
+        if isinstance(content, str):
+            return [
+                _block(f"input-{index}", kind, content, role=role, turn=index, path=("input", index, "content"))
+            ], True
+        if isinstance(content, list):
+            blocks: list[ContextBlock] = []
+            safe = True
+            for j, part in enumerate(content):
+                if not isinstance(part, Mapping):
+                    safe = False
+                    continue
+                part_type = part.get("type")
+                text = part.get("text")
+                if part_type not in {"input_text", "output_text", "text"} or not isinstance(text, str):
+                    safe = False
+                    continue
+                blocks.append(
+                    _block(
+                        f"input-{index}-content-{j}",
+                        kind,
+                        text,
+                        role=role,
+                        turn=index,
+                        path=("input", index, "content", j, "text"),
+                    )
+                )
+            return blocks, safe
+        return [], content is None
+
+
+class ChatCompletionsAdapter(_AdapterBase):
+    name = "chat_completions"
+
+    def parse(self, payload: Mapping[str, Any]) -> RequestEnvelope:
+        original = deepcopy(dict(payload))
+        blocks: list[ContextBlock] = self._tool_blocks(payload.get("tools"))
+        safe = True
+        messages = payload.get("messages")
+        if not isinstance(messages, list):
+            safe = False
+            messages = []
+
+        for i, message in enumerate(messages):
+            if not isinstance(message, Mapping):
+                safe = False
+                continue
+            role = message.get("role")
+            if not isinstance(role, str):
+                safe = False
+                continue
+            kind = _kind_for_role(role)
+            content = message.get("content")
+            if isinstance(content, str):
+                blocks.append(
+                    _block(f"message-{i}", kind, content, role=role, turn=i, path=("messages", i, "content"))
+                )
+            elif isinstance(content, list):
+                for j, part in enumerate(content):
+                    if not isinstance(part, Mapping) or part.get("type") != "text" or not isinstance(part.get("text"), str):
+                        safe = False
+                        continue
+                    blocks.append(
+                        _block(
+                            f"message-{i}-content-{j}", kind, part["text"], role=role, turn=i,
+                            path=("messages", i, "content", j, "text"),
+                        )
+                    )
+            elif content is not None:
+                safe = False
+            tool_calls = message.get("tool_calls")
+            if tool_calls is not None:
+                if not isinstance(tool_calls, list):
+                    safe = False
+                else:
+                    blocks.append(_block(f"message-{i}-tool-calls", "tool_call", json.dumps(tool_calls, sort_keys=True), role=role, turn=i))
+
+        return RequestEnvelope(
+            blocks=tuple(blocks),
+            opaque={"original_payload": original},
+            adapter=self.name,
+            wire_safe=safe,
+        )

token_runtime/agent_integrations.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+from .capabilities import CapabilityDetector, CapabilityKey
+from .compatibility import CompatibilityState
+from .contracts import AgentIntegrationContract
+from .feature_flags import (
+    FeatureEffect,
+    FeatureFlagState,
+    permitted_effect,
+)
+
+
+class AgentIntegrationMode(str, Enum):
+    TOKEN = "TOKEN"
+    PASSTHROUGH = "PASSTHROUGH"
+
+
+@dataclass(frozen=True, slots=True)
+class AgentIntegrationDecision:
+    agent_id: str
+    capability_key: CapabilityKey
+    endpoint: str
+    compatibility_state: CompatibilityState
+    effect: FeatureEffect
+    evidence_id: str
+    reason: str | None
+    mode: AgentIntegrationMode
+
+
+class AgentIntegrationFramework:
+    def __init__(
+        self,
+        detector: CapabilityDetector,
+        *,
+        flag_state: FeatureFlagState = FeatureFlagState.ENABLED,
+    ) -> None:
+        self.detector = detector
+        self.flag_state = flag_state
+
+    def resolve(
+        self,
+        integration: AgentIntegrationContract,
+    ) -> AgentIntegrationDecision:
+        detected = self.detector.detect(integration.capability_key)
+        if detected.profile is None:
+            compatibility_state = CompatibilityState.PASSTHROUGH_ONLY
+            evidence_id = "unknown"
+            reason = (
+                "unknown_capability"
+                if detected.compatibility.reason == "unknown_capability"
+                else "missing_capability_profile"
+            )
+        elif detected.compatibility.reason == "unknown_capability":
+            compatibility_state = CompatibilityState.PASSTHROUGH_ONLY
+            evidence_id = "unknown"
+            reason = "unknown_capability"
+        elif (
+            detected.profile.evidence_id == "unknown"
+            or detected.compatibility.evidence_id == "unknown"
+            or detected.profile.evidence_id != detected.compatibility.evidence_id
+        ):
+            compatibility_state = CompatibilityState.PASSTHROUGH_ONLY
+            evidence_id = "unknown"
+            reason = "capability_evidence_mismatch"
+        else:
+            compatibility_state = detected.compatibility.state
+            evidence_id = detected.compatibility.evidence_id
+            reason = detected.compatibility.reason
+        effect = permitted_effect(self.flag_state, compatibility_state)
+        mode = (
+            AgentIntegrationMode.TOKEN
+            if effect is FeatureEffect.EXECUTE
+            else AgentIntegrationMode.PASSTHROUGH
+        )
+        return AgentIntegrationDecision(
+            agent_id=integration.agent_id,
+            capability_key=integration.capability_key,
+            endpoint=integration.endpoint,
+            compatibility_state=compatibility_state,
+            effect=effect,
+            evidence_id=evidence_id,
+            reason=reason,
+            mode=mode,
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class _AgentIntegrationDescriptor:
+    agent_id: str
+    capability_key: CapabilityKey
+    endpoint: str
+
+
+def codex_01540_reference() -> AgentIntegrationContract:
+    from .codex_recertification import CODEX_01540_RESPONSES_KEY
+
+    return _AgentIntegrationDescriptor(
+        agent_id="codex",
+        capability_key=CODEX_01540_RESPONSES_KEY,
+        endpoint="/v1/responses",
+    )
+
+
+def build_codex_01540_agent_framework(
+    *,
+    flag_state: FeatureFlagState = FeatureFlagState.ENABLED,
+) -> AgentIntegrationFramework:
+    from .codex_recertification import build_codex_01540_recertification
+
+    bundle = build_codex_01540_recertification()
+    return AgentIntegrationFramework(bundle.detector, flag_state=flag_state)

token_runtime/anthropic_adapter.py

@@ -0,0 +1,301 @@
+from __future__ import annotations
+
+from copy import deepcopy
+import json
+from typing import Any, Mapping, Sequence
+
+from .model import ContextBlock, RequestEnvelope
+
+
+def _set_path(root: Any, path: Sequence[Any], value: str) -> None:
+    target = root
+    for key in path[:-1]:
+        target = target[key]
+    target[path[-1]] = value
+
+
+def _block(
+    block_id: str,
+    kind: str,
+    text: str,
+    *,
+    role: str | None = None,
+    turn: int = 0,
+    path: Sequence[Any] | None = None,
+) -> ContextBlock:
+    metadata = {} if path is None else {"path": tuple(path)}
+    return ContextBlock(
+        block_id,
+        kind,
+        text,
+        role=role,
+        turn_index=turn,
+        metadata=metadata,
+    )
+
+
+def _encoded(value: Mapping[str, Any]) -> str:
+    return json.dumps(dict(value), sort_keys=True, separators=(",", ":"))
+
+
+class AnthropicMessagesAdapter:
+    protocol_id = "anthropic_messages"
+
+    def serialize(self, envelope: RequestEnvelope) -> dict[str, Any]:
+        payload = deepcopy(envelope.opaque["original_payload"])
+        for block in envelope.blocks:
+            path = block.metadata.get("path")
+            if path:
+                _set_path(payload, path, block.text)
+        return payload
+
+    def parse(self, payload: Mapping[str, Any]) -> RequestEnvelope:
+        original = deepcopy(dict(payload))
+        blocks: list[ContextBlock] = []
+        safe = True
+
+        system_blocks, system_safe = self._parse_system(payload.get("system"))
+        blocks.extend(system_blocks)
+        safe = safe and system_safe
+
+        tool_blocks, tools_safe = self._parse_tools(payload.get("tools"))
+        blocks.extend(tool_blocks)
+        safe = safe and tools_safe
+
+        messages = payload.get("messages")
+        if not isinstance(messages, list):
+            safe = False
+            messages = []
+
+        for index, message in enumerate(messages):
+            message_blocks, message_safe = self._parse_message(index, message)
+            blocks.extend(message_blocks)
+            safe = safe and message_safe
+
+        return RequestEnvelope(
+            blocks=tuple(blocks),
+            opaque={"original_payload": original},
+            adapter=self.protocol_id,
+            wire_safe=safe,
+        )
+
+    @staticmethod
+    def _parse_system(system: Any) -> tuple[list[ContextBlock], bool]:
+        if system is None:
+            return [], True
+        if isinstance(system, str):
+            return [
+                _block("system", "system", system, role="system", path=("system",))
+            ], True
+        if not isinstance(system, list):
+            return [], False
+
+        blocks: list[ContextBlock] = []
+        safe = True
+        for index, part in enumerate(system):
+            if (
+                not isinstance(part, Mapping)
+                or part.get("type") != "text"
+                or not isinstance(part.get("text"), str)
+            ):
+                safe = False
+                continue
+            cited = part.get("citations") is not None
+            blocks.append(
+                _block(
+                    f"system-{index}",
+                    "system",
+                    part["text"],
+                    role="system",
+                    path=None if cited else ("system", index, "text"),
+                )
+            )
+            safe = safe and not cited
+        return blocks, safe
+
+    @staticmethod
+    def _parse_tools(tools: Any) -> tuple[list[ContextBlock], bool]:
+        if tools is None:
+            return [], True
+        if not isinstance(tools, list):
+            return [], False
+        blocks: list[ContextBlock] = []
+        safe = True
+        for index, tool in enumerate(tools):
+            if not isinstance(tool, Mapping):
+                safe = False
+                continue
+            blocks.append(
+                _block(
+                    f"tool-schema-{index}",
+                    "tool_schema",
+                    _encoded(tool),
+                )
+            )
+        return blocks, safe
+
+    def _parse_message(
+        self,
+        index: int,
+        message: Any,
+    ) -> tuple[list[ContextBlock], bool]:
+        if not isinstance(message, Mapping):
+            return [], False
+        role = message.get("role")
+        if role not in {"user", "assistant"}:
+            return [], False
+
+        content = message.get("content")
+        if isinstance(content, str):
+            return [
+                _block(
+                    f"message-{index}",
+                    role,
+                    content,
+                    role=role,
+                    turn=index,
+                    path=("messages", index, "content"),
+                )
+            ], True
+
+        if not isinstance(content, list):
+            return [], False
+
+        blocks: list[ContextBlock] = []
+        safe = True
+        for part_index, part in enumerate(content):
+            part_blocks, part_safe = self._parse_content_block(
+                message_index=index,
+                part_index=part_index,
+                role=role,
+                part=part,
+            )
+            blocks.extend(part_blocks)
+            safe = safe and part_safe
+        return blocks, safe
+
+    def _parse_content_block(
+        self,
+        *,
+        message_index: int,
+        part_index: int,
+        role: str,
+        part: Any,
+    ) -> tuple[list[ContextBlock], bool]:
+        if not isinstance(part, Mapping):
+            return [], False
+
+        part_type = part.get("type")
+        if part_type == "text":
+            text = part.get("text")
+            if not isinstance(text, str):
+                return [], False
+            cited = part.get("citations") is not None
+            return [
+                _block(
+                    f"message-{message_index}-content-{part_index}",
+                    role,
+                    text,
+                    role=role,
+                    turn=message_index,
+                    path=(
+                        None
+                        if cited
+                        else ("messages", message_index, "content", part_index, "text")
+                    ),
+                )
+            ], not cited
+
+        if part_type == "tool_use":
+            valid_tool_use = (
+                role == "assistant"
+                and isinstance(part.get("id"), str)
+                and isinstance(part.get("name"), str)
+                and isinstance(part.get("input"), Mapping)
+            )
+            return [
+                _block(
+                    f"message-{message_index}-tool-use-{part_index}",
+                    "tool_call",
+                    _encoded(part),
+                    role=role,
+                    turn=message_index,
+                )
+            ], valid_tool_use
+
+        if part_type == "tool_result":
+            return self._parse_tool_result(
+                message_index=message_index,
+                part_index=part_index,
+                role=role,
+                part=part,
+            )
+
+        if part_type in {"thinking", "redacted_thinking"}:
+            return [
+                _block(
+                    f"message-{message_index}-protocol-{part_index}",
+                    "protocol_state",
+                    _encoded(part),
+                    role=role,
+                    turn=message_index,
+                )
+            ], False
+
+        return [], False
+
+    @staticmethod
+    def _parse_tool_result(
+        *,
+        message_index: int,
+        part_index: int,
+        role: str,
+        part: Mapping[str, Any],
+    ) -> tuple[list[ContextBlock], bool]:
+        content = part.get("content")
+        base_path = ("messages", message_index, "content", part_index, "content")
+        valid_reference = role == "user" and isinstance(part.get("tool_use_id"), str)
+        if isinstance(content, str):
+            return [
+                _block(
+                    f"message-{message_index}-tool-result-{part_index}",
+                    "tool_output",
+                    content,
+                    role="tool",
+                    turn=message_index,
+                    path=base_path if valid_reference else None,
+                )
+            ], valid_reference
+        if not isinstance(content, list):
+            return [], False
+
+        blocks: list[ContextBlock] = []
+        safe = valid_reference
+        for nested_index, nested in enumerate(content):
+            if (
+                not isinstance(nested, Mapping)
+                or nested.get("type") != "text"
+                or not isinstance(nested.get("text"), str)
+            ):
+                safe = False
+                continue
+            cited = nested.get("citations") is not None
+            blocks.append(
+                _block(
+                    (
+                        f"message-{message_index}-tool-result-"
+                        f"{part_index}-content-{nested_index}"
+                    ),
+                    "tool_output",
+                    nested["text"],
+                    role="tool",
+                    turn=message_index,
+                    path=(
+                        None
+                        if cited or not valid_reference
+                        else base_path + (nested_index, "text")
+                    ),
+                )
+            )
+            safe = safe and not cited
+        return blocks, safe