clink-cli 0.2.1__py3-none-any.whl

clink/parsers/codex_generation.py

@@ -0,0 +1,212 @@
+"""Strict, text-only Codex JSONL parsing for the separate generation profile."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+from .base import (
+    MAX_METADATA_NUMBER,
+    BaseParser,
+    ParsedCLIResponse,
+    ParserError,
+    cli_failure,
+    load_json,
+    model_identifier,
+)
+
+MAX_EVENTS = 16384
+_MAX_SOURCE_BYTES = 1024 * 1024
+_USAGE_REQUIRED = frozenset({"input_tokens", "cached_input_tokens", "output_tokens"})
+_USAGE_OPTIONAL = frozenset({"cache_write_input_tokens", "reasoning_output_tokens"})
+_ITEM_EVENTS = frozenset({"item.started", "item.updated", "item.completed"})
+_TOOL_ITEMS = frozenset(
+    {
+        "command_execution",
+        "file_change",
+        "mcp_tool_call",
+        "collab_tool_call",
+        "web_search",
+        "todo_list",
+        "image_generation",
+    }
+)
+
+# Admission binds this explicit acceptance contract independently of advisory and
+# native parsers. Change it whenever generation acceptance semantics change.
+PARSER_CONTRACT = (
+    "codex-generation-jsonl/v1\n"
+    "strict-json:no-duplicate-keys,no-nonfinite-numbers;jsonl-newline=LF;max-events=16384\n"
+    "thread.started:{type,thread_id:nonempty-utf8-text};turn.started:{type};one-thread,one-turn,in-order\n"
+    "item.started|item.updated|item.completed:{type,item};item:{type:reasoning|agent_message,text:utf8,id?:nonempty-utf8}\n"
+    "text-item-lifecycle:no-reused-completed-id,no-type-change,no-duplicate-start,no-incomplete-items\n"
+    "final:exactly-one-completed-nonblank-agent_message;preserve-text-exactly;no-item-after-final\n"
+    "turn.completed:{type,usage,model?:null|bounded-native-id};one-terminal,last-event\n"
+    "usage:required=input_tokens,cached_input_tokens,output_tokens;"
+    "optional=cache_write_input_tokens,reasoning_output_tokens;nonnegative-safe-json-integers\n"
+    "error:{type,message:utf8};turn.failed:{type,error:{message:utf8}};item-error:{type,message:utf8,id?}\n"
+    "unknown-keys,unknown-events,unknown-items,malformed,failed,incomplete=protocol_error;"
+    "known-tool-items=command_execution,file_change,mcp_tool_call,collab_tool_call,web_search,todo_list,image_generation:"
+    "unexpected_tool_event;known-failure-diagnostics=authentication_required|rate_limited\n"
+    "metadata:usage,actual_model=null-when-unreported;no-thread-or-item-identities\n"
+    "identity:sha256(canonical-json(contract,source_sha256(codex_generation.py,base.py)));source-limit=1048576-bytes-each\n"
+)
+
+
+def parser_contract_digest() -> str:
+    """Bind the acceptance contract and both parser implementations for admission."""
+
+    sources = {}
+    try:
+        for name in ("codex_generation.py", "base.py"):
+            with Path(__file__).with_name(name).open("rb") as source:
+                content = source.read(_MAX_SOURCE_BYTES + 1)
+            if len(content) > _MAX_SOURCE_BYTES:
+                raise _protocol_error("Codex generation parser source exceeds its identity limit")
+            sources[name] = hashlib.sha256(content).hexdigest()
+    except OSError:
+        raise _protocol_error("Codex generation parser source identity is unavailable") from None
+    identity = json.dumps(
+        {"contract": PARSER_CONTRACT, "source_sha256": sources},
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=False,
+    )
+    return hashlib.sha256(identity.encode("utf-8")).hexdigest()
+
+
+def _protocol_error(message: str) -> ParserError:
+    return ParserError(message, code="protocol_error")
+
+
+def _object(value: object, required: frozenset[str], optional: frozenset[str] = frozenset()) -> dict[str, Any]:
+    if not isinstance(value, dict) or not required <= value.keys() or value.keys() - (required | optional):
+        raise _protocol_error("Codex generation record has an unsupported shape")
+    return value
+
+
+def _text(value: object, *, nonempty: bool = False) -> str:
+    if not isinstance(value, str) or (nonempty and not value.strip()):
+        raise _protocol_error("Codex generation text field is invalid")
+    try:
+        value.encode("utf-8")
+    except UnicodeError:
+        raise _protocol_error("Codex generation text is not valid UTF-8") from None
+    return value
+
+
+def _failure(value: dict[str, Any], stderr: str) -> ParserError:
+    return cli_failure(value, stderr, message="Codex generation reported a failure", code="protocol_error")
+
+
+class CodexGenerationParser(BaseParser):
+    """Require a complete, unambiguous text-only generation turn."""
+
+    name = "codex_generation_jsonl"
+
+    def parse(self, stdout: str, stderr: str = "") -> ParsedCLIResponse:
+        if not isinstance(stdout, str) or not isinstance(stderr, str):
+            raise _protocol_error("Codex generation streams must be textual")
+        # splitlines() also splits valid literal Unicode separators inside JSON
+        # strings. Only LF delimits JSONL; CRLF remains valid JSON whitespace.
+        lines = [line for line in stdout.split("\n") if line.strip(" \t\r")]
+        if not lines or len(lines) > MAX_EVENTS:
+            raise cli_failure(stderr, message="Codex generation output is empty or too large", code="protocol_error")
+
+        thread_started = False
+        turn_started = False
+        terminal = False
+        final_text: str | None = None
+        item_types: dict[str | None, str] = {}
+        active_items: set[str | None] = set()
+        completed_items: set[str] = set()
+        metadata: dict[str, Any] = {"actual_model": None}
+
+        for line in lines:
+            try:
+                event = load_json(line)
+            except ParserError:
+                raise _protocol_error("Codex generation returned malformed JSON") from None
+            if not isinstance(event, dict) or not isinstance(event.get("type"), str):
+                raise _protocol_error("Codex generation events must be typed objects")
+            if terminal:
+                raise _protocol_error("Codex generation continued after its terminal result")
+            event_type = event["type"]
+
+            if event_type == "error":
+                _object(event, frozenset({"type", "message"}))
+                _text(event["message"])
+                raise _failure(event, stderr)
+            if event_type == "turn.failed":
+                _object(event, frozenset({"type", "error"}))
+                error = _object(event["error"], frozenset({"message"}))
+                _text(error["message"])
+                raise _failure(event, stderr)
+
+            if event_type == "thread.started":
+                _object(event, frozenset({"type", "thread_id"}))
+                _text(event["thread_id"], nonempty=True)
+                if thread_started or turn_started:
+                    raise _protocol_error("Codex generation thread lifecycle is contradictory")
+                thread_started = True
+            elif event_type == "turn.started":
+                _object(event, frozenset({"type"}))
+                if not thread_started or turn_started:
+                    raise _protocol_error("Codex generation turn lifecycle is contradictory")
+                turn_started = True
+            elif event_type in _ITEM_EVENTS:
+                _object(event, frozenset({"type", "item"}))
+                item = event["item"]
+                if not isinstance(item, dict) or not isinstance(item.get("type"), str):
+                    raise _protocol_error("Codex generation items must be typed objects")
+                item_type = item["type"]
+                if item_type in _TOOL_ITEMS:
+                    raise ParserError("Codex generation emitted an unexpected tool item", code="unexpected_tool_event")
+                if item_type == "error":
+                    _object(item, frozenset({"type", "message"}), frozenset({"id"}))
+                    if "id" in item:
+                        _text(item["id"], nonempty=True)
+                    _text(item["message"])
+                    raise _failure(item, stderr)
+                if item_type not in {"agent_message", "reasoning"}:
+                    raise _protocol_error("Codex generation item type is unsupported")
+                _object(item, frozenset({"type", "text"}), frozenset({"id"}))
+                text = _text(item["text"])
+                item_id = _text(item["id"], nonempty=True) if "id" in item else None
+                if not turn_started or final_text is not None:
+                    raise _protocol_error("Codex generation text appeared outside the active turn")
+                if item_id in completed_items or (item_id in item_types and item_types[item_id] != item_type):
+                    raise _protocol_error("Codex generation item lifecycle is contradictory")
+                if event_type == "item.started" and item_id in active_items:
+                    raise _protocol_error("Codex generation item was started more than once")
+                item_types[item_id] = item_type
+                if event_type == "item.completed":
+                    active_items.discard(item_id)
+                    if item_id is not None:
+                        completed_items.add(item_id)
+                    else:
+                        item_types.pop(None, None)
+                    if item_type == "agent_message":
+                        final_text = _text(text, nonempty=True)
+                else:
+                    active_items.add(item_id)
+            elif event_type == "turn.completed":
+                _object(event, frozenset({"type", "usage"}), frozenset({"model"}))
+                if not turn_started or active_items or final_text is None:
+                    raise _protocol_error("Codex generation turn is incomplete")
+                usage = _object(event["usage"], _USAGE_REQUIRED, _USAGE_OPTIONAL)
+                if any(type(count) is not int or not 0 <= count <= MAX_METADATA_NUMBER for count in usage.values()):
+                    raise _protocol_error("Codex generation usage is invalid")
+                actual_model = event.get("model")
+                if actual_model is not None and model_identifier(actual_model) is None:
+                    raise _protocol_error("Codex generation actual model is invalid")
+                metadata.update(usage=usage.copy(), actual_model=actual_model)
+                terminal = True
+            else:
+                raise _protocol_error("Codex generation event type is unsupported")
+
+        if not terminal or final_text is None:
+            raise cli_failure(stderr, message="Codex generation output is incomplete", code="protocol_error")
+        return ParsedCLIResponse(content=final_text, metadata=metadata)

clink/parsers/codex_native.py

@@ -0,0 +1,236 @@
+"""Parse native Codex JSONL without retaining tool payloads or public sessions."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from clink.native_contracts import ParsedNativeResponse
+from clink.parsers.base import ParserError, bounded_counts, cli_failure, load_json
+
+RESULT_CHARACTERS = 20000
+MAX_EVENTS = 16384
+_SESSION = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\Z")
+_EVENT_TYPES = {
+    "thread.started",
+    "turn.started",
+    "item.started",
+    "item.updated",
+    "item.completed",
+    "turn.completed",
+    "turn.failed",
+    "error",
+}
+_ITEM_TYPES = {
+    "agent_message",
+    "reasoning",
+    "command_execution",
+    "file_change",
+    "mcp_tool_call",
+    "collab_tool_call",
+    "web_search",
+    "todo_list",
+    "error",
+}
+_TOOL_TYPES = {"command_execution", "file_change", "mcp_tool_call", "collab_tool_call"}
+_USAGE_FIELDS = (
+    "input_tokens",
+    "cached_input_tokens",
+    "cache_write_input_tokens",
+    "output_tokens",
+    "total_tokens",
+    "reasoning_output_tokens",
+)
+_EXEC_POLICY_DENIAL = re.compile(
+    r"`(?P<command>[^`\r\n]+)` rejected: policy forbids commands starting with `(?P<prefix>[^`\r\n]+)`\Z"
+)
+_COMMAND_APPROVAL_DENIALS = {
+    "approval required by policy, but AskForApproval is set to Never",
+    "approval required by policy, but AskForApproval::Granular.sandbox_approval is false",
+    "approval required by policy rule, but AskForApproval::Granular.rules is false",
+}
+_MCP_APPROVAL_DENIALS = {
+    "MCP tool call requires approval, but approval policy is never",
+    "MCP tool call cancelled: interactive approval is unavailable in exec mode",
+}
+
+
+def validate_session_id(value: str | None) -> str | None:
+    """Accept only canonical UUID-shaped private Codex thread identities."""
+
+    if value is not None and (not isinstance(value, str) or not _SESSION.fullmatch(value)):
+        raise ParserError("Codex native session identity is invalid")
+    return value
+
+
+def _validate_error(value: object) -> None:
+    if not isinstance(value, dict) or not isinstance(value.get("message"), str):
+        raise ParserError("Codex native error must contain a textual message")
+
+
+def _validate_item(value: object) -> dict[str, Any]:
+    if not isinstance(value, dict) or not isinstance(value.get("type"), str) or value["type"] not in _ITEM_TYPES:
+        raise ParserError("Codex native item type is unsupported")
+    if not isinstance(value.get("id"), str) or not value["id"]:
+        raise ParserError("Codex native item identity is invalid")
+    item_type = value["type"]
+    if item_type in {"agent_message", "reasoning"} and not isinstance(value.get("text"), str):
+        raise ParserError("Codex native message must be textual")
+    if item_type == "error" and not isinstance(value.get("message"), str):
+        raise ParserError("Codex native item error must be textual")
+    if item_type in _TOOL_TYPES:
+        statuses = {"in_progress", "completed", "failed"}
+        if item_type == "command_execution":
+            # Pinned native status includes declined for both policy rejections
+            # and runtime setup failures, so it does not alone prove a denial.
+            statuses.add("declined")
+        if not isinstance(value.get("status"), str) or value["status"] not in statuses:
+            raise ParserError("Codex native tool status is invalid")
+    if item_type == "command_execution":
+        if not isinstance(value.get("command"), str) or not isinstance(value.get("aggregated_output"), str):
+            raise ParserError("Codex native command fields must be textual")
+        if "exit_code" not in value:
+            raise ParserError("Codex native command exit code is missing")
+        exit_code = value["exit_code"]
+        if exit_code is not None and (not isinstance(exit_code, int) or isinstance(exit_code, bool)):
+            raise ParserError("Codex native command exit code must be an integer")
+    if item_type == "mcp_tool_call":
+        if not isinstance(value.get("server"), str) or not isinstance(value.get("tool"), str):
+            raise ParserError("Codex native MCP tool identity must be textual")
+        if value.get("error") is not None:
+            _validate_error(value["error"])
+        if value.get("result") is not None and not isinstance(value["result"], dict):
+            raise ParserError("Codex native MCP result must be an object")
+    return value
+
+
+def _permission_denial(item: dict[str, Any]) -> bool:
+    """Recognize a conservative subset of pinned 0.166.0 native diagnostics.
+
+    The protocol loses typed rejection provenance; generic declined items can
+    also represent setup failures. These exact native strings are a versioned
+    dependency, not a general interpretation of assistant or command prose.
+    """
+
+    if item["type"] == "command_execution" and item["status"] == "declined" and item["exit_code"] == -1:
+        if item["aggregated_output"] in _COMMAND_APPROVAL_DENIALS:
+            return True
+        marker = _EXEC_POLICY_DENIAL.fullmatch(item["aggregated_output"])
+        return (
+            marker is not None
+            and marker["command"] == item["command"]
+            and bool(marker["command"].strip())
+            and bool(marker["prefix"].strip())
+        )
+    if item["type"] == "mcp_tool_call" and item["status"] == "failed":
+        error = item.get("error")
+        return (
+            "result" in item
+            and item["result"] is None
+            and isinstance(error, dict)
+            and error["message"] in _MCP_APPROVAL_DENIALS
+        )
+    return False
+
+
+class CodexNativeParser:
+    """Use the terminal turn outcome while preserving recoverable tool failures."""
+
+    def parse(self, stdout: str, stderr: str = "") -> ParsedNativeResponse:
+        if not isinstance(stdout, str):
+            raise ParserError("Codex native output must be textual")
+        lines = [line for line in stdout.splitlines() if line.strip()]
+        if not lines or len(lines) > MAX_EVENTS:
+            raise ParserError("Codex native output was empty or exceeded the event limit")
+        session_id = None
+        terminal: dict[str, Any] | None = None
+        fatal: dict[str, Any] | None = None
+        item_types: dict[str, str] = {}
+        completed_items: set[str] = set()
+        tool_errors = 0
+        permission_denials = 0
+        content = ""
+        message_count = 0
+
+        for line in lines:
+            event = load_json(line, allow_codex_web_search_ids=True)
+            if not isinstance(event, dict) or not isinstance(event.get("type"), str):
+                raise ParserError("Codex native events must be typed JSON objects")
+            event_type = event["type"]
+            if event_type not in _EVENT_TYPES:
+                raise ParserError("Codex native event type is unsupported")
+            # The installed CLI emits `error` followed by `turn.failed` for one
+            # fatal outcome. No other post-terminal event or duplicate is valid.
+            if terminal is not None or (fatal is not None and event_type != "turn.failed"):
+                raise ParserError("Codex native output continued after its terminal outcome")
+            if "thread_id" in event:
+                observed_session = event["thread_id"]
+                validate_session_id(observed_session)
+                if observed_session is None:
+                    raise ParserError("Codex native thread identity is missing")
+                if session_id is not None and session_id != observed_session:
+                    raise ParserError("Codex native output changed its session identity")
+                session_id = observed_session
+            elif event_type == "thread.started":
+                raise ParserError("Codex native thread identity is missing")
+
+            if event_type.startswith("item."):
+                item = _validate_item(event.get("item"))
+                item_id, item_type = item["id"], item["type"]
+                if item_id in completed_items or (item_id in item_types and item_types[item_id] != item_type):
+                    raise ParserError("Codex native item lifecycle is contradictory")
+                item_types[item_id] = item_type
+                if event_type == "item.completed":
+                    completed_items.add(item_id)
+                    if item_type in _TOOL_TYPES and item["status"] == "in_progress":
+                        raise ParserError("Codex native completed tool is still in progress")
+                    failed = item_type == "error" or (
+                        item_type in _TOOL_TYPES
+                        and (
+                            item["status"] in {"failed", "declined"}
+                            or (item_type == "command_execution" and item.get("exit_code") not in (None, 0))
+                            or (item_type == "mcp_tool_call" and item.get("error") is not None)
+                        )
+                    )
+                    tool_errors += int(failed)
+                    permission_denials += int(_permission_denial(item))
+                    if item_type == "agent_message":
+                        separator = "\n\n" if message_count else ""
+                        remaining = RESULT_CHARACTERS + 1 - len(content)
+                        if remaining > 0:
+                            content += (separator + item["text"][:remaining])[:remaining]
+                        message_count += 1
+            elif event_type == "error":
+                _validate_error(event)
+                fatal = event
+            elif event_type == "turn.failed":
+                _validate_error(event.get("error"))
+                terminal = event
+            elif event_type == "turn.completed":
+                if (
+                    event.get("status", "completed") not in ("completed", "success")
+                    or event.get("error") is not None
+                    or event.get("errors")
+                    or event.get("is_error", False) is not False
+                ):
+                    raise ParserError("Codex native successful turn contains contradictory failure markers")
+                if not isinstance(event.get("usage"), dict):
+                    raise ParserError("Codex native usage must be an object")
+                terminal = event
+
+        if terminal is None and fatal is None:
+            raise cli_failure(stderr, message="Codex native output had no terminal outcome")
+        success = terminal is not None and terminal["type"] == "turn.completed"
+        if success and session_id is None:
+            raise ParserError("Codex native successful turn omitted its session identity")
+        state = "permission_needed" if permission_denials else "completed" if success else "failed"
+        metadata: dict[str, Any] = {
+            "tool_error_count": tool_errors,
+            "permission_denial_count": permission_denials,
+            "truncated": len(content) > RESULT_CHARACTERS,
+        }
+        if success and (usage := bounded_counts(terminal.get("usage"), _USAGE_FIELDS)):
+            metadata["usage"] = usage
+        if state == "failed":
+            metadata["error_code"] = cli_failure(terminal, fatal, stderr, code="native_error").code
+        return ParsedNativeResponse(state, content[:RESULT_CHARACTERS], session_id, metadata)