atif-sql 0.2.0__py3-none-any.whl

atif_converter/domain/codex_fidelity.py

@@ -0,0 +1,124 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""The fidelity policy for Codex CLI: what harbor's Codex converter loses.
+
+Pure value objects — no harbor import, no I/O. The Claude Code half of this
+policy lives in :mod:`atif_converter.domain.fidelity`; this module is its
+Codex counterpart and follows the same discipline: NAME each upstream gap
+rather than fix it, and let the unit tests pin it so a harbor bump that
+changes behavior trips the suite.
+
+Every member below was verified empirically on 2026-09-11 against
+harbor==0.22.0's Codex conversion over real rollouts from ``~/.codex/sessions``
+written by codex-cli 0.153.4 and 0.154.0. Our converter
+(:mod:`atif_converter.domain.codex_conversion`) is a parity port of that
+conversion, so these losses are ours now, by choice.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+
+
+class CodexRecordType(Enum):
+    """Taxonomy of raw Codex rollout JSONL record types.
+
+    The top-level ``type`` field values seen in the wild, plus ``OTHER`` for
+    anything new. Only ``RESPONSE_ITEM`` records become ATIF steps;
+    ``EVENT_MSG`` and ``TURN_CONTEXT`` are read for turn ids and token
+    counts but produce no step of their own, and the rest are dropped
+    (:attr:`CodexFidelityGap.NON_ITEM_RECORDS_DROPPED`).
+    """
+
+    SESSION_META = "session_meta"
+    RESPONSE_ITEM = "response_item"
+    EVENT_MSG = "event_msg"
+    TURN_CONTEXT = "turn_context"
+    TOKEN_USAGE_RECORD = "token_usage_record"  # noqa: S105 — a record type, not a secret
+    WORLD_STATE = "world_state"
+    COMPACTED = "compacted"
+    OTHER = "other"
+
+
+#: Record types harbor 0.22.0 turns into ATIF steps. ``EVENT_MSG`` and
+#: ``TURN_CONTEXT`` are deliberately absent: the converter reads them for
+#: ``token_count`` metrics and turn ids, and emits no step for either, so
+#: counting them as converted would overstate what the trajectory carries.
+CODEX_CONVERTIBLE_RECORD_TYPES: frozenset[CodexRecordType] = frozenset(
+    {CodexRecordType.RESPONSE_ITEM}
+)
+
+#: ``response_item`` payload types that DO reach a step, so a rollout whose
+#: response items are all of some other type is honestly reported as
+#: converting none of them.
+CODEX_CONVERTIBLE_ITEM_TYPES: frozenset[str] = frozenset(
+    {
+        "message",
+        "function_call",
+        "function_call_output",
+        "custom_tool_call",
+        "custom_tool_call_output",
+        "web_search_call",
+    }
+)
+
+
+class CodexFidelityGap(Enum):
+    """The known Codex conversion gaps, inherited from harbor 0.22.0.
+
+    Values are namespaced ``codex_*`` because they share the
+    ``gaps_observed`` array in ``loss_report.json`` with the Claude Code
+    gaps: a reader of one corpus must be able to tell which policy a gap
+    came from without consulting a second column.
+    """
+
+    #: (1) RETIRED. ``codex_single_rollout_per_directory`` named harbor's
+    #: directory globbing, which converted only ``max(*.jsonl)`` of a session
+    #: dir. Our converter reads the one rollout it is given, so there is no
+    #: directory and no gap. The value is not reused; a corpus materialized
+    #: before the port still carries it.
+
+    #: (2) Only ``response_item`` records become steps. ``world_state``,
+    #: ``token_usage_record`` and ``compacted`` records are dropped outright,
+    #: and ``event_msg`` / ``turn_context`` contribute only turn ids and token
+    #: counts. Verified: a 17-record rollout yields 7 steps.
+    NON_ITEM_RECORDS_DROPPED = "codex_non_item_records_dropped"
+
+    #: (3) ``compacted`` records carry the compaction summary plus the
+    #: ``replacement_history`` the model actually saw afterwards. Neither
+    #: reaches the trajectory, so a compaction boundary is invisible in the
+    #: step list and the pre-compaction turns simply stop.
+    COMPACTION_UNHANDLED = "codex_compaction_unhandled"
+
+    #: (4) Reasoning is read from ``reasoning.summary`` only. Recent Codex
+    #: builds ship an EMPTY summary and put the content in
+    #: ``encrypted_content``, so reasoning is lost entirely: verified on a
+    #: rollout with 4 reasoning records and 0 steps carrying
+    #: ``reasoning_content``.
+    REASONING_ENCRYPTED_DROPPED = "codex_reasoning_encrypted_dropped"
+
+    #: (5) ``tool_search_call`` / ``tool_search_output`` items (Codex's
+    #: deferred-tool-loading search) match no branch of the payload-type
+    #: dispatch, so a tool search and its result vanish without a warning.
+    TOOL_SEARCH_CALLS_DROPPED = "codex_tool_search_calls_dropped"
+
+    #: (6) ``response_item.payload.id`` (``msg_*`` / ``call_*`` / ``rs_*``) is
+    #: never carried into ``Step.extra``, so step-to-raw-record identity is
+    #: unrecoverable from the trajectory alone — the same shape of loss as
+    #: Claude Code's gap 7, and repaired the same way by the enrichment pass.
+    ITEM_IDS_NOT_PRESERVED = "codex_item_ids_not_preserved"
+
+    #: (7) A ``developer`` role message becomes ATIF ``source="system"``,
+    #: which is also where a genuine system message lands. The two are
+    #: indistinguishable in the trajectory; ``edges.jsonl`` keeps the raw
+    #: role, so the distinction survives in the corpus but not in the steps.
+    DEVELOPER_ROLE_FLATTENED_TO_SYSTEM = "codex_developer_role_flattened_to_system"
+
+
+#: Gaps every harbor 0.22.0 Codex conversion exhibits regardless of rollout
+#: content — the structural set, mirroring the Claude Code policy's.
+CODEX_STRUCTURAL_GAPS: frozenset[CodexFidelityGap] = frozenset(
+    {
+        CodexFidelityGap.ITEM_IDS_NOT_PRESERVED,
+    }
+)

atif_converter/domain/codex_nested_calls.py

@@ -0,0 +1,165 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""A Codex code-mode script's nested MCP calls and commands, as ATIF tool calls.
+
+In code mode Codex calls one ``exec`` tool whose JavaScript calls
+``tools.mcp__<server>__<tool>(...)`` and ``tools.exec_command(...)``. The
+rollout records each nested call as an ``event_msg`` ``item_completed`` item
+(``McpToolCall`` or ``CommandExecution``, id ``exec-<uuid>``) between the
+script's ``custom_tool_call`` and its ``custom_tool_call_output``, and harbor's
+converter turns none of them into a tool call. Pure functions over one such
+item: the :class:`~atif_converter.domain.atif.ToolCall` and
+:class:`~atif_converter.domain.atif.ObservationResult` JSON it becomes. Which
+script an item belongs to is :mod:`atif_converter.domain.result_signals`'s
+attribution; that pass also writes the result's ``is_error`` / ``exit_code``.
+
+The tool call
+-------------
+``tool_call_id``
+    The item's own ``id``.
+``function_name``
+    ``mcp__<server>__<tool>`` for an MCP call, the name the script called;
+    ``exec_command`` for a command.
+``arguments``
+    The MCP call's ``arguments`` object; ``{"cmd": <script>}`` for a command,
+    unwrapped from Codex's ``[<shell>, "-c" | "-lc", <script>]`` argv.
+``extra``
+    ``nested_in`` (the ``exec`` call's id), ``via`` (``"exec"``) and, when the
+    item states it, ``duration_ms``.
+
+The result's content is the MCP result's content blocks rendered the way the
+port renders a top-level tool output's blocks
+(:func:`~atif_converter.domain.codex_conversion.structured_output_text`), or a
+command's ``aggregated_output``.
+"""
+
+from __future__ import annotations
+
+import json
+import shlex
+from typing import Any, Final
+
+from atif_converter.domain.codex_conversion import structured_output_text
+
+#: Completed item types that become a nested tool call.
+NESTED_CALL_ITEM_TYPES: Final[frozenset[str]] = frozenset({"McpToolCall", "CommandExecution"})
+
+#: The ``function_name`` a nested command gets: the tool the script called.
+COMMAND_TOOL_NAME: Final = "exec_command"
+
+#: ``extra.via`` on every nested call: the tool that ran it.
+VIA_EXEC: Final = "exec"
+
+#: A shell's flags before the script in Codex's command argv.
+_SHELL_SCRIPT_FLAGS: Final[frozenset[str]] = frozenset({"-c", "-lc"})
+
+#: The length of a ``[<shell>, <flag>, <script>]`` argv.
+_SHELL_ARGV_LENGTH: Final = 3
+
+_NANOS_PER_MS: Final = 1_000_000
+_MS_PER_SEC: Final = 1000
+
+
+def _json_text(value: Any) -> str:
+    return json.dumps(value, ensure_ascii=False)
+
+
+def _arguments(value: Any) -> dict[str, Any]:
+    """An MCP item's ``arguments`` as an object, by the port's rule for a call's arguments."""
+    if isinstance(value, str):
+        try:
+            value = json.loads(value)
+        except json.JSONDecodeError:
+            return {"input": value}
+    if value is None:
+        return {}
+    return value if isinstance(value, dict) else {"value": value}
+
+
+def _command_text(command: Any) -> Any:
+    """The script a command ran: unwrapped from its shell argv, else the argv as one line."""
+    if isinstance(command, list) and all(isinstance(part, str) for part in command):
+        if len(command) == _SHELL_ARGV_LENGTH and command[1] in _SHELL_SCRIPT_FLAGS:
+            return command[2]
+        return shlex.join(command)
+    return command
+
+
+def _duration_ms(item: dict[str, Any], payload: dict[str, Any]) -> float | None:
+    """The call's duration: the item's ``{secs, nanos}``, else the payload's two timestamps."""
+    duration = item.get("duration")
+    if isinstance(duration, dict):
+        secs, nanos = duration.get("secs"), duration.get("nanos")
+        if isinstance(secs, int) and isinstance(nanos, int):
+            return round(secs * _MS_PER_SEC + nanos / _NANOS_PER_MS, 3)
+    started, completed = payload.get("started_at_ms"), payload.get("completed_at_ms")
+    if isinstance(started, int) and isinstance(completed, int) and completed >= started:
+        return float(completed - started)
+    return None
+
+
+def nested_tool_call(payload: dict[str, Any], exec_call_id: str) -> dict[str, Any]:
+    """The ``ToolCall`` JSON one ``item_completed`` payload's item becomes, nested in ``exec_call_id``."""
+    item: dict[str, Any] = payload["item"]
+    if item.get("type") == "CommandExecution":
+        function_name = COMMAND_TOOL_NAME
+        arguments: dict[str, Any] = {"cmd": _command_text(item.get("command"))}
+    else:
+        function_name = f"mcp__{item.get('server')}__{item.get('tool')}"
+        arguments = _arguments(item.get("arguments"))
+    extra: dict[str, Any] = {"nested_in": exec_call_id, "via": VIA_EXEC}
+    duration = _duration_ms(item, payload)
+    if duration is not None:
+        extra["duration_ms"] = duration
+    return {
+        "tool_call_id": item["id"],
+        "function_name": function_name,
+        "arguments": arguments,
+        "extra": extra,
+    }
+
+
+def _text(value: Any) -> str:
+    return value if isinstance(value, str) else _json_text(value)
+
+
+def _result_text(result: dict[str, Any]) -> str:
+    """An MCP result's text: its ``Err``, else its content blocks, else its JSON text."""
+    if "Err" in result:
+        return _text(result["Err"])
+    content = result.get("content")
+    if not isinstance(content, list):
+        return _json_text(result)
+    text = structured_output_text(content)
+    if text is not None:
+        return text
+    return _json_text(content) if content else ""
+
+
+def _mcp_content(item: dict[str, Any]) -> str | None:
+    """The result's text, else the error's message (an item that failed before any result)."""
+    result = item.get("result")
+    if isinstance(result, dict):
+        return _result_text(result)
+    error = item.get("error")
+    message = error.get("message") if isinstance(error, dict) else None
+    if isinstance(message, str):
+        return message
+    return None if error is None else _text(error)
+
+
+def nested_result_content(item: dict[str, Any]) -> str | None:
+    """The observation content of one nested item: its MCP result or command output."""
+    if item.get("type") == "CommandExecution":
+        output = item.get("aggregated_output")
+        return output if isinstance(output, str) else None
+    return _mcp_content(item)
+
+
+__all__ = [
+    "COMMAND_TOOL_NAME",
+    "NESTED_CALL_ITEM_TYPES",
+    "VIA_EXEC",
+    "nested_result_content",
+    "nested_tool_call",
+]

atif_converter/domain/edges.py

@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""edges.jsonl: one line per RAW session record — the uuid-keyed skeleton.
+
+Pure functions over already-parsed record dicts; no harbor, no I/O. The
+corpus contract (docs/CONTRACT.md §corpus layout) fixes the line shape:
+
+    {uuid, parent_uuid, message_id, type, ts, is_sidechain,
+     is_compact_summary, source_file, tool_use_ids: [..]}
+
+``tool_use_ids`` carries the ids of ``tool_use`` blocks for assistant
+records and the ``tool_use_id`` of ``tool_result`` blocks for user records
+— the join key back to ATIF ``tool_calls`` / ``observation.source_call_ids``.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+#: Stable key order for one edges.jsonl line (contract-fixed shape).
+EDGE_FIELDS: tuple[str, ...] = (
+    "uuid",
+    "parent_uuid",
+    "message_id",
+    "type",
+    "ts",
+    "is_sidechain",
+    "is_compact_summary",
+    "source_file",
+    "tool_use_ids",
+)
+
+
+def _tool_use_ids(record: dict[str, Any]) -> list[str]:
+    """Extract tool_use ids (assistant) or tool_result ids (user) in block order."""
+    message = record.get("message")
+    if not isinstance(message, dict):
+        return []
+    content = message.get("content")
+    if not isinstance(content, list):
+        return []
+
+    record_type = record.get("type")
+    ids: list[str] = []
+    for block in content:
+        if not isinstance(block, dict):
+            continue
+        if record_type == "assistant" and block.get("type") == "tool_use":
+            block_id = block.get("id")
+            if isinstance(block_id, str) and block_id:
+                ids.append(block_id)
+        elif record_type == "user" and block.get("type") == "tool_result":
+            block_id = block.get("tool_use_id")
+            if isinstance(block_id, str) and block_id:
+                ids.append(block_id)
+    return ids
+
+
+def edge_from_record(record: dict[str, Any], source_file: str) -> dict[str, Any]:
+    """Project one raw record onto the contract's edge shape.
+
+    ``source_file`` is the record's origin path RELATIVE to the session
+    JSONL's parent directory (e.g. ``<sid>.jsonl`` or
+    ``<sid>/subagents/workflows/wf_x/agent-y.jsonl``).
+    """
+    message = record.get("message")
+    message_id = message.get("id") if isinstance(message, dict) else None
+    return {
+        "uuid": record.get("uuid"),
+        "parent_uuid": record.get("parentUuid"),
+        "message_id": message_id if isinstance(message_id, str) else None,
+        "type": record.get("type"),
+        "ts": record.get("timestamp"),
+        "is_sidechain": bool(record.get("isSidechain", False)),
+        "is_compact_summary": bool(record.get("isCompactSummary", False)),
+        "source_file": source_file,
+        "tool_use_ids": _tool_use_ids(record),
+    }
+
+
+def build_edges(records: list[tuple[dict[str, Any], str]]) -> list[dict[str, Any]]:
+    """Build all edges from ``(record, source_file)`` pairs.
+
+    Ordered deterministically by ``(ts, uuid)``, with missing values sorting
+    first as empty strings.
+    """
+    edges = [edge_from_record(record, source_file) for record, source_file in records]
+    try:
+        edges.sort(key=lambda e: (e["ts"] or "", e["uuid"] or ""))
+    except TypeError:
+        # A `timestamp` or `uuid` that is not a string (an object, a list, a number beside a
+        # string) cannot be compared with the others; the fuzzer found the `TypeError` this
+        # raised. Such values sort as missing ones do. Only an order that raised takes this
+        # branch, so every input that converted before keeps its exact order.
+        edges.sort(key=lambda e: (_sort_text(e["ts"]), _sort_text(e["uuid"])))
+    return edges
+
+
+def _sort_text(value: object) -> str:
+    """A sort key part: the string itself, or ``""`` for a missing or non-string value."""
+    return value if isinstance(value, str) else ""
+
+
+def edge_to_jsonl_line(edge: dict[str, Any]) -> str:
+    """Serialize one edge as a compact, key-order-stable JSONL line."""
+    ordered = {key: edge.get(key) for key in EDGE_FIELDS}
+    return json.dumps(ordered, separators=(",", ":"))
+
+
+def edges_jsonl_lines(records: list[tuple[dict[str, Any], str]]) -> list[str]:
+    """``build_edges`` + serialization: the ready-to-write edges.jsonl body."""
+    return [edge_to_jsonl_line(edge) for edge in build_edges(records)]