python-codex 0.1.14__tar.gz → 0.2.0__tar.gz

{python_codex-0.1.14 → python_codex-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-codex
-Version: 0.1.14
+Version: 0.2.0
 Summary: A minimal Python extraction of Codex's main agent loop
 License-File: LICENSE
 Requires-Python: >=3.6.2

{python_codex-0.1.14 → python_codex-0.2.0}/pycodex/agent.py

@@ -5,7 +5,7 @@ import re
 from typing import Callable
 
 from .context import ContextManager
-from .model import ModelClient, ResponsesIncompleteError
+from .model import ModelClient
 from .protocol import (
     AgentEvent,
     AssistantMessage,
@@ -18,6 +18,7 @@ from .protocol import (
     UserMessage,
 )
 from .tools import ExecCommandTool, ToolContext, ToolRegistry, UnifiedExecManager
+from .utils.truncation import truncate_tool_results_for_history
 from .utils import uuid7_string
 import typing
 
@@ -206,6 +207,7 @@ class Agent:
                     )
 
                 tool_results = await self._execute_tool_batch(turn_id, tool_calls)
+                tool_results = truncate_tool_results_for_history(tool_results)
                 self._history.extend(tool_results)
                 self._persist_history_items(tool_results)
                 follow_up_messages = self._build_follow_up_messages(tool_results)
@@ -219,6 +221,9 @@ class Agent:
         except TurnInterrupted:
             self._turn_running = False
             raise
+        except asyncio.CancelledError:
+            self._turn_running = False
+            raise
         except Exception as exc:
             context_usage = _usage_from_context_length_error(str(exc))
             if context_usage is not None:
@@ -349,14 +354,11 @@ class Agent:
     def _record_model_response_items(
         self,
         items: 'typing.Iterable[object]',
-        include_tool_calls: 'bool' = True,
     ) -> 'typing.Tuple[typing.Tuple[ConversationItem, ...], typing.List[ToolCall], typing.Union[str, None]]':
         persisted_response_items: 'typing.List[ConversationItem]' = []
         tool_calls: 'typing.List[ToolCall]' = []
         last_assistant_message = None
         for item in items:
-            if isinstance(item, ToolCall) and not include_tool_calls:
-                continue
             if not isinstance(item, (AssistantMessage, ToolCall, ReasoningItem)):
                 continue
             self._history.append(item)
@@ -413,13 +415,6 @@ class Agent:
                     prompt,
                     lambda event: self._handle_model_stream_event(turn_id, event),
                 )
-            except ResponsesIncompleteError as exc:
-                if exc.reason == "max_output_tokens":
-                    self._record_model_response_items(
-                        exc.partial_items,
-                        include_tool_calls=False,
-                    )
-                raise
             except Exception as exc:
                 error_message = str(exc)
                 if (

{python_codex-0.1.14 → python_codex-0.2.0}/pycodex/cli.py

@@ -289,6 +289,7 @@ def build_agent(
     system_prompt: 'typing.Union[str, None]' = None,
     session_mode: 'CliSessionMode' = "exec",
     collaboration_mode: 'CollaborationMode' = DEFAULT_COLLABORATION_MODE,
+    extra_contextual_user_messages: 'typing.Iterable[str]' = (),
 ) -> 'Agent':
     config_path = str(config_path)
     context_manager = ContextManager.from_codex_config(
@@ -297,6 +298,7 @@ def build_agent(
         base_instructions_override=system_prompt,
         collaboration_mode=collaboration_mode,
         include_collaboration_instructions=session_mode == "tui",
+        extra_contextual_user_messages=extra_contextual_user_messages,
     )
     session_id = getattr(client, "_session_id", None) or uuid7_string()
     if hasattr(client, "_session_id"):
@@ -306,6 +308,7 @@ def build_agent(
         profile,
         base_instructions_override=system_prompt,
         include_collaboration_instructions=False,
+        extra_contextual_user_messages=extra_contextual_user_messages,
     )
     runtime_environment = create_agent_runtime_environment()
     runtime_environment.request_user_input_manager.set_handler(None)

{python_codex-0.1.14 → python_codex-0.2.0}/pycodex/context.py

@@ -149,6 +149,7 @@ class ContextManager:
         include_permissions_instructions: 'bool' = True,
         include_skills_instructions: 'bool' = True,
         network_access: 'str' = "enabled",
+        extra_contextual_user_messages: 'typing.Iterable[str]' = (),
     ) -> 'None':
         self.cwd = Path.cwd().resolve()
         self._shell = get_shell_name()
@@ -166,6 +167,14 @@ class ContextManager:
         self._include_permissions_instructions = include_permissions_instructions
         self._include_skills_instructions = include_skills_instructions
         self._network_access = network_access
+        self._extra_contextual_user_messages = tuple(
+            text
+            for text in (
+                _normalize_text(message)
+                for message in extra_contextual_user_messages
+            )
+            if text is not None
+        )
         self._default_base_instructions = DEFAULT_BASE_INSTRUCTIONS_PATH.read_text(
             encoding="utf-8"
         )
@@ -183,6 +192,7 @@ class ContextManager:
         include_permissions_instructions: 'bool' = True,
         include_skills_instructions: 'bool' = True,
         network_access: 'str' = "enabled",
+        extra_contextual_user_messages: 'typing.Iterable[str]' = (),
     ) -> 'ContextManager':
         config = ContextConfig.from_codex_config(config_path, profile)
         return cls(
@@ -193,6 +203,7 @@ class ContextManager:
             include_permissions_instructions=include_permissions_instructions,
             include_skills_instructions=include_skills_instructions,
             network_access=network_access,
+            extra_contextual_user_messages=extra_contextual_user_messages,
         )
 
     @property
@@ -421,6 +432,7 @@ class ContextManager:
                 )
             )
         sections.append(self._serialize_environment_context())
+        sections.extend(self._extra_contextual_user_messages)
         if not sections:
             return []
         return [

{python_codex-0.1.14 → python_codex-0.2.0}/pycodex/model.py

@@ -298,13 +298,20 @@ class ResponsesModelClient:
                     prompt,
                     event_handler,
                 )
-            except ResponsesRetryableError as exc:
-                if _is_context_length_error_message(str(exc)):
+            except (ResponsesRetryableError, ResponsesIncompleteError) as exc:
+                if (
+                    isinstance(exc, ResponsesRetryableError)
+                    and _is_context_length_error_message(str(exc))
+                ):
                     raise ResponsesApiError(str(exc)) from exc
                 if retries >= max_retries:
                     raise
                 retries += 1
-                delay_seconds = exc.retry_delay_seconds
+                delay_seconds = (
+                    exc.retry_delay_seconds
+                    if isinstance(exc, ResponsesRetryableError)
+                    else None
+                )
                 if delay_seconds is None:
                     delay_seconds = self._retry_delay_seconds(retries)
                 event_handler(

{python_codex-0.1.14 → python_codex-0.2.0}/pycodex/runtime.py

@@ -165,6 +165,9 @@ class CliSubmissionQueue:
 
     async def _next_submission(self) -> '_QueuedSubmission':
         while True:
+            if self._agent._turn_running and not self._has_queue_active_turn():
+                await self._wait_for_agent_idle()
+                continue
             async with self._queue_lock:
                 queued: 'typing.Union[_QueuedSubmission, None]' = None
                 if self._steer_queue:
@@ -197,9 +200,16 @@ class CliSubmissionQueue:
                 future.set_exception(exc)
 
     def _has_active_turn(self) -> 'bool':
+        return self._has_queue_active_turn() or self._agent._turn_running
+
+    def _has_queue_active_turn(self) -> 'bool':
         current_task = self._current_task
         return current_task is not None and not current_task.done()
 
+    async def _wait_for_agent_idle(self) -> 'None':
+        while self._agent._turn_running:
+            await asyncio.sleep(0.01)
+
     def _handle_agent_event(self, event: 'AgentEvent') -> 'None':
         queued = self._current_submission
         if queued is None:

{python_codex-0.1.14 → python_codex-0.2.0}/pycodex/tools/code_mode_manager.py

@@ -21,11 +21,11 @@ from loguru import logger
 
 from ..compat import is_ascii, stream_writer_is_closing
 from ..protocol import JSONDict, JSONValue, ToolCall
+from ..utils.truncation import DEFAULT_MAX_OUTPUT_TOKENS
 from .base_tool import StructuredToolOutput, ToolContext, ToolRegistry
 import typing
 
 DEFAULT_WAIT_YIELD_TIME_MS = 10_000
-DEFAULT_MAX_OUTPUT_TOKENS = 10_000
 CHARS_PER_TOKEN = 4
 EXEC_PRAGMA_PREFIX = "// @exec:"
 WAIT_COMPLETION_GRACE_SECONDS = 0.02

{python_codex-0.1.14 → python_codex-0.2.0}/pycodex/tools/unified_exec_manager.py

@@ -21,15 +21,18 @@ from pathlib import Path
 from loguru import logger
 
 from ..compat import shlex_join, stream_writer_is_closing
+from ..utils.truncation import (
+    DEFAULT_MAX_OUTPUT_TOKENS,
+    approx_token_count,
+    formatted_truncate_text,
+)
 import typing
 
 DEFAULT_EXEC_YIELD_TIME_MS = 10_000
 DEFAULT_WRITE_STDIN_YIELD_TIME_MS = 250
-DEFAULT_MAX_OUTPUT_TOKENS = 10_000
 DEFAULT_LOGIN = True
 DEFAULT_TTY = False
 DEFAULT_SESSION_ID_START = 1000
-APPROX_BYTES_PER_TOKEN = 4
 UNIFIED_EXEC_OUTPUT_MAX_BYTES = 1024 * 1024
 UNIFIED_EXEC_OUTPUT_SCHEMA = {
     "type": "object",
@@ -63,94 +66,6 @@ UNIFIED_EXEC_OUTPUT_SCHEMA = {
     "additionalProperties": False,
 }
 
-
-def _approx_token_count(text: 'str') -> 'int':
-    if not text:
-        return 0
-    byte_length = len(text.encode("utf-8"))
-    return max(1, (byte_length + APPROX_BYTES_PER_TOKEN - 1) // APPROX_BYTES_PER_TOKEN)
-
-
-def _approx_bytes_for_tokens(token_count: 'int') -> 'int':
-    return max(token_count, 0) * APPROX_BYTES_PER_TOKEN
-
-
-def _approx_tokens_from_byte_count(byte_count: 'int') -> 'int':
-    if byte_count <= 0:
-        return 0
-    return (byte_count + APPROX_BYTES_PER_TOKEN - 1) // APPROX_BYTES_PER_TOKEN
-
-
-def _split_budget(byte_budget: 'int') -> 'typing.Tuple[int, int]':
-    left_budget = byte_budget // 2
-    return left_budget, byte_budget - left_budget
-
-
-def _split_string(
-    text: 'str',
-    beginning_bytes: 'int',
-    end_bytes: 'int',
-) -> 'typing.Tuple[str, str]':
-    if not text:
-        return "", ""
-
-    total_bytes = len(text.encode("utf-8"))
-    tail_start_target = max(total_bytes - end_bytes, 0)
-    prefix_end = 0
-    suffix_start = len(text)
-    suffix_started = False
-    current_byte = 0
-
-    for index, char in enumerate(text):
-        char_bytes = len(char.encode("utf-8"))
-        char_start = current_byte
-        char_end = current_byte + char_bytes
-        if char_end <= beginning_bytes:
-            prefix_end = index + 1
-            current_byte = char_end
-            continue
-        if char_start >= tail_start_target:
-            if not suffix_started:
-                suffix_start = index
-                suffix_started = True
-            current_byte = char_end
-            continue
-        current_byte = char_end
-
-    if suffix_start < prefix_end:
-        suffix_start = prefix_end
-
-    return text[:prefix_end], text[suffix_start:]
-
-
-def _truncate_text(text: 'str', max_tokens: 'int') -> 'str':
-    if not text:
-        return ""
-
-    max_bytes = _approx_bytes_for_tokens(max_tokens)
-    total_bytes = len(text.encode("utf-8"))
-    if total_bytes <= max_bytes:
-        return text
-
-    removed_tokens = _approx_tokens_from_byte_count(total_bytes - max_bytes)
-    marker = f"\u2026{removed_tokens} tokens truncated\u2026"
-    if max_bytes == 0:
-        return marker
-
-    left_budget, right_budget = _split_budget(max_bytes)
-    prefix, suffix = _split_string(text, left_budget, right_budget)
-    return f"{prefix}{marker}{suffix}"
-
-
-def _formatted_truncate_text(text: 'str', max_tokens: 'int') -> 'str':
-    byte_budget = _approx_bytes_for_tokens(max_tokens)
-    if len(text.encode("utf-8")) <= byte_budget:
-        return text
-
-    total_lines = len(text.splitlines())
-    return f"Total output lines: {total_lines}\n\n{_truncate_text(text, max_tokens)}"
-
-
 @dataclass
 class _HeadTailBuffer:
     max_bytes: 'int' = UNIFIED_EXEC_OUTPUT_MAX_BYTES
@@ -431,11 +346,11 @@ class UnifiedExecManager:
         return [shell_path, "-lc" if login else "-c", cmd]
 
     def _estimate_token_count(self, output: 'str') -> 'typing.Union[int, None]':
-        return _approx_token_count(output)
+        return approx_token_count(output)
 
     def _truncate_output(self, output: 'str', max_output_tokens: 'typing.Union[int, None]') -> 'str':
         token_budget = DEFAULT_MAX_OUTPUT_TOKENS if max_output_tokens is None else max_output_tokens
-        return _formatted_truncate_text(output, max(token_budget, 0))
+        return formatted_truncate_text(output, max(token_budget, 0))
 
     def _tty_echo(self, chars: 'str') -> 'bytes':
         normalized = chars.replace("\n", "\r\n")

{python_codex-0.1.14 → python_codex-0.2.0}/pycodex/utils/session_persist.py

@@ -198,7 +198,18 @@ def load_resumed_session(
     session = sessions[resume_index - 1]
     thread_id = session["thread_id"]
     rollout_path = Path(session["rollout_path"])
-    thread_name = _latest_thread_names_by_id(codex_home).get(thread_id)
+    return load_resumed_session_path(
+        rollout_path,
+        thread_name=_latest_thread_names_by_id(codex_home).get(thread_id),
+    )
+
+
+def load_resumed_session_path(
+    rollout_path: 'typing.Union[str, Path]',
+    thread_name: 'typing.Union[str, None]' = None,
+) -> 'typing.Dict[str, object]':
+    rollout_path = Path(rollout_path)
+    thread_id = _thread_id_from_rollout_path(rollout_path) or ""
     session_id = thread_id
     history: 'typing.List[ConversationItem]' = []
     saw_user_turn = False
@@ -245,6 +256,10 @@ def load_resumed_session(
     if not history:
         raise ValueError(f"No resumable history found in {rollout_path}")
 
+    history = _trim_incomplete_tool_call_tail(history)
+    if not history:
+        raise ValueError(f"No resumable history found in {rollout_path}")
+
     turns = conversation_history_to_turns(history)
     title = thread_name or (shorten_title(turns[0][0]) if turns else thread_id)
     return {
@@ -277,6 +292,32 @@ def conversation_history_to_turns(
     return tuple(turns)
 
 
+def _trim_incomplete_tool_call_tail(
+    history: 'typing.List[ConversationItem]',
+) -> 'typing.List[ConversationItem]':
+    pending_call_ids: 'typing.Set[str]' = set()
+    call_indexes: 'typing.Dict[str, int]' = {}
+
+    for index, item in enumerate(history):
+        if isinstance(item, ToolCall):
+            pending_call_ids.add(item.call_id)
+            call_indexes[item.call_id] = index
+            continue
+        if isinstance(item, ToolResult):
+            pending_call_ids.discard(item.call_id)
+
+    if not pending_call_ids:
+        return history
+
+    trim_start = min(call_indexes[call_id] for call_id in pending_call_ids)
+    while trim_start > 0 and isinstance(
+        history[trim_start - 1],
+        (AssistantMessage, ReasoningItem, ToolCall),
+    ):
+        trim_start -= 1
+    return history[:trim_start]
+
+
 def _latest_thread_names_by_id(codex_home: 'Path') -> 'typing.Dict[str, str]':
     index_path = codex_home / SESSION_INDEX_FILENAME
     if not index_path.exists():

python_codex-0.2.0/pycodex/utils/truncation.py

@@ -0,0 +1,206 @@
+"""Shared truncation helpers for model-visible tool output."""
+
+import math
+
+from ..protocol import JSONValue, ToolResult
+import typing
+
+DEFAULT_MAX_OUTPUT_TOKENS = 10_000
+TRUNCATION_SERIALIZATION_BUDGET_MULTIPLIER = 1.2
+HISTORY_TOOL_OUTPUT_TOKENS = int(
+    math.ceil(DEFAULT_MAX_OUTPUT_TOKENS * TRUNCATION_SERIALIZATION_BUDGET_MULTIPLIER)
+)
+APPROX_BYTES_PER_TOKEN = 4
+
+
+def approx_token_count(text: 'str') -> 'int':
+    """Estimate token count using the upstream Codex 4-bytes-per-token rule."""
+    if not text:
+        return 0
+    byte_length = len(text.encode("utf-8"))
+    return max(
+        1,
+        (byte_length + APPROX_BYTES_PER_TOKEN - 1) // APPROX_BYTES_PER_TOKEN,
+    )
+
+
+def formatted_truncate_text(text: 'str', max_tokens: 'int') -> 'str':
+    """Format a direct tool response with line count plus middle truncation."""
+    byte_budget = _approx_bytes_for_tokens(max_tokens)
+    if len(text.encode("utf-8")) <= byte_budget:
+        return text
+
+    total_lines = len(text.splitlines())
+    return f"Total output lines: {total_lines}\n\n{_truncate_text(text, max_tokens)}"
+
+
+def truncate_tool_result_for_history(
+    result: 'ToolResult',
+) -> 'ToolResult':
+    """Truncate model-visible ToolResult content before storing it in history."""
+    if result.content_items is not None:
+        truncated_content_items = _truncate_content_items(
+            result.content_items,
+            HISTORY_TOOL_OUTPUT_TOKENS,
+        )
+        if truncated_content_items == result.content_items:
+            return result
+        return ToolResult(
+            call_id=result.call_id,
+            name=result.name,
+            output=result.output,
+            content_items=truncated_content_items,
+            success=result.success,
+            is_error=result.is_error,
+            tool_type=result.tool_type,
+        )
+
+    output_text = _tool_output_text(result.output)
+    truncated_output = _truncate_text(output_text, HISTORY_TOOL_OUTPUT_TOKENS)
+    if truncated_output == output_text:
+        return result
+    return ToolResult(
+        call_id=result.call_id,
+        name=result.name,
+        output=truncated_output,
+        success=result.success,
+        is_error=result.is_error,
+        tool_type=result.tool_type,
+    )
+
+
+def truncate_tool_results_for_history(
+    results: 'typing.Iterable[ToolResult]',
+) -> 'typing.List[ToolResult]':
+    """Apply history-layer truncation to a batch of completed tool results."""
+    return [
+        truncate_tool_result_for_history(result)
+        for result in results
+    ]
+
+
+def _tool_output_text(output: 'JSONValue') -> 'str':
+    if isinstance(output, str):
+        return output
+
+    import json
+
+    return json.dumps(
+        output,
+        ensure_ascii=False,
+        separators=(",", ":"),
+    )
+
+
+def _truncate_content_items(
+    content_items: 'typing.Tuple[typing.Dict[str, typing.Any], ...]',
+    token_limit: 'int',
+) -> 'typing.Tuple[typing.Dict[str, typing.Any], ...]':
+    output: 'typing.List[typing.Dict[str, typing.Any]]' = []
+    remaining_budget = token_limit
+    omitted_text_items = 0
+
+    for item in content_items:
+        if item.get("type") != "input_text":
+            output.append(dict(item))
+            continue
+
+        text = str(item.get("text", ""))
+        if remaining_budget <= 0:
+            omitted_text_items += 1
+            continue
+
+        cost = approx_token_count(text)
+        if cost <= remaining_budget:
+            output.append(dict(item))
+            remaining_budget -= cost
+            continue
+
+        truncated_text = _truncate_text(text, remaining_budget)
+        if truncated_text:
+            next_item = dict(item)
+            next_item["text"] = truncated_text
+            output.append(next_item)
+        else:
+            omitted_text_items += 1
+        remaining_budget = 0
+
+    if omitted_text_items > 0:
+        output.append(
+            {
+                "type": "input_text",
+                "text": f"[omitted {omitted_text_items} text items ...]",
+            }
+        )
+    return tuple(output)
+
+
+def _approx_tokens_from_byte_count(byte_count: 'int') -> 'int':
+    if byte_count <= 0:
+        return 0
+    return (byte_count + APPROX_BYTES_PER_TOKEN - 1) // APPROX_BYTES_PER_TOKEN
+
+
+def _approx_bytes_for_tokens(token_count: 'int') -> 'int':
+    return max(token_count, 0) * APPROX_BYTES_PER_TOKEN
+
+
+def _split_budget(byte_budget: 'int') -> 'typing.Tuple[int, int]':
+    left_budget = byte_budget // 2
+    return left_budget, byte_budget - left_budget
+
+
+def _split_string(
+    text: 'str',
+    beginning_bytes: 'int',
+    end_bytes: 'int',
+) -> 'typing.Tuple[str, str]':
+    if not text:
+        return "", ""
+
+    total_bytes = len(text.encode("utf-8"))
+    tail_start_target = max(total_bytes - end_bytes, 0)
+    prefix_end = 0
+    suffix_start = len(text)
+    suffix_started = False
+    current_byte = 0
+
+    for index, char in enumerate(text):
+        char_bytes = len(char.encode("utf-8"))
+        char_start = current_byte
+        char_end = current_byte + char_bytes
+        if char_end <= beginning_bytes:
+            prefix_end = index + 1
+            current_byte = char_end
+            continue
+        if char_start >= tail_start_target:
+            if not suffix_started:
+                suffix_start = index
+                suffix_started = True
+            current_byte = char_end
+            continue
+        current_byte = char_end
+
+    if suffix_start < prefix_end:
+        suffix_start = prefix_end
+
+    return text[:prefix_end], text[suffix_start:]
+
+
+def _truncate_text(text: 'str', max_tokens: 'int') -> 'str':
+    if not text:
+        return ""
+
+    max_bytes = _approx_bytes_for_tokens(max_tokens)
+    total_bytes = len(text.encode("utf-8"))
+    if total_bytes <= max_bytes:
+        return text
+
+    removed_tokens = _approx_tokens_from_byte_count(total_bytes - max_bytes)
+    marker = f"\u2026{removed_tokens} tokens truncated\u2026"
+    if max_bytes == 0:
+        return marker
+
+    left_budget, right_budget = _split_budget(max_bytes)
+    prefix, suffix = _split_string(text, left_budget, right_budget)
+    return f"{prefix}{marker}{suffix}"

{python_codex-0.1.14 → python_codex-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "python-codex"
-version = "0.1.14"
+version = "0.2.0"
 description = "A minimal Python extraction of Codex's main agent loop"
 readme = "README.md"
 requires-python = ">=3.6.2"
@@ -47,6 +47,9 @@ default-groups = []
 packages = ["pycodex", "responses_server", "workspace_server"]
 artifacts = ["workspace_server/workspace.html"]
 
+[tool.hatch.build.targets.sdist]
+exclude = [".venv*"]
+
 [tool.pytest.ini_options]
 pythonpath = ["."]
 asyncio_mode = "auto"