python-codex 0.1.14__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pycodex/agent.py +6 -11
- pycodex/cli.py +3 -0
- pycodex/context.py +12 -0
- pycodex/model.py +10 -3
- pycodex/runtime.py +10 -0
- pycodex/tools/code_mode_manager.py +1 -1
- pycodex/tools/unified_exec_manager.py +7 -92
- pycodex/utils/session_persist.py +42 -1
- pycodex/utils/truncation.py +206 -0
- {python_codex-0.1.14.dist-info → python_codex-0.2.0.dist-info}/METADATA +1 -1
- {python_codex-0.1.14.dist-info → python_codex-0.2.0.dist-info}/RECORD +17 -16
- workspace_server/__init__.py +2 -0
- workspace_server/app.py +473 -109
- workspace_server/workspace.html +144 -68
- {python_codex-0.1.14.dist-info → python_codex-0.2.0.dist-info}/WHEEL +0 -0
- {python_codex-0.1.14.dist-info → python_codex-0.2.0.dist-info}/entry_points.txt +0 -0
- {python_codex-0.1.14.dist-info → python_codex-0.2.0.dist-info}/licenses/LICENSE +0 -0
pycodex/agent.py
CHANGED
|
@@ -5,7 +5,7 @@ import re
|
|
|
5
5
|
from typing import Callable
|
|
6
6
|
|
|
7
7
|
from .context import ContextManager
|
|
8
|
-
from .model import ModelClient
|
|
8
|
+
from .model import ModelClient
|
|
9
9
|
from .protocol import (
|
|
10
10
|
AgentEvent,
|
|
11
11
|
AssistantMessage,
|
|
@@ -18,6 +18,7 @@ from .protocol import (
|
|
|
18
18
|
UserMessage,
|
|
19
19
|
)
|
|
20
20
|
from .tools import ExecCommandTool, ToolContext, ToolRegistry, UnifiedExecManager
|
|
21
|
+
from .utils.truncation import truncate_tool_results_for_history
|
|
21
22
|
from .utils import uuid7_string
|
|
22
23
|
import typing
|
|
23
24
|
|
|
@@ -206,6 +207,7 @@ class Agent:
|
|
|
206
207
|
)
|
|
207
208
|
|
|
208
209
|
tool_results = await self._execute_tool_batch(turn_id, tool_calls)
|
|
210
|
+
tool_results = truncate_tool_results_for_history(tool_results)
|
|
209
211
|
self._history.extend(tool_results)
|
|
210
212
|
self._persist_history_items(tool_results)
|
|
211
213
|
follow_up_messages = self._build_follow_up_messages(tool_results)
|
|
@@ -219,6 +221,9 @@ class Agent:
|
|
|
219
221
|
except TurnInterrupted:
|
|
220
222
|
self._turn_running = False
|
|
221
223
|
raise
|
|
224
|
+
except asyncio.CancelledError:
|
|
225
|
+
self._turn_running = False
|
|
226
|
+
raise
|
|
222
227
|
except Exception as exc:
|
|
223
228
|
context_usage = _usage_from_context_length_error(str(exc))
|
|
224
229
|
if context_usage is not None:
|
|
@@ -349,14 +354,11 @@ class Agent:
|
|
|
349
354
|
def _record_model_response_items(
|
|
350
355
|
self,
|
|
351
356
|
items: 'typing.Iterable[object]',
|
|
352
|
-
include_tool_calls: 'bool' = True,
|
|
353
357
|
) -> 'typing.Tuple[typing.Tuple[ConversationItem, ...], typing.List[ToolCall], typing.Union[str, None]]':
|
|
354
358
|
persisted_response_items: 'typing.List[ConversationItem]' = []
|
|
355
359
|
tool_calls: 'typing.List[ToolCall]' = []
|
|
356
360
|
last_assistant_message = None
|
|
357
361
|
for item in items:
|
|
358
|
-
if isinstance(item, ToolCall) and not include_tool_calls:
|
|
359
|
-
continue
|
|
360
362
|
if not isinstance(item, (AssistantMessage, ToolCall, ReasoningItem)):
|
|
361
363
|
continue
|
|
362
364
|
self._history.append(item)
|
|
@@ -413,13 +415,6 @@ class Agent:
|
|
|
413
415
|
prompt,
|
|
414
416
|
lambda event: self._handle_model_stream_event(turn_id, event),
|
|
415
417
|
)
|
|
416
|
-
except ResponsesIncompleteError as exc:
|
|
417
|
-
if exc.reason == "max_output_tokens":
|
|
418
|
-
self._record_model_response_items(
|
|
419
|
-
exc.partial_items,
|
|
420
|
-
include_tool_calls=False,
|
|
421
|
-
)
|
|
422
|
-
raise
|
|
423
418
|
except Exception as exc:
|
|
424
419
|
error_message = str(exc)
|
|
425
420
|
if (
|
pycodex/cli.py
CHANGED
|
@@ -289,6 +289,7 @@ def build_agent(
|
|
|
289
289
|
system_prompt: 'typing.Union[str, None]' = None,
|
|
290
290
|
session_mode: 'CliSessionMode' = "exec",
|
|
291
291
|
collaboration_mode: 'CollaborationMode' = DEFAULT_COLLABORATION_MODE,
|
|
292
|
+
extra_contextual_user_messages: 'typing.Iterable[str]' = (),
|
|
292
293
|
) -> 'Agent':
|
|
293
294
|
config_path = str(config_path)
|
|
294
295
|
context_manager = ContextManager.from_codex_config(
|
|
@@ -297,6 +298,7 @@ def build_agent(
|
|
|
297
298
|
base_instructions_override=system_prompt,
|
|
298
299
|
collaboration_mode=collaboration_mode,
|
|
299
300
|
include_collaboration_instructions=session_mode == "tui",
|
|
301
|
+
extra_contextual_user_messages=extra_contextual_user_messages,
|
|
300
302
|
)
|
|
301
303
|
session_id = getattr(client, "_session_id", None) or uuid7_string()
|
|
302
304
|
if hasattr(client, "_session_id"):
|
|
@@ -306,6 +308,7 @@ def build_agent(
|
|
|
306
308
|
profile,
|
|
307
309
|
base_instructions_override=system_prompt,
|
|
308
310
|
include_collaboration_instructions=False,
|
|
311
|
+
extra_contextual_user_messages=extra_contextual_user_messages,
|
|
309
312
|
)
|
|
310
313
|
runtime_environment = create_agent_runtime_environment()
|
|
311
314
|
runtime_environment.request_user_input_manager.set_handler(None)
|
pycodex/context.py
CHANGED
|
@@ -149,6 +149,7 @@ class ContextManager:
|
|
|
149
149
|
include_permissions_instructions: 'bool' = True,
|
|
150
150
|
include_skills_instructions: 'bool' = True,
|
|
151
151
|
network_access: 'str' = "enabled",
|
|
152
|
+
extra_contextual_user_messages: 'typing.Iterable[str]' = (),
|
|
152
153
|
) -> 'None':
|
|
153
154
|
self.cwd = Path.cwd().resolve()
|
|
154
155
|
self._shell = get_shell_name()
|
|
@@ -166,6 +167,14 @@ class ContextManager:
|
|
|
166
167
|
self._include_permissions_instructions = include_permissions_instructions
|
|
167
168
|
self._include_skills_instructions = include_skills_instructions
|
|
168
169
|
self._network_access = network_access
|
|
170
|
+
self._extra_contextual_user_messages = tuple(
|
|
171
|
+
text
|
|
172
|
+
for text in (
|
|
173
|
+
_normalize_text(message)
|
|
174
|
+
for message in extra_contextual_user_messages
|
|
175
|
+
)
|
|
176
|
+
if text is not None
|
|
177
|
+
)
|
|
169
178
|
self._default_base_instructions = DEFAULT_BASE_INSTRUCTIONS_PATH.read_text(
|
|
170
179
|
encoding="utf-8"
|
|
171
180
|
)
|
|
@@ -183,6 +192,7 @@ class ContextManager:
|
|
|
183
192
|
include_permissions_instructions: 'bool' = True,
|
|
184
193
|
include_skills_instructions: 'bool' = True,
|
|
185
194
|
network_access: 'str' = "enabled",
|
|
195
|
+
extra_contextual_user_messages: 'typing.Iterable[str]' = (),
|
|
186
196
|
) -> 'ContextManager':
|
|
187
197
|
config = ContextConfig.from_codex_config(config_path, profile)
|
|
188
198
|
return cls(
|
|
@@ -193,6 +203,7 @@ class ContextManager:
|
|
|
193
203
|
include_permissions_instructions=include_permissions_instructions,
|
|
194
204
|
include_skills_instructions=include_skills_instructions,
|
|
195
205
|
network_access=network_access,
|
|
206
|
+
extra_contextual_user_messages=extra_contextual_user_messages,
|
|
196
207
|
)
|
|
197
208
|
|
|
198
209
|
@property
|
|
@@ -421,6 +432,7 @@ class ContextManager:
|
|
|
421
432
|
)
|
|
422
433
|
)
|
|
423
434
|
sections.append(self._serialize_environment_context())
|
|
435
|
+
sections.extend(self._extra_contextual_user_messages)
|
|
424
436
|
if not sections:
|
|
425
437
|
return []
|
|
426
438
|
return [
|
pycodex/model.py
CHANGED
|
@@ -298,13 +298,20 @@ class ResponsesModelClient:
|
|
|
298
298
|
prompt,
|
|
299
299
|
event_handler,
|
|
300
300
|
)
|
|
301
|
-
except ResponsesRetryableError as exc:
|
|
302
|
-
if
|
|
301
|
+
except (ResponsesRetryableError, ResponsesIncompleteError) as exc:
|
|
302
|
+
if (
|
|
303
|
+
isinstance(exc, ResponsesRetryableError)
|
|
304
|
+
and _is_context_length_error_message(str(exc))
|
|
305
|
+
):
|
|
303
306
|
raise ResponsesApiError(str(exc)) from exc
|
|
304
307
|
if retries >= max_retries:
|
|
305
308
|
raise
|
|
306
309
|
retries += 1
|
|
307
|
-
delay_seconds =
|
|
310
|
+
delay_seconds = (
|
|
311
|
+
exc.retry_delay_seconds
|
|
312
|
+
if isinstance(exc, ResponsesRetryableError)
|
|
313
|
+
else None
|
|
314
|
+
)
|
|
308
315
|
if delay_seconds is None:
|
|
309
316
|
delay_seconds = self._retry_delay_seconds(retries)
|
|
310
317
|
event_handler(
|
pycodex/runtime.py
CHANGED
|
@@ -165,6 +165,9 @@ class CliSubmissionQueue:
|
|
|
165
165
|
|
|
166
166
|
async def _next_submission(self) -> '_QueuedSubmission':
|
|
167
167
|
while True:
|
|
168
|
+
if self._agent._turn_running and not self._has_queue_active_turn():
|
|
169
|
+
await self._wait_for_agent_idle()
|
|
170
|
+
continue
|
|
168
171
|
async with self._queue_lock:
|
|
169
172
|
queued: 'typing.Union[_QueuedSubmission, None]' = None
|
|
170
173
|
if self._steer_queue:
|
|
@@ -197,9 +200,16 @@ class CliSubmissionQueue:
|
|
|
197
200
|
future.set_exception(exc)
|
|
198
201
|
|
|
199
202
|
def _has_active_turn(self) -> 'bool':
|
|
203
|
+
return self._has_queue_active_turn() or self._agent._turn_running
|
|
204
|
+
|
|
205
|
+
def _has_queue_active_turn(self) -> 'bool':
|
|
200
206
|
current_task = self._current_task
|
|
201
207
|
return current_task is not None and not current_task.done()
|
|
202
208
|
|
|
209
|
+
async def _wait_for_agent_idle(self) -> 'None':
|
|
210
|
+
while self._agent._turn_running:
|
|
211
|
+
await asyncio.sleep(0.01)
|
|
212
|
+
|
|
203
213
|
def _handle_agent_event(self, event: 'AgentEvent') -> 'None':
|
|
204
214
|
queued = self._current_submission
|
|
205
215
|
if queued is None:
|
|
@@ -21,11 +21,11 @@ from loguru import logger
|
|
|
21
21
|
|
|
22
22
|
from ..compat import is_ascii, stream_writer_is_closing
|
|
23
23
|
from ..protocol import JSONDict, JSONValue, ToolCall
|
|
24
|
+
from ..utils.truncation import DEFAULT_MAX_OUTPUT_TOKENS
|
|
24
25
|
from .base_tool import StructuredToolOutput, ToolContext, ToolRegistry
|
|
25
26
|
import typing
|
|
26
27
|
|
|
27
28
|
DEFAULT_WAIT_YIELD_TIME_MS = 10_000
|
|
28
|
-
DEFAULT_MAX_OUTPUT_TOKENS = 10_000
|
|
29
29
|
CHARS_PER_TOKEN = 4
|
|
30
30
|
EXEC_PRAGMA_PREFIX = "// @exec:"
|
|
31
31
|
WAIT_COMPLETION_GRACE_SECONDS = 0.02
|
|
@@ -21,15 +21,18 @@ from pathlib import Path
|
|
|
21
21
|
from loguru import logger
|
|
22
22
|
|
|
23
23
|
from ..compat import shlex_join, stream_writer_is_closing
|
|
24
|
+
from ..utils.truncation import (
|
|
25
|
+
DEFAULT_MAX_OUTPUT_TOKENS,
|
|
26
|
+
approx_token_count,
|
|
27
|
+
formatted_truncate_text,
|
|
28
|
+
)
|
|
24
29
|
import typing
|
|
25
30
|
|
|
26
31
|
DEFAULT_EXEC_YIELD_TIME_MS = 10_000
|
|
27
32
|
DEFAULT_WRITE_STDIN_YIELD_TIME_MS = 250
|
|
28
|
-
DEFAULT_MAX_OUTPUT_TOKENS = 10_000
|
|
29
33
|
DEFAULT_LOGIN = True
|
|
30
34
|
DEFAULT_TTY = False
|
|
31
35
|
DEFAULT_SESSION_ID_START = 1000
|
|
32
|
-
APPROX_BYTES_PER_TOKEN = 4
|
|
33
36
|
UNIFIED_EXEC_OUTPUT_MAX_BYTES = 1024 * 1024
|
|
34
37
|
UNIFIED_EXEC_OUTPUT_SCHEMA = {
|
|
35
38
|
"type": "object",
|
|
@@ -63,94 +66,6 @@ UNIFIED_EXEC_OUTPUT_SCHEMA = {
|
|
|
63
66
|
"additionalProperties": False,
|
|
64
67
|
}
|
|
65
68
|
|
|
66
|
-
|
|
67
|
-
def _approx_token_count(text: 'str') -> 'int':
|
|
68
|
-
if not text:
|
|
69
|
-
return 0
|
|
70
|
-
byte_length = len(text.encode("utf-8"))
|
|
71
|
-
return max(1, (byte_length + APPROX_BYTES_PER_TOKEN - 1) // APPROX_BYTES_PER_TOKEN)
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
def _approx_bytes_for_tokens(token_count: 'int') -> 'int':
|
|
75
|
-
return max(token_count, 0) * APPROX_BYTES_PER_TOKEN
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
def _approx_tokens_from_byte_count(byte_count: 'int') -> 'int':
|
|
79
|
-
if byte_count <= 0:
|
|
80
|
-
return 0
|
|
81
|
-
return (byte_count + APPROX_BYTES_PER_TOKEN - 1) // APPROX_BYTES_PER_TOKEN
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
def _split_budget(byte_budget: 'int') -> 'typing.Tuple[int, int]':
|
|
85
|
-
left_budget = byte_budget // 2
|
|
86
|
-
return left_budget, byte_budget - left_budget
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
def _split_string(
|
|
90
|
-
text: 'str',
|
|
91
|
-
beginning_bytes: 'int',
|
|
92
|
-
end_bytes: 'int',
|
|
93
|
-
) -> 'typing.Tuple[str, str]':
|
|
94
|
-
if not text:
|
|
95
|
-
return "", ""
|
|
96
|
-
|
|
97
|
-
total_bytes = len(text.encode("utf-8"))
|
|
98
|
-
tail_start_target = max(total_bytes - end_bytes, 0)
|
|
99
|
-
prefix_end = 0
|
|
100
|
-
suffix_start = len(text)
|
|
101
|
-
suffix_started = False
|
|
102
|
-
current_byte = 0
|
|
103
|
-
|
|
104
|
-
for index, char in enumerate(text):
|
|
105
|
-
char_bytes = len(char.encode("utf-8"))
|
|
106
|
-
char_start = current_byte
|
|
107
|
-
char_end = current_byte + char_bytes
|
|
108
|
-
if char_end <= beginning_bytes:
|
|
109
|
-
prefix_end = index + 1
|
|
110
|
-
current_byte = char_end
|
|
111
|
-
continue
|
|
112
|
-
if char_start >= tail_start_target:
|
|
113
|
-
if not suffix_started:
|
|
114
|
-
suffix_start = index
|
|
115
|
-
suffix_started = True
|
|
116
|
-
current_byte = char_end
|
|
117
|
-
continue
|
|
118
|
-
current_byte = char_end
|
|
119
|
-
|
|
120
|
-
if suffix_start < prefix_end:
|
|
121
|
-
suffix_start = prefix_end
|
|
122
|
-
|
|
123
|
-
return text[:prefix_end], text[suffix_start:]
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
def _truncate_text(text: 'str', max_tokens: 'int') -> 'str':
|
|
127
|
-
if not text:
|
|
128
|
-
return ""
|
|
129
|
-
|
|
130
|
-
max_bytes = _approx_bytes_for_tokens(max_tokens)
|
|
131
|
-
total_bytes = len(text.encode("utf-8"))
|
|
132
|
-
if total_bytes <= max_bytes:
|
|
133
|
-
return text
|
|
134
|
-
|
|
135
|
-
removed_tokens = _approx_tokens_from_byte_count(total_bytes - max_bytes)
|
|
136
|
-
marker = f"\u2026{removed_tokens} tokens truncated\u2026"
|
|
137
|
-
if max_bytes == 0:
|
|
138
|
-
return marker
|
|
139
|
-
|
|
140
|
-
left_budget, right_budget = _split_budget(max_bytes)
|
|
141
|
-
prefix, suffix = _split_string(text, left_budget, right_budget)
|
|
142
|
-
return f"{prefix}{marker}{suffix}"
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
def _formatted_truncate_text(text: 'str', max_tokens: 'int') -> 'str':
|
|
146
|
-
byte_budget = _approx_bytes_for_tokens(max_tokens)
|
|
147
|
-
if len(text.encode("utf-8")) <= byte_budget:
|
|
148
|
-
return text
|
|
149
|
-
|
|
150
|
-
total_lines = len(text.splitlines())
|
|
151
|
-
return f"Total output lines: {total_lines}\n\n{_truncate_text(text, max_tokens)}"
|
|
152
|
-
|
|
153
|
-
|
|
154
69
|
@dataclass
|
|
155
70
|
class _HeadTailBuffer:
|
|
156
71
|
max_bytes: 'int' = UNIFIED_EXEC_OUTPUT_MAX_BYTES
|
|
@@ -431,11 +346,11 @@ class UnifiedExecManager:
|
|
|
431
346
|
return [shell_path, "-lc" if login else "-c", cmd]
|
|
432
347
|
|
|
433
348
|
def _estimate_token_count(self, output: 'str') -> 'typing.Union[int, None]':
|
|
434
|
-
return
|
|
349
|
+
return approx_token_count(output)
|
|
435
350
|
|
|
436
351
|
def _truncate_output(self, output: 'str', max_output_tokens: 'typing.Union[int, None]') -> 'str':
|
|
437
352
|
token_budget = DEFAULT_MAX_OUTPUT_TOKENS if max_output_tokens is None else max_output_tokens
|
|
438
|
-
return
|
|
353
|
+
return formatted_truncate_text(output, max(token_budget, 0))
|
|
439
354
|
|
|
440
355
|
def _tty_echo(self, chars: 'str') -> 'bytes':
|
|
441
356
|
normalized = chars.replace("\n", "\r\n")
|
pycodex/utils/session_persist.py
CHANGED
|
@@ -198,7 +198,18 @@ def load_resumed_session(
|
|
|
198
198
|
session = sessions[resume_index - 1]
|
|
199
199
|
thread_id = session["thread_id"]
|
|
200
200
|
rollout_path = Path(session["rollout_path"])
|
|
201
|
-
|
|
201
|
+
return load_resumed_session_path(
|
|
202
|
+
rollout_path,
|
|
203
|
+
thread_name=_latest_thread_names_by_id(codex_home).get(thread_id),
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def load_resumed_session_path(
|
|
208
|
+
rollout_path: 'typing.Union[str, Path]',
|
|
209
|
+
thread_name: 'typing.Union[str, None]' = None,
|
|
210
|
+
) -> 'typing.Dict[str, object]':
|
|
211
|
+
rollout_path = Path(rollout_path)
|
|
212
|
+
thread_id = _thread_id_from_rollout_path(rollout_path) or ""
|
|
202
213
|
session_id = thread_id
|
|
203
214
|
history: 'typing.List[ConversationItem]' = []
|
|
204
215
|
saw_user_turn = False
|
|
@@ -245,6 +256,10 @@ def load_resumed_session(
|
|
|
245
256
|
if not history:
|
|
246
257
|
raise ValueError(f"No resumable history found in {rollout_path}")
|
|
247
258
|
|
|
259
|
+
history = _trim_incomplete_tool_call_tail(history)
|
|
260
|
+
if not history:
|
|
261
|
+
raise ValueError(f"No resumable history found in {rollout_path}")
|
|
262
|
+
|
|
248
263
|
turns = conversation_history_to_turns(history)
|
|
249
264
|
title = thread_name or (shorten_title(turns[0][0]) if turns else thread_id)
|
|
250
265
|
return {
|
|
@@ -277,6 +292,32 @@ def conversation_history_to_turns(
|
|
|
277
292
|
return tuple(turns)
|
|
278
293
|
|
|
279
294
|
|
|
295
|
+
def _trim_incomplete_tool_call_tail(
|
|
296
|
+
history: 'typing.List[ConversationItem]',
|
|
297
|
+
) -> 'typing.List[ConversationItem]':
|
|
298
|
+
pending_call_ids: 'typing.Set[str]' = set()
|
|
299
|
+
call_indexes: 'typing.Dict[str, int]' = {}
|
|
300
|
+
|
|
301
|
+
for index, item in enumerate(history):
|
|
302
|
+
if isinstance(item, ToolCall):
|
|
303
|
+
pending_call_ids.add(item.call_id)
|
|
304
|
+
call_indexes[item.call_id] = index
|
|
305
|
+
continue
|
|
306
|
+
if isinstance(item, ToolResult):
|
|
307
|
+
pending_call_ids.discard(item.call_id)
|
|
308
|
+
|
|
309
|
+
if not pending_call_ids:
|
|
310
|
+
return history
|
|
311
|
+
|
|
312
|
+
trim_start = min(call_indexes[call_id] for call_id in pending_call_ids)
|
|
313
|
+
while trim_start > 0 and isinstance(
|
|
314
|
+
history[trim_start - 1],
|
|
315
|
+
(AssistantMessage, ReasoningItem, ToolCall),
|
|
316
|
+
):
|
|
317
|
+
trim_start -= 1
|
|
318
|
+
return history[:trim_start]
|
|
319
|
+
|
|
320
|
+
|
|
280
321
|
def _latest_thread_names_by_id(codex_home: 'Path') -> 'typing.Dict[str, str]':
|
|
281
322
|
index_path = codex_home / SESSION_INDEX_FILENAME
|
|
282
323
|
if not index_path.exists():
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
"""Shared truncation helpers for model-visible tool output."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
|
|
5
|
+
from ..protocol import JSONValue, ToolResult
|
|
6
|
+
import typing
|
|
7
|
+
|
|
8
|
+
DEFAULT_MAX_OUTPUT_TOKENS = 10_000
|
|
9
|
+
TRUNCATION_SERIALIZATION_BUDGET_MULTIPLIER = 1.2
|
|
10
|
+
HISTORY_TOOL_OUTPUT_TOKENS = int(
|
|
11
|
+
math.ceil(DEFAULT_MAX_OUTPUT_TOKENS * TRUNCATION_SERIALIZATION_BUDGET_MULTIPLIER)
|
|
12
|
+
)
|
|
13
|
+
APPROX_BYTES_PER_TOKEN = 4
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def approx_token_count(text: 'str') -> 'int':
|
|
17
|
+
"""Estimate token count using the upstream Codex 4-bytes-per-token rule."""
|
|
18
|
+
if not text:
|
|
19
|
+
return 0
|
|
20
|
+
byte_length = len(text.encode("utf-8"))
|
|
21
|
+
return max(
|
|
22
|
+
1,
|
|
23
|
+
(byte_length + APPROX_BYTES_PER_TOKEN - 1) // APPROX_BYTES_PER_TOKEN,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def formatted_truncate_text(text: 'str', max_tokens: 'int') -> 'str':
|
|
28
|
+
"""Format a direct tool response with line count plus middle truncation."""
|
|
29
|
+
byte_budget = _approx_bytes_for_tokens(max_tokens)
|
|
30
|
+
if len(text.encode("utf-8")) <= byte_budget:
|
|
31
|
+
return text
|
|
32
|
+
|
|
33
|
+
total_lines = len(text.splitlines())
|
|
34
|
+
return f"Total output lines: {total_lines}\n\n{_truncate_text(text, max_tokens)}"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def truncate_tool_result_for_history(
|
|
38
|
+
result: 'ToolResult',
|
|
39
|
+
) -> 'ToolResult':
|
|
40
|
+
"""Truncate model-visible ToolResult content before storing it in history."""
|
|
41
|
+
if result.content_items is not None:
|
|
42
|
+
truncated_content_items = _truncate_content_items(
|
|
43
|
+
result.content_items,
|
|
44
|
+
HISTORY_TOOL_OUTPUT_TOKENS,
|
|
45
|
+
)
|
|
46
|
+
if truncated_content_items == result.content_items:
|
|
47
|
+
return result
|
|
48
|
+
return ToolResult(
|
|
49
|
+
call_id=result.call_id,
|
|
50
|
+
name=result.name,
|
|
51
|
+
output=result.output,
|
|
52
|
+
content_items=truncated_content_items,
|
|
53
|
+
success=result.success,
|
|
54
|
+
is_error=result.is_error,
|
|
55
|
+
tool_type=result.tool_type,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
output_text = _tool_output_text(result.output)
|
|
59
|
+
truncated_output = _truncate_text(output_text, HISTORY_TOOL_OUTPUT_TOKENS)
|
|
60
|
+
if truncated_output == output_text:
|
|
61
|
+
return result
|
|
62
|
+
return ToolResult(
|
|
63
|
+
call_id=result.call_id,
|
|
64
|
+
name=result.name,
|
|
65
|
+
output=truncated_output,
|
|
66
|
+
success=result.success,
|
|
67
|
+
is_error=result.is_error,
|
|
68
|
+
tool_type=result.tool_type,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def truncate_tool_results_for_history(
|
|
73
|
+
results: 'typing.Iterable[ToolResult]',
|
|
74
|
+
) -> 'typing.List[ToolResult]':
|
|
75
|
+
"""Apply history-layer truncation to a batch of completed tool results."""
|
|
76
|
+
return [
|
|
77
|
+
truncate_tool_result_for_history(result)
|
|
78
|
+
for result in results
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _tool_output_text(output: 'JSONValue') -> 'str':
|
|
83
|
+
if isinstance(output, str):
|
|
84
|
+
return output
|
|
85
|
+
|
|
86
|
+
import json
|
|
87
|
+
|
|
88
|
+
return json.dumps(
|
|
89
|
+
output,
|
|
90
|
+
ensure_ascii=False,
|
|
91
|
+
separators=(",", ":"),
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _truncate_content_items(
|
|
96
|
+
content_items: 'typing.Tuple[typing.Dict[str, typing.Any], ...]',
|
|
97
|
+
token_limit: 'int',
|
|
98
|
+
) -> 'typing.Tuple[typing.Dict[str, typing.Any], ...]':
|
|
99
|
+
output: 'typing.List[typing.Dict[str, typing.Any]]' = []
|
|
100
|
+
remaining_budget = token_limit
|
|
101
|
+
omitted_text_items = 0
|
|
102
|
+
|
|
103
|
+
for item in content_items:
|
|
104
|
+
if item.get("type") != "input_text":
|
|
105
|
+
output.append(dict(item))
|
|
106
|
+
continue
|
|
107
|
+
|
|
108
|
+
text = str(item.get("text", ""))
|
|
109
|
+
if remaining_budget <= 0:
|
|
110
|
+
omitted_text_items += 1
|
|
111
|
+
continue
|
|
112
|
+
|
|
113
|
+
cost = approx_token_count(text)
|
|
114
|
+
if cost <= remaining_budget:
|
|
115
|
+
output.append(dict(item))
|
|
116
|
+
remaining_budget -= cost
|
|
117
|
+
continue
|
|
118
|
+
|
|
119
|
+
truncated_text = _truncate_text(text, remaining_budget)
|
|
120
|
+
if truncated_text:
|
|
121
|
+
next_item = dict(item)
|
|
122
|
+
next_item["text"] = truncated_text
|
|
123
|
+
output.append(next_item)
|
|
124
|
+
else:
|
|
125
|
+
omitted_text_items += 1
|
|
126
|
+
remaining_budget = 0
|
|
127
|
+
|
|
128
|
+
if omitted_text_items > 0:
|
|
129
|
+
output.append(
|
|
130
|
+
{
|
|
131
|
+
"type": "input_text",
|
|
132
|
+
"text": f"[omitted {omitted_text_items} text items ...]",
|
|
133
|
+
}
|
|
134
|
+
)
|
|
135
|
+
return tuple(output)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _approx_tokens_from_byte_count(byte_count: 'int') -> 'int':
|
|
139
|
+
if byte_count <= 0:
|
|
140
|
+
return 0
|
|
141
|
+
return (byte_count + APPROX_BYTES_PER_TOKEN - 1) // APPROX_BYTES_PER_TOKEN
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _approx_bytes_for_tokens(token_count: 'int') -> 'int':
|
|
145
|
+
return max(token_count, 0) * APPROX_BYTES_PER_TOKEN
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _split_budget(byte_budget: 'int') -> 'typing.Tuple[int, int]':
|
|
149
|
+
left_budget = byte_budget // 2
|
|
150
|
+
return left_budget, byte_budget - left_budget
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _split_string(
|
|
154
|
+
text: 'str',
|
|
155
|
+
beginning_bytes: 'int',
|
|
156
|
+
end_bytes: 'int',
|
|
157
|
+
) -> 'typing.Tuple[str, str]':
|
|
158
|
+
if not text:
|
|
159
|
+
return "", ""
|
|
160
|
+
|
|
161
|
+
total_bytes = len(text.encode("utf-8"))
|
|
162
|
+
tail_start_target = max(total_bytes - end_bytes, 0)
|
|
163
|
+
prefix_end = 0
|
|
164
|
+
suffix_start = len(text)
|
|
165
|
+
suffix_started = False
|
|
166
|
+
current_byte = 0
|
|
167
|
+
|
|
168
|
+
for index, char in enumerate(text):
|
|
169
|
+
char_bytes = len(char.encode("utf-8"))
|
|
170
|
+
char_start = current_byte
|
|
171
|
+
char_end = current_byte + char_bytes
|
|
172
|
+
if char_end <= beginning_bytes:
|
|
173
|
+
prefix_end = index + 1
|
|
174
|
+
current_byte = char_end
|
|
175
|
+
continue
|
|
176
|
+
if char_start >= tail_start_target:
|
|
177
|
+
if not suffix_started:
|
|
178
|
+
suffix_start = index
|
|
179
|
+
suffix_started = True
|
|
180
|
+
current_byte = char_end
|
|
181
|
+
continue
|
|
182
|
+
current_byte = char_end
|
|
183
|
+
|
|
184
|
+
if suffix_start < prefix_end:
|
|
185
|
+
suffix_start = prefix_end
|
|
186
|
+
|
|
187
|
+
return text[:prefix_end], text[suffix_start:]
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _truncate_text(text: 'str', max_tokens: 'int') -> 'str':
|
|
191
|
+
if not text:
|
|
192
|
+
return ""
|
|
193
|
+
|
|
194
|
+
max_bytes = _approx_bytes_for_tokens(max_tokens)
|
|
195
|
+
total_bytes = len(text.encode("utf-8"))
|
|
196
|
+
if total_bytes <= max_bytes:
|
|
197
|
+
return text
|
|
198
|
+
|
|
199
|
+
removed_tokens = _approx_tokens_from_byte_count(total_bytes - max_bytes)
|
|
200
|
+
marker = f"\u2026{removed_tokens} tokens truncated\u2026"
|
|
201
|
+
if max_bytes == 0:
|
|
202
|
+
return marker
|
|
203
|
+
|
|
204
|
+
left_budget, right_budget = _split_budget(max_bytes)
|
|
205
|
+
prefix, suffix = _split_string(text, left_budget, right_budget)
|
|
206
|
+
return f"{prefix}{marker}{suffix}"
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
pycodex/__init__.py,sha256=I1P7OHkabjHP7g32A31o-L9NMd3lKYJ2xhV8AHLPYXA,3196
|
|
2
|
-
pycodex/agent.py,sha256=
|
|
3
|
-
pycodex/cli.py,sha256=
|
|
2
|
+
pycodex/agent.py,sha256=HKzWVFRag_Le8LVR1qqfdfVzk23bRsXkykZV8Zq2hLE,21659
|
|
3
|
+
pycodex/cli.py,sha256=6URw4uSV-GPevu8NiqoraWIa180QmSUFOHScCZHO2Mc,20616
|
|
4
4
|
pycodex/collaboration.py,sha256=yQ6pBD-R3ZWR4_FAYQFoS7KF0m4LLD42otXIbPqw2ys,641
|
|
5
5
|
pycodex/compat.py,sha256=l35JE0vGAOCn9NWbWbqwGURGk83HXddXQ5wJIfcG41o,3254
|
|
6
|
-
pycodex/context.py,sha256=
|
|
6
|
+
pycodex/context.py,sha256=Hwh3vU-qSvaleUDdOWuWiGyNmbULVAZmboxiu9iiHt8,26549
|
|
7
7
|
pycodex/doctor.py,sha256=De3M4hRBJq8ZeqsUJgHz0vitqrH18YugrEnz7oHhTdQ,10572
|
|
8
8
|
pycodex/feishu_card.py,sha256=De6pM--3MfhgGo-WcWfhm-fop5UtzjwKrZ4gI2Lls3w,26198
|
|
9
9
|
pycodex/feishu_link.py,sha256=XGV93CIorcc03pNgBnXay9SkivnDL4i2GkLR5NjBjq0,15588
|
|
10
10
|
pycodex/interactive_session.py,sha256=VmQ4xLk5b9QrI4LvgrdyhXTvIk0K65loKxh2XbfV2FA,15617
|
|
11
|
-
pycodex/model.py,sha256=
|
|
11
|
+
pycodex/model.py,sha256=O5Gf35qCMAGrqvr_5d1ZQNk3vHmNRD5Uv-8k7-zbpXg,36679
|
|
12
12
|
pycodex/portable.py,sha256=gxl2E2h5uZJbasMEPPs-nyALFPIvX79T2ZYsu6vXZrg,15656
|
|
13
13
|
pycodex/portable_server.py,sha256=6I3pQkWj3e_SFlDXY2mGdCPns1w_3PSxByBV9wv5epI,7331
|
|
14
14
|
pycodex/protocol.py,sha256=4qiEcBQc3d3RZqsIKBjwORsHsQa78cqwvNljtnIuNbM,10795
|
|
15
|
-
pycodex/runtime.py,sha256=
|
|
15
|
+
pycodex/runtime.py,sha256=dXxLkcsjufBZ8F7o4QKFXF--mkgq3cpxhyUbPPaBYy4,8684
|
|
16
16
|
pycodex/runtime_services.py,sha256=6PQMI4MM8F9imijaVKBIz_0ADtoKDcdYs6vQ-c4frtQ,13931
|
|
17
17
|
pycodex/prompts/collaboration_default.md,sha256=MBTmPuMubeWfZgIeFVj49wwnwD4n_o3fVYAbgWKwu6Q,955
|
|
18
18
|
pycodex/prompts/collaboration_plan.md,sha256=IzjQAA5oHJz-3FmJdOjsJ4LHq6LW1tlEYMoy09n0HKk,8777
|
|
@@ -31,7 +31,7 @@ pycodex/tools/agent_tool_schemas.py,sha256=8cL9ml4So0H6IIeK0ZzjAo9bk3VeprDOSYPpI
|
|
|
31
31
|
pycodex/tools/apply_patch_tool.py,sha256=wLHk5sl_UkR7iv0yqMJBMLHbGkZlMmCuX9a0yTevzps,13802
|
|
32
32
|
pycodex/tools/base_tool.py,sha256=0aQ69ygvSYKtDq1wHXr_j7EF3NtKIfmj0A9AbB-pD4Q,5431
|
|
33
33
|
pycodex/tools/close_agent_tool.py,sha256=79qr4ljPTvjnpH-Oe83yFTUGZdgNNnGoLZbpdqp4eOY,2031
|
|
34
|
-
pycodex/tools/code_mode_manager.py,sha256=
|
|
34
|
+
pycodex/tools/code_mode_manager.py,sha256=T8RHM8tyJvHWwS6AD2g9hsaOmvApMmlWuSCRfmDVMDE,19053
|
|
35
35
|
pycodex/tools/exec_command_tool.py,sha256=KIt-pni7b2I41EMXGa1NW9YWJ1AoiWlPW_Ljlu7kLpk,4082
|
|
36
36
|
pycodex/tools/exec_runtime.js,sha256=DR1uocKailTqNWAcJNFJuQgFFMSUzTpT_uQsRaneg2s,3643
|
|
37
37
|
pycodex/tools/exec_tool.py,sha256=6uACLanhMxa5_5dZGO8CPztnM20O2ACANbaf9AUBS8M,2278
|
|
@@ -46,7 +46,7 @@ pycodex/tools/send_input_tool.py,sha256=vRg-f7LI2Sr01j1LeCPKeXto_ROQZ7bcUC5GIjx9
|
|
|
46
46
|
pycodex/tools/shell_command_tool.py,sha256=wUw4lw8VLGIQ-7BIgyEsI0oqjj-Yr-MZn-_VDrnimAw,4202
|
|
47
47
|
pycodex/tools/shell_tool.py,sha256=1m-Tcbn3His4ggyK5ec8Mkg6ihopqTvBd9F6fJlm6m4,4054
|
|
48
48
|
pycodex/tools/spawn_agent_tool.py,sha256=7Me4Frjz7_fTpjMMFCvNmDHPOK2G1qTfXUr2Ugs61bU,6194
|
|
49
|
-
pycodex/tools/unified_exec_manager.py,sha256=
|
|
49
|
+
pycodex/tools/unified_exec_manager.py,sha256=cKW_YNjSweVlGt8xNdeK_7gVwsGXxumCqjr-OlW5Xe0,12868
|
|
50
50
|
pycodex/tools/update_plan_tool.py,sha256=UsChtCBqI1RnVnPQbByPmD2LMZXMoCwfe6NpGqP8qu0,3174
|
|
51
51
|
pycodex/tools/view_image_tool.py,sha256=2Xu5Vx7djVpz7-IV-LKrDKJjvVJc9AND-MFXp3CWplg,3892
|
|
52
52
|
pycodex/tools/wait_agent_tool.py,sha256=0Uj9-IrXe2dSvOtOMq-RAc0XzaidwFH5q7Mri3BXWyM,3135
|
|
@@ -60,8 +60,9 @@ pycodex/utils/debug.py,sha256=JeEB5JfzYfbdG0fXlrWFmXyR1ts86fKsI_97IqgF6R0,296
|
|
|
60
60
|
pycodex/utils/dotenv.py,sha256=rGKmurHjm7GdP4giyjHBPpSPv2Oi45qBqDB6HG3CnfA,1866
|
|
61
61
|
pycodex/utils/get_env.py,sha256=5fNhcNhujOakWV6AS66rGW3jEA68WGpuE4YVXJZFE6U,7427
|
|
62
62
|
pycodex/utils/random_ids.py,sha256=zBphjVGc7OXk9ZNExAbxRi_bk7ipyLG491qTv7hi8jM,380
|
|
63
|
-
pycodex/utils/session_persist.py,sha256=
|
|
63
|
+
pycodex/utils/session_persist.py,sha256=LnyKbQQ2SyONXhZ_XF_5pUUz_ZqhWsA7dOL5pCcKsCg,18491
|
|
64
64
|
pycodex/utils/toolcall_visualize.py,sha256=zIqmdsOfyYaLy_P4jpKnRxDsfTgYLRBx55R8m1P_lBE,24708
|
|
65
|
+
pycodex/utils/truncation.py,sha256=B_RvfXC2-M1oKz--eQIqDLqMD0g7_J-MSQd3WD6Rh08,6110
|
|
65
66
|
pycodex/utils/visualize.py,sha256=oTl1vqYW3nFBrdLcDyzwYfkNaPEW4OhVahJMHFSVkEg,20591
|
|
66
67
|
responses_server/__init__.py,sha256=3yPv_zeGT7P11tTnmj5kXktISLNsNW-02MUnnbiZcb0,394
|
|
67
68
|
responses_server/__main__.py,sha256=9SRp-Yw7ShGxc6DhSIXcDLKgGEdAVm3oBZ59rBOPjT0,62
|
|
@@ -76,12 +77,12 @@ responses_server/trajectory_dump.py,sha256=XCwYaZZmlAxSsSXOfhk3zRvyfDpOHX5R8Kzsp
|
|
|
76
77
|
responses_server/tools/__init__.py,sha256=ivsBSEy0SBUhY-Uea5v1XMLXShkwHdCVl0id-1FwdZg,150
|
|
77
78
|
responses_server/tools/custom_adapter.py,sha256=LxO7ldydvR-GWachDz8GKC0Q8KGGFoFPbZxM0QvxuZ0,8350
|
|
78
79
|
responses_server/tools/web_search.py,sha256=pm4ZUiHUfxc0bGY1kEvt-BCzDrZIyP24xzPUcga2ul0,8908
|
|
79
|
-
workspace_server/__init__.py,sha256=
|
|
80
|
+
workspace_server/__init__.py,sha256=XbSU6aAeYBMsBTVi9Ug6ZW9sxcwefqPS7T_7wpt_VMo,462
|
|
80
81
|
workspace_server/__main__.py,sha256=9SRp-Yw7ShGxc6DhSIXcDLKgGEdAVm3oBZ59rBOPjT0,62
|
|
81
|
-
workspace_server/app.py,sha256=
|
|
82
|
-
workspace_server/workspace.html,sha256=
|
|
83
|
-
python_codex-0.
|
|
84
|
-
python_codex-0.
|
|
85
|
-
python_codex-0.
|
|
86
|
-
python_codex-0.
|
|
87
|
-
python_codex-0.
|
|
82
|
+
workspace_server/app.py,sha256=aDfUqIwyMsndavSTP47rfcBKIwgo37F8U_lkCgUfUc0,49260
|
|
83
|
+
workspace_server/workspace.html,sha256=7d1eTxQ_4Hl-8YbMFHMTmRPpMRXE-a0KSJn1224JpXw,25563
|
|
84
|
+
python_codex-0.2.0.dist-info/METADATA,sha256=oCzNpTEbt1eODGxsJeHJE-gjTf53He7UDvcs-bovqAc,16812
|
|
85
|
+
python_codex-0.2.0.dist-info/WHEEL,sha256=KGYbc1zXlYddvwxnNty23BeaKzh7YuoSIvIMO4jEhvw,87
|
|
86
|
+
python_codex-0.2.0.dist-info/entry_points.txt,sha256=vkV2UWCtEKvQNMJuPNjt8HyBKiwp83JyqBatrBNGDp8,80
|
|
87
|
+
python_codex-0.2.0.dist-info/licenses/LICENSE,sha256=0X8ifk312hYAORM4hlzg8wVSEXYKNmiPgWlB1YIy2Nw,10926
|
|
88
|
+
python_codex-0.2.0.dist-info/RECORD,,
|