codex-agent-framework 0.1.56__tar.gz → 0.1.57__tar.gz

{codex_agent_framework-0.1.56 → codex_agent_framework-0.1.57}/CHANGELOG.md

@@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file.
 
 This project loosely follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and uses semantic versioning where practical.
 
+## [0.1.57] - 2026-05-17
+### Fixed
+- Fix stream helper fan-out edge cases so `Tee`, `Splitter`, and `MappingStreamSplitter` readers keep their own queues and reusable mapping splitters do not redirect old readers to new streams.
+- Make `utils.Logger` create its parent directory before truncating the log file so first use works on fresh paths.
+
+### Changed
+- Simplify small utility helpers for line-preserving truncation, message packing, and fenced stream pattern construction while preserving their existing behavior.
+
+### Tests
+- Add focused regression coverage for the stream helpers, logger initialization, and utility formatting/truncation helpers; validate with the full offline CI suite (580 passed).
+
 ## [0.1.56] - 2026-05-17
 ### Changed
 - Replace the TUI chat surface with a widget-based timeline inspired by Textual chat patterns so live assistant output updates an active message widget instead of mutating a monolithic transcript buffer.

{codex_agent_framework-0.1.56/codex_agent_framework.egg-info → codex_agent_framework-0.1.57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codex-agent-framework
-Version: 0.1.56
+Version: 0.1.57
 Summary: A lightweight event-driven Codex agent runtime.
 Author: Baptiste
 License-Expression: MIT

{codex_agent_framework-0.1.56 → codex_agent_framework-0.1.57}/codex_agent/stream_utils.py

@@ -53,17 +53,16 @@ class Tee:
         for k in self.queues:
             self.queues[k].put(END)
         
-    def tee(self,stream, n=2):
-        self.queues={k:Queue() for k in range(n)}
-        self.thread=Thread(target=self.target, args=(stream,),kwargs=dict(n=n),daemon=True)
+    def tee(self, stream, n=2):
+        self.queues = {k: Queue() for k in range(n)}
+        self.thread = Thread(target=self.target, args=(stream,), daemon=True)
         self.thread.start()
-        readers={}
-        for k in self.queues:
-            def reader():
-                while (processed_chunk:=self.queues[k].get()) is not END:
-                    yield processed_chunk
-            readers[k]=reader()
-        return tuple(readers.values())
+
+        def reader(queue):
+            while (processed_chunk := queue.get()) is not END:
+                yield processed_chunk
+
+        return tuple(reader(queue) for queue in self.queues.values())
     
 class Splitter:
 
@@ -86,17 +85,16 @@ class Splitter:
         self.queue_false.put(END)
         
     def split(self, stream):
-        self.queue_true=Queue()
-        self.queue_false=Queue()
-        self.thread=Thread(target=self.target, args=(stream,),daemon=True)
+        self.queue_true = Queue()
+        self.queue_false = Queue()
+        self.thread = Thread(target=self.target, args=(stream,), daemon=True)
         self.thread.start()
-        def reader_true():
-            while (chunk:=self.queue_true.get()) is not END:
-                yield chunk
-        def reader_false():
-            while (chunk:=self.queue_false.get()) is not END:
+
+        def reader(queue):
+            while (chunk := queue.get()) is not END:
                 yield chunk
-        return reader_true(),reader_false()
+
+        return reader(self.queue_true), reader(self.queue_false)
   
 class Streamer:
 
@@ -159,61 +157,27 @@ class StreamLogger(Streamer):
             with open(self.file,'a', encoding='utf-8') as f:
                 f.write(output)
         
-def fenced(start: Union[str,regex.Pattern,re.Pattern], end: Union[str,regex.Pattern,re.Pattern], flags=0) -> regex.Pattern:
-    r"""
-    Generates a regex pattern that matches text enclosed between a start and an end delimiter,
-    The inner part is captured as the 'content' group (default).
-    There can be other captured groups defined in the start and end patterns.
-    The end delimiter can include backreferences to captured groups from the opening fence into placeholders in the closing fence.
-
-    The closing fence pattern may include placeholders in the form of "#1#" for the first captured group,
-    "#2#" for the second, or "#name#" for a named capturing group; these will be replaced by the appropriate
-    backreference. This mechanism is completely generic and does not enforce any specific delimiter shapes.
-
-    Example:
-        # Using a numeric placeholder:
-        pattern = fenced(r"START (\w+)", r"END #1#", flags=regex.DOTALL)
-        text = "START content END content"
-        match = pattern.match(text)
-        if match:
-            print(match.group())
-
-        # Using a named capturing group:
-        pattern = fenced(r"BEGIN (?P<tag>\w+)", r"FIN #tag#", flags=regex.DOTALL)
-        text = "BEGIN example FIN example"
-        match = pattern.match(text)
-        if match:
-            print(match.group())
-
-    :param start: Opening delimiter (str or regex.Pattern). May contain capturing groups.
-    :param end: Closing delimiter (str or regex.Pattern) which may include placeholders (e.g., "#1#", "#name#").
-    :param flags: Regex flags to modify behavior (e.g., regex.DOTALL, regex.MULTILINE).
-    :return: A compiled regex pattern.
+def fenced(start: Union[str, regex.Pattern, re.Pattern], end: Union[str, regex.Pattern, re.Pattern], flags=0) -> regex.Pattern:
+    r"""Compile a fenced-content pattern with the inner text captured as ``content``.
+
+    ``end`` may reference capture groups from ``start`` with ``#1#`` or
+    ``#name#`` placeholders, which are converted to regex backreferences.
     """
-    def prepare_pattern(pattern):
-        if isinstance(pattern, (regex.Pattern,re.Pattern)):
-            return pattern.pattern
+    def pattern_source(pattern):
         if isinstance(pattern, str):
             return pattern
-        raise ValueError("`start` and `end` must be either a string or a regex Pattern object")
-    
-    start_regex = prepare_pattern(start)   
-    end_regex   = prepare_pattern(end)
-    
-    # Replace placeholders in end pattern with regex backrefs
-    def replace_placeholder(match):
+        if isinstance(pattern, (regex.Pattern, re.Pattern)):
+            return pattern.pattern
+        raise ValueError("`start` and `end` must be strings or regex Pattern objects")
+
+    def placeholder_to_backref(match):
         key = match.group(1)
-        if key.isdigit():
-            return rf"\{key}"
-        else:
-            return rf"(?P={key})"
-    end_regex = regex.sub(r"#(\w+)#", replace_placeholder, end_regex)
-    
-    # Named content group
-    inner = rf'(?P<content>(?:(?!{end_regex})[\s\S])*)'
-    
-    full_pattern = rf'{start_regex}{inner}(?:{end_regex}|$)'
-    return regex.compile(full_pattern, flags)
+        return rf"\{key}" if key.isdigit() else rf"(?P={key})"
+
+    start_regex = pattern_source(start)
+    end_regex = regex.sub(r"#(\w+)#", placeholder_to_backref, pattern_source(end))
+    inner = rf"(?P<content>(?:(?!{end_regex})[\s\S])*)"
+    return regex.compile(rf"{start_regex}{inner}(?:{end_regex}|$)", flags)
 
 def is_compiled_pattern(pattern):
     if isinstance(pattern, (re.Pattern,regex.Pattern)):
@@ -528,16 +492,19 @@ class MappingStreamSplitter:
     def maybe_init_readers(self, data):
         for key in data.keys():
             if key not in self.readers:
-                self.queues[key]=Queue()
-                def reader(key=key):
-                    while (value:=self.queues[key].get()) is not END:
+                queue = self.queues[key] = Queue()
+
+                def reader(queue=queue):
+                    while (value := queue.get()) is not END:
                         if isinstance(value, StreamError):
                             raise value.error
                         yield value
-                self.readers[key]=reader()
+
+                self.readers[key] = reader()
 
     def process(self,stream):
-        self.queues=dict()
+        self.readers = {}
+        self.queues = {}
         self.done=False
         if self.defaults:
             self.maybe_init_readers(self.defaults)

{codex_agent_framework-0.1.56 → codex_agent_framework-0.1.57}/codex_agent/utils.py

@@ -230,69 +230,39 @@ def type_hierarchy(cls, root):
 
 
 def truncate(string, max_tokens=2000, start_line=1, model=None):
-    """
-    Truncate a string to a maximum number of tokens using line-based chunking.
-
-    Args:
-        string: The string to truncate
-        max_tokens: Maximum number of tokens to keep
-        start_line: The line number of the first line in the string (1-indexed).
-                   Used for skipping content and tracking line numbers in truncation messages.
-
-    Returns:
-        Truncated string with indication of removed content
-    """
-    # line-based pagination
+    """Return a line-preserving prefix that fits within ``max_tokens``."""
     lines = string.splitlines(keepends=True)
-    if start_line > 1:
-        if start_line > len(lines):
-            return ""
-        string = ''.join(lines[start_line - 1:])
-        lines = string.splitlines(keepends=True)
-
-    # Quick check - if already under limit after pagination, return as-is
-    tokens = tokenize(string, model=model)
-    if len(tokens) <= max_tokens:
-        return string
+    if start_line > len(lines):
+        return "" if start_line > 1 else string
 
-    # Calculate the actual last line number after skipping
-    remaining_lines = len(lines)
-
-    # Assemble lines up to max_tokens
-    result_lines = []
-    current_tokens = 0
+    lines = lines[max(0, start_line - 1):]
+    string = "".join(lines)
+    if len(tokenize(string, model=model)) <= max_tokens:
+        return string
 
+    kept_lines = []
+    kept_tokens = 0
     for line in lines:
         line_tokens = token_count(line, model=model)
-        if current_tokens + line_tokens <= max_tokens:
-            result_lines.append(line)
-            current_tokens += line_tokens
-        else:
+        if kept_tokens + line_tokens > max_tokens:
             break
-
-    if result_lines:
-        result = ''.join(result_lines)
-        num_kept = len(result_lines)
-
-        # Only show truncation message if we didn't include all lines
-        if num_kept < remaining_lines:
-            first_truncated = start_line + num_kept
-            last_line = start_line + remaining_lines - 1
-
-            if not result.endswith('\n'):
-                result += '\n'
-            result += f"\n...\n\n[Lines {first_truncated}-{last_line} truncated]\n"
-
-        return result
-    else:
-        return f"[Lines {start_line}-{start_line + remaining_lines - 1} truncated - first line exceeds token limit]\n"
+        kept_lines.append(line)
+        kept_tokens += line_tokens
+
+    last_line = start_line + len(lines) - 1
+    if not kept_lines:
+        return f"[Lines {start_line}-{last_line} truncated - first line exceeds token limit]\n"
+
+    result = "".join(kept_lines)
+    first_truncated = start_line + len(kept_lines)
+    if first_truncated <= last_line:
+        if not result.endswith("\n"):
+            result += "\n"
+        result += f"\n...\n\n[Lines {first_truncated}-{last_line} truncated]\n"
+    return result
 
 def pack_msgs(messages):
-    text = ''
-    for message in messages:
-        text += message.name + ':\n'
-        text += message.content.strip() + '\n\n'
-    return text
+    return "".join(f"{message.name}:\n{message.content.strip()}\n\n" for message in messages)
 
 def msg_token_count(msg, model=None):
     return msg.api_token_count(model=model)
@@ -468,8 +438,11 @@ def upload_file_to_folder(source, folder, name=None):
 class Logger:
 
     def __init__(self,file=None):
-        self.file=file or root_join('logs',"log.txt")
-        open(self.file,'w', encoding='utf-8').close()
+        self.file = os.fspath(file or root_join('logs', "log.txt"))
+        folder = os.path.dirname(self.file)
+        if folder:
+            os.makedirs(folder, exist_ok=True)
+        open(self.file, 'w', encoding='utf-8').close()
 
     def log(self,message):
         with open(self.file,'a',encoding="utf-8") as f:

codex_agent_framework-0.1.57/codex_agent/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.57"

{codex_agent_framework-0.1.56 → codex_agent_framework-0.1.57/codex_agent_framework.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codex-agent-framework
-Version: 0.1.56
+Version: 0.1.57
 Summary: A lightweight event-driven Codex agent runtime.
 Author: Baptiste
 License-Expression: MIT

{codex_agent_framework-0.1.56 → codex_agent_framework-0.1.57}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "codex-agent-framework"
-version = "0.1.56"
+version = "0.1.57"
 description = "A lightweight event-driven Codex agent runtime."
 readme = "README.md"
 requires-python = ">=3.11"

{codex_agent_framework-0.1.56 → codex_agent_framework-0.1.57}/tests/test_utils.py

@@ -6,7 +6,7 @@ from PIL import Image as PILImage
 from codex_agent.message import AssistantMessage, CompactionMessage, DeveloperMessage, InterruptMessage, ServerToolResponseMessage, ToolResultMessage, UserMessage
 from codex_agent import utils
 from codex_agent.image import ImageMessage, image_input_token_count
-from codex_agent.stream_utils import MappingStreamProcessor
+from codex_agent.stream_utils import MappingStreamProcessor, MappingStreamSplitter, Splitter, Tee, fenced
 from codex_agent.utils import split_history_turns
 from codex_agent.utils import project_python_env
 
@@ -29,6 +29,28 @@ def test_split_history_turns_groups_complete_agent_loop():
     assert remainder == []
 
 
+def test_pack_msgs_formats_named_messages():
+    class Message:
+        def __init__(self, name, content):
+            self.name = name
+            self.content = content
+
+    messages = [
+        Message("user", " hello "),
+        Message("assistant", "done"),
+    ]
+
+    assert utils.pack_msgs(messages) == "user:\nhello\n\nassistant:\ndone\n\n"
+
+
+def test_logger_creates_parent_directory_and_truncates_file(tmp_path):
+    log_file = tmp_path / "logs" / "log.txt"
+
+    utils.Logger(file=log_file)
+
+    assert log_file.read_text(encoding="utf-8") == ""
+
+
 def test_project_python_env_prefers_running_interpreter():
     env = project_python_env(env={"PATH": "/usr/bin:/bin"}, executable="/opt/project-python/bin/python")
 
@@ -37,6 +59,30 @@ def test_project_python_env_prefers_running_interpreter():
     assert env["PYTHON_EXECUTABLE"] == "/opt/project-python/bin/python"
 
 
+def test_tee_fans_out_stream_to_independent_readers():
+    first, second = Tee().tee(iter(["a", "b"]), n=2)
+
+    assert list(first) == ["a", "b"]
+    assert list(second) == ["a", "b"]
+
+
+def test_splitter_routes_stream_to_independent_readers():
+    even, odd = Splitter(lambda number: number % 2 == 0).split(iter([1, 2, 3, 4]))
+
+    assert list(even) == [2, 4]
+    assert list(odd) == [1, 3]
+
+
+def test_mapping_stream_splitter_can_be_reused():
+    splitter = MappingStreamSplitter(threaded=False)
+
+    first = splitter.split(iter([{"text": "a"}]))
+    second = splitter.split(iter([{"text": "b"}]))
+
+    assert list(first["text"]) == ["a"]
+    assert list(second["text"]) == ["b"]
+
+
 def test_mapping_stream_processor_is_direct_passthrough_without_processors():
     source = iter([{"content": "a"}, {"content": "b"}])
     processor = MappingStreamProcessor(processors={"content": []}, threaded=True)
@@ -47,6 +93,24 @@ def test_mapping_stream_processor_is_direct_passthrough_without_processors():
     assert list(result) == [{"content": "a"}, {"content": "b"}]
 
 
+def test_fenced_reuses_numbered_capture_in_closing_pattern():
+    pattern = fenced(r"BEGIN (\w+)\n", r"\nEND #1#")
+
+    match = pattern.fullmatch("BEGIN tag\nhello\nEND tag")
+
+    assert match is not None
+    assert match.group("content") == "hello"
+
+
+def test_fenced_reuses_named_capture_in_closing_pattern():
+    pattern = fenced(r"<(?P<tag>\w+)>", r"</#tag#>")
+
+    match = pattern.fullmatch("<note>hello</note>")
+
+    assert match is not None
+    assert match.group("content") == "hello"
+
+
 def test_read_document_content_adds_line_numbers_by_default(tmp_path):
     file = tmp_path / "sample.txt"
     file.write_text("alpha\nbeta\ngamma", encoding="utf-8")
@@ -390,6 +454,28 @@ def test_image_message_token_count_ignores_base64_text_volume():
     assert message.estimated_token_count["gpt-5"]["count"] == count
 
 
+def test_truncate_returns_requested_page_when_under_token_limit(monkeypatch):
+    monkeypatch.setattr(utils, "tokenize", lambda text, model=None: list(text))
+    monkeypatch.setattr(utils, "token_count", lambda text, model=None: len(text))
+
+    assert utils.truncate("a\nb\nc\n", max_tokens=10, start_line=2) == "b\nc\n"
+
+
+def test_truncate_appends_line_range_notice(monkeypatch):
+    monkeypatch.setattr(utils, "tokenize", lambda text, model=None: list(text))
+    monkeypatch.setattr(utils, "token_count", lambda text, model=None: len(text))
+
+    assert utils.truncate("aa\nbb\ncc\n", max_tokens=3) == "aa\n\n...\n\n[Lines 2-3 truncated]\n"
+
+
+def test_truncate_reports_first_line_when_it_exceeds_limit(monkeypatch):
+    monkeypatch.setattr(utils, "tokenize", lambda text, model=None: list(text))
+    monkeypatch.setattr(utils, "token_count", lambda text, model=None: len(text))
+
+    assert utils.truncate("aaaa\nbb\n", max_tokens=2, start_line=3) == ""
+    assert utils.truncate("aaaa\nbb\n", max_tokens=2) == "[Lines 1-2 truncated - first line exceeds token limit]\n"
+
+
 def test_load_runtime_env_reads_runtime_dotenv(isolated_runtime_dir, monkeypatch):
     isolated_runtime_dir.mkdir(parents=True, exist_ok=True)
     env_file = isolated_runtime_dir / ".env"

codex_agent_framework-0.1.56/codex_agent/version.py

@@ -1 +0,0 @@
-__version__ = "0.1.56"