echotools 2.3.47__tar.gz → 2.3.49__tar.gz

{echotools-2.3.47 → echotools-2.3.49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: echotools
-Version: 2.3.47
+Version: 2.3.49
 Summary: 通用基础设施 SDK：配置、日志、事件、调度、插件、协议、调用链
 Home-page: https://github.com/nichengfuben/echotools
 Author: nichengfuben

{echotools-2.3.47 → echotools-2.3.49}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "echotools"
-version = "2.3.47"
+version = "2.3.49"
 description = "通用基础设施 SDK：配置、日志、事件、调度、插件、协议、调用链"
 readme = "docs/README.md"
 requires-python = ">=3.8"

{echotools-2.3.47 → echotools-2.3.49}/src/echotools/exec/fncall/protocols/entml_stream_json.py

@@ -72,7 +72,11 @@ def _parse_parameter_entries(body: str) -> List[Tuple[str, str, bool]]:
 
 
 def build_streaming_json_snapshot(body: str) -> str:
-    """构造当前应已发出的 partial_json 累积串（可未完成）。"""
+    """构造当前应已发出的 partial_json 累积串（可未完成）。
+
+    仅在 ``</entml:invoke>`` 出现后才闭合最外层 ``}``，避免多参数时
+    中间态变成完整 JSON 对象，导致后续 delta 拼接出现 Extra data。
+    """
     invoke_closed = _INVOKE_CLOSE in body
     inner = body[: body.index(_INVOKE_CLOSE)] if invoke_closed else body
     entries = _parse_parameter_entries(inner)
@@ -92,9 +96,7 @@ def build_streaming_json_snapshot(body: str) -> str:
         else:
             return "".join(parts)
 
-    if entries and all(entry[2] for entry in entries):
-        parts.append("}")
-    elif invoke_closed:
+    if invoke_closed:
         parts.append("}")
     return "".join(parts)
 

{echotools-2.3.47 → echotools-2.3.49}/src/echotools/exec/fncall/protocols/entml_think/parse.py

@@ -15,6 +15,7 @@ THINKING_BLOCK_RE = re.compile(
 
 _THINKING_OPEN_PREFIX = "<entml:thinking"
 _THINKING_CLOSE = "</entml:thinking>"
+_FAULT_THINKING_CLOSE = "</thinking>"
 # 与 thinking 共享 `<entml:` 前缀的其它标签：歧义 holdback 应交由工具流式状态机处理
 _AMBIGUOUS_ENTML_PREFIXES = (
     "<entml:invoke",
@@ -142,10 +143,13 @@ class EntmlThinkingStreamFilter:
         self._in_block = False
         # 当前 thinking 块是否已输出过任何内容（用于首片去前导空白）
         self._thinking_started = False
+        # 见到 ``</thinking>`` 后暂存，仅当后续出现 invoke 才视为闭合；否则当纯文本
+        self._fault_watch = False
+        self._fault_buffer = ""
 
     def in_open_thinking(self) -> bool:
-        """思考块已开始且尚未收到 </entml:thinking>（含开/闭标签分片 hold）。"""
-        if self._in_block:
+        """思考块已开始且尚未收到 ``</entml:thinking>``（含开/闭标签分片 hold）。"""
+        if self._in_block or self._fault_watch:
             return True
         open_at = self._pending.find(_THINKING_OPEN_PREFIX)
         if open_at >= 0:
@@ -157,47 +161,47 @@ class EntmlThinkingStreamFilter:
             return True
         return False
 
-    def _split_thinking_at_tool_markup(self, out: List[Tuple[str, str]]) -> bool:
-        """thinking 块内一旦出现工具 entml 前缀，立即切出为正文（含未收齐开标签）。"""
-        tool_at = find_ambiguous_entml_tool_prefix(self._pending)
-        if tool_at < 0:
+    def _try_resolve_fault_watch_on_invoke(self) -> bool:
+        """``</thinking>`` 之后若出现完整 invoke 开标签，则在该处结束 thinking。"""
+        if not self._fault_watch or not self._fault_buffer:
             return False
-        close_at = self._pending.find(_THINKING_CLOSE)
-        if close_at >= 0 and tool_at >= close_at:
+        invoke_at = find_complete_entml_invoke_open(self._fault_buffer)
+        if invoke_at < len(_FAULT_THINKING_CLOSE):
             return False
-        if tool_at > 0:
-            emitted = self._emit_thinking_piece(self._pending[:tool_at])
-            if emitted:
-                out.append(("thinking", emitted))
         self._in_block = False
+        self._fault_watch = False
         self._thinking_started = False
-        self._pending = self._pending[tool_at:]
+        self._pending = self._fault_buffer[len(_FAULT_THINKING_CLOSE) :]
+        self._fault_buffer = ""
         return True
 
-    def _split_thinking_at_invoke(self, out: List[Tuple[str, str]]) -> bool:
-        """thinking 未闭合但已出现完整 invoke 开标签时，提前结束 thinking 块。"""
-        invoke_at = find_complete_entml_invoke_open(self._pending)
-        if invoke_at < 0:
+    def _try_resolve_fault_watch_on_entml_close(self, out: List[Tuple[str, str]]) -> bool:
+        """未见 invoke 时出现 ``</entml:thinking>`` → ``</thinking>`` 仅为思考内纯文本。"""
+        if not self._fault_watch:
             return False
-        close_at = self._pending.find(_THINKING_CLOSE)
-        if close_at >= 0 and invoke_at >= close_at:
+        close_at = self._fault_buffer.find(_THINKING_CLOSE)
+        if close_at < 0:
             return False
-        if invoke_at > 0:
-            emitted = self._emit_thinking_piece(self._pending[:invoke_at])
-            if emitted:
-                out.append(("thinking", emitted))
+        emitted = self._emit_thinking_piece(self._fault_buffer[:close_at])
+        if emitted:
+            out.append(("thinking", emitted))
         self._in_block = False
+        self._fault_watch = False
         self._thinking_started = False
-        self._pending = self._pending[invoke_at:]
+        self._pending = self._fault_buffer[close_at + len(_THINKING_CLOSE) :]
+        self._fault_buffer = ""
         return True
 
-    def _feed_in_block(self, out: List[Tuple[str, str]]) -> bool:
-        """处理块内 pending；返回 False 表示应退出 feed 主循环。"""
-        if self._split_thinking_at_invoke(out):
+    def _feed_fault_watch(self, out: List[Tuple[str, str]]) -> bool:
+        """处理 ``</thinking>`` 容错观察期；返回 True 表示可继续主循环。"""
+        if self._try_resolve_fault_watch_on_invoke():
             return True
-        if self._split_thinking_at_tool_markup(out):
+        if self._try_resolve_fault_watch_on_entml_close(out):
             return True
+        return False
 
+    def _feed_in_block(self, out: List[Tuple[str, str]]) -> bool:
+        """处理块内 pending；返回 False 表示应退出 feed 主循环。"""
         close_at = self._pending.find(_THINKING_CLOSE)
         if close_at >= 0:
             piece = self._pending[:close_at]
@@ -205,10 +209,23 @@ class EntmlThinkingStreamFilter:
             if emitted:
                 out.append(("thinking", emitted))
             self._in_block = False
+            self._fault_watch = False
+            self._fault_buffer = ""
             self._thinking_started = False
             self._pending = self._pending[close_at + len(_THINKING_CLOSE) :]
             return True
 
+        fault_at = self._pending.find(_FAULT_THINKING_CLOSE)
+        if fault_at >= 0:
+            before = self._pending[:fault_at]
+            emitted = self._emit_thinking_piece(before)
+            if emitted:
+                out.append(("thinking", emitted))
+            self._fault_watch = True
+            self._fault_buffer = self._pending[fault_at:]
+            self._pending = ""
+            return False
+
         safe, tool_hold = _hold_ambiguous_tool_markup(self._pending)
         if tool_hold:
             if safe:
@@ -219,6 +236,8 @@ class EntmlThinkingStreamFilter:
             return False
 
         safe, hold = _hold_prefix(self._pending, _THINKING_CLOSE)
+        if not hold:
+            safe, hold = _hold_prefix(safe, _FAULT_THINKING_CLOSE)
         if safe:
             emitted = self._emit_thinking_piece(safe)
             if emitted:
@@ -257,6 +276,12 @@ class EntmlThinkingStreamFilter:
         out: List[Tuple[str, str]] = []
 
         while self._pending:
+            if self._fault_watch:
+                self._fault_buffer += self._pending
+                self._pending = ""
+                if self._feed_fault_watch(out):
+                    continue
+                break
             if self._in_block:
                 if not self._feed_in_block(out):
                     break
@@ -279,39 +304,37 @@ class EntmlThinkingStreamFilter:
 
     def finalize(self) -> List[Tuple[str, str]]:
         out: List[Tuple[str, str]] = []
+        if self._fault_watch:
+            if self._try_resolve_fault_watch_on_invoke():
+                pass
+            elif self._try_resolve_fault_watch_on_entml_close(out):
+                pass
+            else:
+                emitted = self._emit_thinking_piece(self._fault_buffer)
+                if emitted:
+                    out.append(("thinking", emitted))
+                self._fault_buffer = ""
+                self._fault_watch = False
+                self._in_block = False
+                self._thinking_started = False
         if self._in_block:
-            if not self._split_thinking_at_invoke(out):
-                if not self._split_thinking_at_tool_markup(out):
-                    # 未闭合且无工具 markup：剩余 pending 视作思考
-                    emitted = self._emit_thinking_piece(self._pending)
-                    if emitted:
-                        out.append(("thinking", emitted))
-                    self._pending = ""
-                    self._in_block = False
-                    self._thinking_started = False
-                    return out
-            # invoke 已切出，继续处理 _pending 中的正文
-            while self._pending:
-                if self._in_block:
-                    if not self._feed_in_block(out):
-                        break
-                    continue
-                if not self._feed_before_block(out):
-                    break
+            emitted = self._emit_thinking_piece(self._pending)
+            if emitted:
+                out.append(("thinking", emitted))
             self._pending = ""
             self._in_block = False
             self._thinking_started = False
-            return out
-        elif self._pending:
+        if self._pending:
             content, thinking = split_entml_thinking(self._pending)
             if thinking:
                 out.append(("thinking", thinking))
             if content:
                 out.append(("content", content))
             elif self._pending and not thinking:
-                # 仅 hold 了半截开标签：原样吐出，避免吞字
                 out.append(("content", self._pending))
         self._pending = ""
         self._in_block = False
+        self._fault_watch = False
+        self._fault_buffer = ""
         self._thinking_started = False
         return out

{echotools-2.3.47 → echotools-2.3.49}/src/echotools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: echotools
-Version: 2.3.47
+Version: 2.3.49
 Summary: 通用基础设施 SDK：配置、日志、事件、调度、插件、协议、调用链
 Home-page: https://github.com/nichengfuben/echotools
 Author: nichengfuben

{echotools-2.3.47 → echotools-2.3.49}/src/echotools.egg-info/SOURCES.txt

@@ -185,6 +185,7 @@ src/tests/test_dispatch_race.py
 src/tests/test_entml_bare_invoke_format.py
 src/tests/test_entml_protocol.py
 src/tests/test_entml_stream_json.py
+src/tests/test_entml_stream_json_exhaustive.py
 src/tests/test_entml_thinking_history.py
 src/tests/test_entml_thinking_parse.py
 src/tests/test_entml_tool_parse_leak.py

{echotools-2.3.47 → echotools-2.3.49}/src/tests/test_entml_stream_json.py

@@ -30,6 +30,7 @@ def test_json_stream_encoder_monotonic() -> None:
         "hel",
         "lo",
         "</entml:parameter>",
+        "</entml:invoke>",
     ]
     merged = ""
     body = ""

echotools-2.3.49/src/tests/test_entml_stream_json_exhaustive.py

@@ -0,0 +1,201 @@
+from __future__ import annotations
+
+"""流式 partial_json 与 invoke 解析的额外边界测试。"""
+
+import json
+from typing import Any, Dict, List, Tuple
+
+import pytest
+
+from echotools.exec.fncall import get_protocol
+from echotools.exec.fncall.parsers.stream import FncallStreamParser
+from echotools.exec.fncall.protocols.entml_stream_json import (
+    EntmlInvokeJsonStreamEncoder,
+    build_streaming_json_snapshot,
+    encode_streaming_invoke_json,
+)
+
+WRITE_TOOL = [
+    {
+        "type": "function",
+        "function": {
+            "name": "Write",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "path": {"type": "string"},
+                    "contents": {"type": "string"},
+                },
+                "required": ["path", "contents"],
+            },
+        },
+    }
+]
+
+SEARCH_TOOL = [
+    {
+        "type": "function",
+        "function": {
+            "name": "search_web",
+            "parameters": {
+                "type": "object",
+                "properties": {"query": {"type": "string"}},
+                "required": ["query"],
+            },
+        },
+    }
+]
+
+
+def _stream_deltas(text: str, *, chunk: int = 1) -> Tuple[str, List[str]]:
+    enc = EntmlInvokeJsonStreamEncoder()
+    merged = ""
+    deltas: List[str] = []
+    body = ""
+    for i in range(0, len(text), chunk):
+        body = text[: i + chunk]
+        piece = enc.poll(body)
+        if piece:
+            deltas.append(piece)
+            merged += piece
+    return merged, deltas
+
+
+def _feed_stream(text: str, tools: List[Dict[str, Any]], *, chunk: int = 3) -> Tuple[str, List[Dict[str, Any]], str]:
+    proto = get_protocol("entml")
+    parser = FncallStreamParser(protocol=proto, tools=tools)
+    delta_parts: List[str] = []
+    for i in range(0, len(text), chunk):
+        parser.feed(text[i : i + chunk])
+        d = parser.consume_stream_delta()
+        if d:
+            delta_parts.append(d[1])
+    clean, calls = parser.finalize()
+    return clean, calls, "".join(delta_parts)
+
+
+@pytest.mark.parametrize(
+    "body,expect_key",
+    [
+        ('<entml:parameter name="contents">hello', "contents"),
+        ('<entml:parameter name="path">a.py</entml:parameter>\n<entml:parameter name="contents">x', "contents"),
+        ('<entml:parameter name="contents">C:\\Users\\x', "contents"),
+        ('<entml:parameter name="contents">say "hi"', "contents"),
+        ('<entml:parameter name="contents">line1\nline2', "contents"),
+        ('<entml:parameter name="contents">中文测试', "contents"),
+        ('<entml:parameter name="contents"><not-a-tag', "contents"),
+    ],
+)
+def test_stream_snapshot_prefix_is_valid_json_fragment(body: str, expect_key: str) -> None:
+    snap = build_streaming_json_snapshot(body)
+    assert snap.startswith("{")
+    assert f'"{expect_key}"' in snap
+    # 未完成时允许缺闭合引号/括号；已完成部分必须是合法 JSON 前缀
+    if snap.endswith("}"):
+        json.loads(snap)
+
+
+def test_multi_param_stream_monotonic_and_final_json() -> None:
+    text = (
+        '<entml:invoke name="Write">\n'
+        '<entml:parameter name="path">src/a.py</entml:parameter>\n'
+        '<entml:parameter name="contents">hello world</entml:parameter>\n'
+        "</entml:invoke>"
+    )
+    merged, deltas = _stream_deltas(text, chunk=2)
+    assert len(deltas) >= 2
+    assert json.loads(merged) == {"path": "src/a.py", "contents": "hello world"}
+    clean, calls, streamed = _feed_stream(text, WRITE_TOOL)
+    assert len(calls) == 1
+    assert json.loads(calls[0]["function"]["arguments"]) == {
+        "path": "src/a.py",
+        "contents": "hello world",
+    }
+    assert json.loads(streamed) == json.loads(calls[0]["function"]["arguments"])
+
+
+def test_special_chars_in_stream_match_batch_parse() -> None:
+    contents = 'path "x"\\nC:\\tmp\\a'
+    text = (
+        '<entml:invoke name="Write">\n'
+        f'<entml:parameter name="path">out.txt</entml:parameter>\n'
+        f'<entml:parameter name="contents">{contents}</entml:parameter>\n'
+        "</entml:invoke>"
+    )
+    proto = get_protocol("entml")
+    _, batch_calls = proto.parse(text, WRITE_TOOL)
+    _, stream_calls, merged = _feed_stream(text, WRITE_TOOL, chunk=1)
+    assert json.loads(batch_calls[0]["function"]["arguments"]) == json.loads(
+        stream_calls[0]["function"]["arguments"]
+    )
+    json.loads(merged)
+
+
+def test_incomplete_param_suffix_strips_markup() -> None:
+    body = '<entml:parameter name="contents">ok</entml:para'
+    snap = build_streaming_json_snapshot(body)
+    assert "</entml" not in snap
+    assert encode_streaming_invoke_json(body) == '{"contents": "ok"}'
+
+
+def test_stream_delta_every_prefix_no_invalid_escape() -> None:
+    text = (
+        '<entml:invoke name="Write">\n'
+        '<entml:parameter name="contents">'
+        + ("X" * 200)
+        + "</entml:parameter>\n</entml:invoke>"
+    )
+    enc = EntmlInvokeJsonStreamEncoder()
+    merged = ""
+    for i in range(1, len(text) + 1):
+        piece = enc.poll(text[:i])
+        if piece:
+            merged += piece
+        # 任意前缀不应产生未转义的控制字符破坏 JSON 结构
+        assert merged.count('"') >= merged.count("\\") or True
+        if merged.endswith("}"):
+            json.loads(merged)
+
+
+def test_thinking_then_invoke_stream_parse() -> None:
+    text = (
+        "<entml:thinking>\nplan\n</entml:thinking>\n"
+        "执行。\n"
+        '<entml:invoke name="search_web">\n'
+        '<entml:parameter name="query">杭州天气</entml:parameter>\n'
+        "</entml:invoke>"
+    )
+    clean, calls, _ = _feed_stream(text, SEARCH_TOOL, chunk=4)
+    assert len(calls) == 1
+    assert json.loads(calls[0]["function"]["arguments"]) == {"query": "杭州天气"}
+    assert "entml:invoke" not in clean
+    assert "执行" in clean
+
+
+def test_invoke_open_only_no_delta_until_name_closed() -> None:
+    proto = get_protocol("entml")
+    parser = FncallStreamParser(protocol=proto, tools=SEARCH_TOOL)
+    partial = '<entml:invoke name="search'
+    for ch in partial:
+        parser.feed(ch)
+        assert parser.consume_stream_delta() is None
+    parser.feed('_web">\n<entml:parameter name="query">x</entml:parameter>\n</entml:invoke>')
+    d = parser.consume_stream_delta()
+    assert d is not None
+    _, calls = parser.finalize()
+    assert len(calls) == 1
+
+
+def test_json_stream_encoder_without_invoke_close_stays_open() -> None:
+    enc = EntmlInvokeJsonStreamEncoder()
+    body = ""
+    merged = ""
+    for part in [
+        '<entml:parameter name="contents">',
+        "hello",
+        "</entml:parameter>",
+    ]:
+        body += part
+        merged += enc.poll(body)
+    assert merged == '{"contents": "hello"'
+    assert not merged.endswith("}")

{echotools-2.3.47 → echotools-2.3.49}/src/tests/test_fncall_stream_thinking.py

@@ -53,19 +53,16 @@ def test_split_entml_thinking_ignores_unclosed_block() -> None:
     assert INVOKE in content
 
 
-def test_invoke_inside_unclosed_thinking_parsed_when_complete() -> None:
-    """完整 invoke 出现在未闭合 thinking 内时，应提前切出并解析（模型常见误排）。"""
+def test_invoke_inside_unclosed_thinking_not_parsed() -> None:
+    """未闭合 thinking 内的 entml 标签仅作思考文本，不解析为 tool call。"""
     parser = FncallStreamParser(protocol=get_protocol("entml"), tools=TOOLS)
     stream = f"<entml:thinking>\nplan {INVOKE}\n"
     for i in range(0, len(stream), 7):
         parser.feed(stream[i : i + 7])
-    assert parser.has_calls
-    assert "plan" in parser.partial_thinking
-    assert INVOKE not in parser.partial_thinking
+    assert not parser.has_calls
+    assert INVOKE in parser.partial_thinking or "entml:invoke" in parser.partial_thinking
     clean, calls = parser.finalize()
-    assert len(calls) == 1
-    assert calls[0]["function"]["name"] == "get_weather"
-    assert INVOKE not in clean
+    assert len(calls) == 0
 
 
 def test_invoke_after_thinking_close_is_parsed() -> None:
@@ -98,16 +95,52 @@ def test_no_thinking_invoke_still_works() -> None:
     assert "Sure" in clean
 
 
-def test_stream_thinking_incremental_invoke_inside_parsed_when_complete() -> None:
+def test_stream_thinking_invoke_inside_stays_thinking_until_close() -> None:
     parser = FncallStreamParser(protocol=get_protocol("entml"), tools=TOOLS)
     parser.feed("<entml:thinking>\n")
     parser.feed("line one\n")
     assert "line one" in parser.partial_thinking
     assert not parser.has_calls
     parser.feed(f"mention {INVOKE}\n")
-    assert INVOKE not in parser.partial_thinking
-    assert parser.has_calls
+    assert not parser.has_calls
+    assert "entml:invoke" in parser.partial_thinking
     parser.feed("</entml:thinking>\nanswer")
     clean, calls = parser.finalize()
+    assert len(calls) == 0
+    assert "answer" in clean
+
+
+def test_fault_thinking_close_then_invoke() -> None:
+    """``</thinking>`` 后若出现 invoke，则在该处结束思考并解析工具。"""
+    parser = FncallStreamParser(protocol=get_protocol("entml"), tools=TOOLS)
+    text = f"<entml:thinking>\nplan\n</thinking>\n{INVOKE}"
+    for i in range(0, len(text), 5):
+        parser.feed(text[i : i + 5])
+    clean, calls = parser.finalize()
+    assert "plan" in parser.partial_thinking
     assert len(calls) == 1
+    assert calls[0]["function"]["name"] == "get_weather"
+    assert INVOKE not in clean
+
+
+def test_fault_thinking_close_without_invoke_is_plain_text() -> None:
+    """``</thinking>`` 后若无 invoke 直到 ``</entml:thinking>``，则视为思考内纯文本。"""
+    parser = FncallStreamParser(protocol=get_protocol("entml"), tools=TOOLS)
+    text = "<entml:thinking>\nplan\n</thinking>\nmore\n</entml:thinking>\nanswer"
+    parser.feed(text)
+    clean, calls = parser.finalize()
+    assert len(calls) == 0
+    assert "plan" in parser.partial_thinking
+    assert "</thinking>" in parser.partial_thinking or "more" in parser.partial_thinking
     assert "answer" in clean
+
+
+def test_fault_thinking_close_stays_open_while_waiting_for_invoke() -> None:
+    filt = EntmlThinkingStreamFilter()
+    filt.feed("<entml:thinking>\nplan\n</thinking>\nstill ")
+    assert filt.in_open_thinking()
+    filt.feed("waiting\n")
+    assert filt.in_open_thinking()
+    filt.feed(INVOKE)
+    kinds = [k for k, _ in filt.feed("")]
+    assert "content" in kinds or filt.in_open_thinking() is False

{echotools-2.3.47 → echotools-2.3.49}/src/tests/test_stream_realtime_parse.py

@@ -112,7 +112,7 @@ def test_char_by_char_ready_matches_final_for_all_tool_cases() -> None:
         assert _args(incremental) == _args(final) == case.expect_args, case.id
 
 
-def test_unclosed_thinking_invoke_parsed_once_inside_block() -> None:
+def test_unclosed_thinking_invoke_not_parsed_until_close() -> None:
     proto = get_protocol("entml")
     tools = TOOLS
     invoke = (
@@ -124,11 +124,10 @@ def test_unclosed_thinking_invoke_parsed_once_inside_block() -> None:
     ready = []
     for ch in f"<entml:thinking>\nplan {invoke}\n":
         ready.extend(parser.feed(ch))
-    assert len(ready) == 1
+    assert ready == []
     for ch in f"</entml:thinking>\n可见\n{invoke}":
         ready.extend(parser.feed(ch))
     ready.extend(parser.get_ready_tool_calls())
     _, final = parser.finalize()
-    # thinking 内第一次 invoke 已解析；闭合后重复 invoke 会再解析一次
-    assert _names(final) == ["get_weather", "get_weather"]
-    assert len(final) == 2
+    assert _names(final) == ["get_weather"]
+    assert len(final) == 1