echotools 2.3.94__tar.gz → 2.3.97__tar.gz

{echotools-2.3.94 → echotools-2.3.97}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: echotools
-Version: 2.3.94
+Version: 2.3.97
 Summary: 通用基础设施 SDK：配置、日志、事件、调度、插件、协议、调用链
 Home-page: https://github.com/nichengfuben/echotools
 Author: nichengfuben

{echotools-2.3.94 → echotools-2.3.97}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "echotools"
-version = "2.3.94"
+version = "2.3.97"
 description = "通用基础设施 SDK：配置、日志、事件、调度、插件、协议、调用链"
 readme = "README.md"
 requires-python = ">=3.8"

echotools-2.3.97/src/echotools/exec/fncall/protocols/entml_fake_structure_markup.py

@@ -0,0 +1,198 @@
+"""剥离模型误生成的 entml 顶层结构标签与 Tool Result ID 注释。"""
+
+from __future__ import annotations
+
+import re
+from typing import Tuple
+
+from echotools.exec.fncall.protocols.entml_tool_result_comment import (
+    leading_partial_tool_result_id_comment_len,
+    strip_complete_tool_result_id_comments,
+    trailing_partial_tool_result_id_comment_len,
+)
+
+# Step 1：带 id 的 result 整块（含内部正文）剥离。
+_ENTML_RESULT_ID_BLOCK_RE = re.compile(
+    r"<entml:result\b[^>]*\bid\s*=[^>]*>[\s\S]*?</entml:result\s*>",
+    re.IGNORECASE,
+)
+_ENTML_RESULT_ID_OPEN_INNER_RE = re.compile(
+    r"<entml:result\b[^>]*\bid\s*=",
+    re.IGNORECASE,
+)
+
+# Step 2：仅剥离开/闭标签（``>`` 闭合即移除，不删标签间/后正文）。
+_TAG_ONLY_OPEN_RES = (
+    re.compile(r"<entml:funtions_results\b[^>]*>", re.IGNORECASE),
+    re.compile(r"<entml:conversation_history\b[^>]*>", re.IGNORECASE),
+    re.compile(r"<entml:calls\b[^>]*>", re.IGNORECASE),
+    re.compile(r"<function_calling_behavior\b[^>]*>", re.IGNORECASE),
+    re.compile(r"<thinking_behavior\b[^>]*>", re.IGNORECASE),
+    re.compile(r"<entml:result\b[^>]*>", re.IGNORECASE),
+)
+_TAG_ONLY_CLOSE_RES = (
+    re.compile(r"</entml:funtions_results\s*>", re.IGNORECASE),
+    re.compile(r"</entml:conversation_history\s*>", re.IGNORECASE),
+    re.compile(r"</entml:calls\s*>", re.IGNORECASE),
+    re.compile(r"</function_calling_behavior\s*>", re.IGNORECASE),
+    re.compile(r"</thinking_behavior\s*>", re.IGNORECASE),
+    re.compile(r"</entml:result\s*>", re.IGNORECASE),
+)
+
+_FAKE_ENTML_TAG_PREFIXES: Tuple[str, ...] = (
+    "<entml:result",
+    "</entml:result",
+    "<entml:funtions_results",
+    "</entml:funtions_results",
+    "<entml:conversation_history",
+    "</entml:conversation_history",
+    "<entml:calls",
+    "</entml:calls",
+    "<function_calling_behavior",
+    "</function_calling_behavior",
+    "<thinking_behavior",
+    "</thinking_behavior",
+    "</entml:invoke",
+    "</entml:parameter",
+    "</entml:thinking",
+)
+
+# 模型误输出的残缺/单行 ``</entml:…``（无完整闭标签名或未收齐 ``>``）。
+_ORPHAN_ENTML_CLOSE_LINE_RE = re.compile(
+    r"(?m)^\s*(?:●\s*)?</entml:?[a-z0-9_-]*\s*$",
+    re.IGNORECASE,
+)
+_TRAILING_INCOMPLETE_ENTML_CLOSE_RE = re.compile(
+    r"</entml:?[a-z0-9_-]*\s*$",
+    re.IGNORECASE,
+)
+
+
+def _strip_fake_entml_result_id_blocks(text: str) -> Tuple[str, bool]:
+    text, n = _ENTML_RESULT_ID_BLOCK_RE.subn("", text)
+    return text, n > 0
+
+
+def _strip_tag_only_markup(text: str) -> Tuple[str, bool]:
+    found = False
+    for pattern in _TAG_ONLY_OPEN_RES + _TAG_ONLY_CLOSE_RES:
+        text, n = pattern.subn("", text)
+        if n:
+            found = True
+    return text, found
+
+
+def _strip_orphan_entml_close_leaks(text: str) -> Tuple[str, bool]:
+    text, n1 = _ORPHAN_ENTML_CLOSE_LINE_RE.subn("", text)
+    text, n2 = _TRAILING_INCOMPLETE_ENTML_CLOSE_RE.subn("", text)
+    return text, (n1 + n2) > 0
+
+
+def _could_be_fake_entml_tag_prefix(fragment: str) -> bool:
+    frag = fragment.lower()
+    if not frag.startswith("<"):
+        return False
+    if ">" in frag:
+        return False
+    for prefix in _FAKE_ENTML_TAG_PREFIXES:
+        pl = prefix.lower()
+        if pl.startswith(frag) or frag.startswith(pl):
+            return True
+    return False
+
+
+def trailing_partial_fake_entml_structure_len(text: str) -> int:
+    """尾部未收齐至 ``>`` 的伪 entml 结构标签应 hold 的字节数。"""
+    hold = trailing_partial_tool_result_id_comment_len(text)
+    if hold:
+        return hold
+    lt = text.rfind("<")
+    if lt < 0:
+        return 0
+    suffix = text[lt:]
+    if ">" in suffix:
+        return 0
+    if _could_be_fake_entml_tag_prefix(suffix):
+        return len(suffix)
+    return 0
+
+
+def leading_partial_fake_entml_structure_len(text: str) -> int:
+    """开头未收齐至 ``>`` 的伪 entml 结构标签应 hold 的字节数。"""
+    hold = leading_partial_tool_result_id_comment_len(text)
+    if hold:
+        return hold
+    stripped = text.lstrip()
+    if not stripped.startswith("<"):
+        return 0
+    if ">" in stripped:
+        return 0
+    if _could_be_fake_entml_tag_prefix(stripped):
+        return len(text)
+    return 0
+
+
+def _truncate_unclosed_fake_result_id_tail(text: str) -> Tuple[str, bool]:
+    """流式/batch：``<entml:result id=...>`` 未闭合至 ``</entml:result>`` 前均不可见。"""
+    found = False
+    search_from = 0
+    while True:
+        open_m = _ENTML_RESULT_ID_OPEN_INNER_RE.search(text[search_from:])
+        if not open_m:
+            break
+        abs_start = search_from + open_m.start()
+        gt = text.find(">", abs_start)
+        if gt < 0:
+            text = text[:abs_start]
+            found = True
+            break
+        abs_end = gt + 1
+        tail = text[abs_end:]
+        close_m = re.search(r"</entml:result\s*>", tail, re.IGNORECASE)
+        if close_m:
+            search_from = abs_end + close_m.end()
+            continue
+        text = text[:abs_start]
+        found = True
+        break
+    return text, found
+
+
+def strip_fake_entml_structure_markup(content: str) -> Tuple[str, bool]:
+    """batch：id result 整块 → 仅标签 → Tool Result ID 注释。"""
+    if not content:
+        return content, False
+    found = False
+    text = strip_complete_tool_result_id_comments(content)
+    if text != content:
+        found = True
+    text, hit = _strip_fake_entml_result_id_blocks(text)
+    found = found or hit
+    text, hit = _truncate_unclosed_fake_result_id_tail(text)
+    found = found or hit
+    text, hit = _strip_tag_only_markup(text)
+    found = found or hit
+    text, hit = _strip_orphan_entml_close_leaks(text)
+    found = found or hit
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text, found
+
+
+def strip_fake_entml_structure_markup_for_display(content: str) -> Tuple[str, bool]:
+    """流式 ``partial_text``：hold 未收齐标签 + 截断未闭合 id result + 剥离。"""
+    if not content:
+        return content, False
+    found = False
+    tail_hold = trailing_partial_fake_entml_structure_len(content)
+    text = content[:-tail_hold] if tail_hold else content
+    if tail_hold:
+        found = True
+    text, hit = _truncate_unclosed_fake_result_id_tail(text)
+    found = found or hit
+    text, hit = strip_fake_entml_structure_markup(text)
+    found = found or hit
+    tail_hold2 = trailing_partial_fake_entml_structure_len(text)
+    if tail_hold2:
+        text = text[:-tail_hold2]
+        found = True
+    return text, found

{echotools-2.3.94 → echotools-2.3.97}/src/echotools/exec/fncall/protocols/entml_think/parse.py

@@ -3,10 +3,11 @@ from __future__ import annotations
 import re
 from typing import List, Optional, Tuple
 
-from echotools.exec.fncall.protocols.entml_patterns import (
-    entml_invoke_open_may_be_streaming,
-    extract_attr_value,
-    normalize_entml_name,
+from echotools.exec.fncall.protocols.entml_fake_structure_markup import (
+    leading_partial_fake_entml_structure_len,
+    strip_fake_entml_structure_markup,
+    strip_fake_entml_structure_markup_for_display,
+    trailing_partial_fake_entml_structure_len,
 )
 
 THINKING_BLOCK_RE = re.compile(
@@ -67,6 +68,9 @@ def leading_entml_tag_holdback_len(
     """buffer 开头是 entml/thinking/invoke 标签真前缀时，应 hold 的字节数。"""
     if not text:
         return 0
+    comment_hold = leading_partial_fake_entml_structure_len(text)
+    if comment_hold:
+        return comment_hold
     prefixes: List[str] = list(_LEADING_HOLD_PREFIXES)
     if thinking_enabled:
         prefixes.append(_PLAIN_THINKING_OPEN_PREFIX)
@@ -110,6 +114,9 @@ def trailing_entml_tag_holdback_len(text: str) -> int:
     """buffer 尾部是 entml/thinking 标签真前缀时，应 hold 的字节数。"""
     if not text:
         return 0
+    comment_hold = trailing_partial_fake_entml_structure_len(text)
+    if comment_hold:
+        return comment_hold
     invoke_idx = text.rfind("<entml:invoke")
     if invoke_idx >= 0:
         tail_from_invoke = text[invoke_idx:]
@@ -171,6 +178,7 @@ def clean_stream_partial_visible(
     tail_hold = trailing_entml_tag_holdback_len(text)
     if tail_hold:
         text = text[:-tail_hold]
+    text = strip_fake_entml_structure_markup_for_display(text)[0]
     if has_calls:
         text = text.rstrip()
     if not text.strip():

echotools-2.3.97/src/echotools/exec/fncall/protocols/entml_tool_result_comment.py

@@ -0,0 +1,70 @@
+"""剥离模型误生成的 ``<!-- Tool Result ID:… -->``（仅 history 注入合法）。"""
+
+from __future__ import annotations
+
+import re
+
+# 与 format_tool_result_id_comment() 输出格式一致；允许空白变化。
+_TOOL_RESULT_ID_COMMENT_RE = re.compile(
+    r"<!--\s*Tool\s+Result\s+ID\s*:\s*.*?-->",
+    re.IGNORECASE | re.DOTALL,
+)
+
+_MARKER_LOWER = "<!-- tool result id:"
+
+
+def _could_be_tool_result_id_comment_open(fragment: str) -> bool:
+    """``fragment`` 以 ``<!--`` 开头且可能长成 Tool Result ID 注释（未闭合）。"""
+    frag = fragment.strip().lower()
+    if not frag.startswith("<!--"):
+        return False
+    if "-->" in frag:
+        return False
+    return _MARKER_LOWER.startswith(frag) or frag.startswith(_MARKER_LOWER)
+
+
+def strip_complete_tool_result_id_comments(text: str) -> str:
+    """移除已完整闭合的 Tool Result ID HTML 注释。"""
+    if not text:
+        return ""
+    return _TOOL_RESULT_ID_COMMENT_RE.sub("", text)
+
+
+def trailing_partial_tool_result_id_comment_len(text: str) -> int:
+    """尾部未闭合的 Tool Result ID 注释应 hold 的字节数（含 ``<!--`` 起）。"""
+    if not text:
+        return 0
+    idx = text.rfind("<!--")
+    if idx < 0:
+        return 0
+    suffix = text[idx:]
+    if "-->" in suffix:
+        return 0
+    if _could_be_tool_result_id_comment_open(suffix):
+        return len(suffix)
+    return 0
+
+
+def leading_partial_tool_result_id_comment_len(text: str) -> int:
+    """开头未闭合的 Tool Result ID 注释应 hold 的字节数。"""
+    if not text:
+        return 0
+    stripped = text.lstrip()
+    if not stripped.startswith("<!--"):
+        return 0
+    if "-->" in stripped:
+        return 0
+    if _could_be_tool_result_id_comment_open(stripped):
+        return len(text)
+    return 0
+
+
+def strip_tool_result_id_comments_for_display(text: str) -> str:
+    """流式可见区：去掉完整注释；截断尾部未闭合的伪注释。"""
+    if not text:
+        return ""
+    cleaned = strip_complete_tool_result_id_comments(text)
+    tail_hold = trailing_partial_tool_result_id_comment_len(cleaned)
+    if tail_hold:
+        cleaned = cleaned[:-tail_hold]
+    return cleaned

{echotools-2.3.94 → echotools-2.3.97}/src/echotools/exec/fncall/shared/history_markup.py

@@ -5,6 +5,10 @@ from __future__ import annotations
 import re
 from typing import Any, Dict, List, Tuple
 
+from echotools.exec.fncall.protocols.entml_fake_structure_markup import (
+    strip_fake_entml_structure_markup,
+    strip_fake_entml_structure_markup_for_display,
+)
 from echotools.exec.fncall.shared.normalization import normalize_content
 
 _FAKE_HISTORY_TAGS = ("assistant", "tool")
@@ -266,7 +270,9 @@ def strip_fake_history_markup(content: str) -> Tuple[str, bool]:
             parts.append(content[open_m.start() :])
             break
 
-    return "".join(parts), found
+    merged = "".join(parts)
+    merged, hit = strip_fake_entml_structure_markup(merged)
+    return merged, found or hit
 
 
 def _truncate_display_fake_tail(cleaned: str) -> Tuple[str, bool]:
@@ -323,4 +329,5 @@ def strip_fake_history_markup_for_display(content: str) -> Tuple[str, bool]:
         ):
             return cleaned, found
     cleaned2, hit = _truncate_display_fake_tail(cleaned)
-    return cleaned2, found or hit
+    cleaned3, hit2 = strip_fake_entml_structure_markup_for_display(cleaned2)
+    return cleaned3, found or hit or hit2

{echotools-2.3.94 → echotools-2.3.97}/src/echotools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: echotools
-Version: 2.3.94
+Version: 2.3.97
 Summary: 通用基础设施 SDK：配置、日志、事件、调度、插件、协议、调用链
 Home-page: https://github.com/nichengfuben/echotools
 Author: nichengfuben

{echotools-2.3.94 → echotools-2.3.97}/src/echotools.egg-info/SOURCES.txt

@@ -59,9 +59,11 @@ src/echotools/exec/fncall/prompt/prompt_helpers.py
 src/echotools/exec/fncall/prompt/templates.py
 src/echotools/exec/fncall/protocols/__init__.py
 src/echotools/exec/fncall/protocols/entml.py
+src/echotools/exec/fncall/protocols/entml_fake_structure_markup.py
 src/echotools/exec/fncall/protocols/entml_invoke.py
 src/echotools/exec/fncall/protocols/entml_patterns.py
 src/echotools/exec/fncall/protocols/entml_schema.py
+src/echotools/exec/fncall/protocols/entml_tool_result_comment.py
 src/echotools/exec/fncall/protocols/entml_stream/__init__.py
 src/echotools/exec/fncall/protocols/entml_stream/body.py
 src/echotools/exec/fncall/protocols/entml_stream/snap.py
@@ -193,6 +195,7 @@ src/tests/test_dispatch_race.py
 src/tests/test_entml_agent_output_formats.py
 src/tests/test_entml_bare_invoke_format.py
 src/tests/test_entml_batch_stream_comprehensive.py
+src/tests/test_entml_fake_structure_markup.py
 src/tests/test_entml_invoke_known_tool_gate.py
 src/tests/test_entml_parser_comprehensive.py
 src/tests/test_entml_protocol.py

echotools-2.3.97/src/tests/test_entml_fake_structure_markup.py

@@ -0,0 +1,211 @@
+from __future__ import annotations
+
+import pytest
+
+from echotools.exec.fncall import get_protocol
+from echotools.exec.fncall.parsers.stream import FncallStreamParser
+from echotools.exec.fncall.protocols.entml_fake_structure_markup import (
+    strip_fake_entml_structure_markup,
+    strip_fake_entml_structure_markup_for_display,
+)
+from echotools.exec.fncall.protocols.entml_tool_result_comment import (
+    trailing_partial_tool_result_id_comment_len,
+)
+
+READ_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "Read",
+        "parameters": {
+            "type": "object",
+            "properties": {"path": {"type": "string"}},
+            "required": ["path"],
+        },
+    },
+}
+
+
+@pytest.mark.parametrize(
+    "raw,expect_sub,expect_absent,expect_present",
+    [
+        (
+            '前言\n<!-- Tool Result ID:toolu_f1a2b3c4 -->\n后缀',
+            "前言",
+            "Tool Result ID",
+            None,
+        ),
+        (
+            '可见\n<entml:result id="toolu_46bd973b2dbd48e681b54714">\n{"ok":1}\n</entml:result>\n尾',
+            "可见",
+            '{"ok":1}',
+            "尾",
+        ),
+        (
+            "说明\n<entml:funtions_results>\n假正文\n</entml:funtions_results>\n继续",
+            "假正文",
+            "funtions_results",
+            "继续",
+        ),
+        (
+            "正文\n<entml:conversation_history>\n历史假数据\n</entml:conversation_history>\n完",
+            "历史假数据",
+            "conversation_history",
+            "完",
+        ),
+        (
+            "前\n<entml:calls>\n调用块正文\n</entml:calls>\n后",
+            "调用块正文",
+            "entml:calls",
+            "后",
+        ),
+        (
+            "前\n<function_calling_behavior>\n行为块正文\n</function_calling_behavior>\n后",
+            "行为块正文",
+            "function_calling_behavior",
+            "后",
+        ),
+        (
+            "前\n<thinking_behavior>\n思考行为\n</thinking_behavior>\n后",
+            "思考行为",
+            "thinking_behavior",
+            "后",
+        ),
+        (
+            "<entml:result>\nbody\n</entml:result>\nok",
+            "body",
+            "entml:result",
+            "ok",
+        ),
+    ],
+)
+def test_strip_fake_entml_structure_batch(
+    raw: str,
+    expect_sub: str,
+    expect_absent: str,
+    expect_present: str | None,
+) -> None:
+    cleaned, found = strip_fake_entml_structure_markup(raw)
+    assert found
+    assert expect_sub in cleaned
+    assert expect_absent not in cleaned
+    if expect_present:
+        assert expect_present in cleaned
+
+
+def test_tool_result_comment_partial_hold() -> None:
+    partial = "可见\n<!-- Tool Result ID:toolu_abc"
+    assert trailing_partial_tool_result_id_comment_len(partial) == len(
+        "<!-- Tool Result ID:toolu_abc"
+    )
+    display, found = strip_fake_entml_structure_markup_for_display(partial)
+    assert found
+    assert display == "可见\n"
+    assert "Tool Result ID" not in display
+
+
+def test_open_tag_stripped_at_gt_content_kept() -> None:
+    partial = "前言\n<entml:funtions_results>"
+    display, found = strip_fake_entml_structure_markup_for_display(partial)
+    assert found
+    assert display == "前言\n"
+    raw = "前言\n<entml:funtions_results>保留正文"
+    cleaned, _ = strip_fake_entml_structure_markup(raw)
+    assert cleaned == "前言\n保留正文"
+    assert "funtions_results" not in cleaned
+
+
+def test_result_id_block_holds_and_strips_body() -> None:
+    partial = '可见\n<entml:result id="toolu_x'
+    display, found = strip_fake_entml_structure_markup_for_display(partial)
+    assert found
+    assert display == "可见\n"
+    complete = partial + '">\nleak'
+    display2, _ = strip_fake_entml_structure_markup_for_display(complete)
+    assert "entml:result" not in display2
+    assert "leak" not in display2
+
+
+def test_bare_result_tags_only_not_body() -> None:
+    raw = "可见\n<entml:result>\n正文保留\n</entml:result>\n尾"
+    cleaned, found = strip_fake_entml_structure_markup(raw)
+    assert found
+    assert "正文保留" in cleaned
+    assert "entml:result" not in cleaned
+    assert "可见" in cleaned
+    assert "尾" in cleaned
+
+
+def test_orphan_incomplete_entml_close_leak() -> None:
+    raw = (
+        "说明正文\n"
+        "<!-- Tool Result ID:toolu_check_theme_btn -->\n"
+        "</entml:"
+    )
+    cleaned, found = strip_fake_entml_structure_markup(raw)
+    assert found
+    assert "说明正文" in cleaned
+    assert "entml" not in cleaned.lower()
+    assert "Tool Result ID" not in cleaned
+
+
+def test_rogator_response_log_corpus_no_entml_tail() -> None:
+    from pathlib import Path
+
+    log = Path(__file__).resolve().parent / "fixtures" / "rogator_entml_close_leak.txt"
+    raw = log.read_text(encoding="utf-8")
+    READ = {
+        "type": "function",
+        "function": {
+            "name": "Grep",
+            "parameters": {
+                "type": "object",
+                "properties": {"pattern": {"type": "string"}, "path": {"type": "string"}},
+                "required": ["pattern"],
+            },
+        },
+    }
+    proto = get_protocol("entml")
+    clean, calls = proto.parse(raw, [READ])
+    assert calls
+    assert "entml" not in clean.lower()
+
+
+def test_batch_parse_strips_fake_structure() -> None:
+    text = (
+        "回答正文\n"
+        '<!-- Tool Result ID:call_0000 -->\n'
+        '<entml:result id="toolu_abc">\n{"x":1}\n</entml:result>\n'
+        "<entml:funtions_results>\n标签间保留\n</entml:funtions_results>"
+    )
+    proto = get_protocol("entml")
+    clean, calls = proto.parse(text, [READ_TOOL])
+    assert calls == []
+    assert "回答正文" in clean
+    assert "Tool Result ID" not in clean
+    assert "entml:result" not in clean
+    assert '{"x":1}' not in clean
+    assert "标签间保留" in clean
+    assert "funtions_results" not in clean
+
+
+@pytest.mark.parametrize("chunk", [1, 4, 17, 32])
+def test_stream_partial_no_fake_structure_leak(chunk: int) -> None:
+    text = (
+        "流式可见\n"
+        '<!-- Tool Result ID:toolu_leak -->\n'
+        '<entml:result id="toolu_x">\n{"fake":true}\n</entml:result>\n'
+        "<entml:conversation_history>\n中间保留\n"
+    )
+    parser = FncallStreamParser(protocol=get_protocol("entml"), tools=[READ_TOOL])
+    for i in range(0, len(text), chunk):
+        parser.feed(text[i : i + chunk])
+        pt = parser.partial_text
+        assert "Tool Result ID" not in pt
+        assert "entml:result" not in pt
+        assert "conversation_history" not in pt
+        assert "fake" not in pt
+    clean, calls = parser.finalize()
+    assert calls == []
+    assert "流式可见" in clean
+    assert "中间保留" in clean
+    assert "Tool Result ID" not in clean