agentengine-sdk-python 0.8.1__py3-none-any.whl → 0.8.2__py3-none-any.whl

ksadk/context_engine/tokenizer.py

@@ -0,0 +1,164 @@
+"""TokenCounter 协议与启发式实现。
+
+对齐方案 8.9。实现顺序应是 provider 官方 tokenizer → 兼容 tokenizer → CJK+ASCII
+启发式。第一个 PR 只落地启发式实现（复用现有 ``estimate_text_tokens``），并记录所用
+tokenizer 名称；只有 heuristic 可用时由调用方自行加安全系数。tiktoken/provider
+tokenizer 接入留后续 PR。
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Protocol, Sequence
+
+HEURISTIC_TOKENIZER_NAME = "heuristic_cjk_ascii"
+
+
+class TokenCounter(Protocol):
+    """token 计数协议。"""
+
+    name: str
+
+    def count_text(self, text: str, *, model: str | None = None) -> int: ...
+
+    def count_messages(self, messages: Sequence[Any], *, model: str | None = None) -> int: ...
+
+
+class HeuristicTokenCounter:
+    """复用 ``ksadk.conversations.model_context.estimate_text_tokens`` 的启发式计数器。
+
+    CJK 字符按约 1.5 token，其他按 4 chars ~= 1 token。不是真实 tokenizer，但比纯英文
+    口径更接近本地中文使用体验。第一个 PR 的 shadow ContextPlan 只用它做可观测估算，
+    不进任何决策路径。
+    """
+
+    name = HEURISTIC_TOKENIZER_NAME
+
+    def count_text(self, text: str, *, model: str | None = None) -> int:
+        from ksadk.conversations.model_context import estimate_text_tokens
+
+        return estimate_text_tokens(text)
+
+    def count_messages(self, messages: Sequence[Any], *, model: str | None = None) -> int:
+        from ksadk.conversations.model_context import estimate_text_tokens
+
+        total = 0
+        for message in messages:
+            total += self._count_message(message, estimate_text_tokens)
+        return total
+
+    @staticmethod
+    def _count_message(message: Any, estimator: Any) -> int:
+        if isinstance(message, str):
+            return estimator(message)
+        if isinstance(message, dict):
+            total = 0
+            for key in ("content", "text", "output"):
+                value = message.get(key)
+                if isinstance(value, str):
+                    total += estimator(value)
+                elif isinstance(value, list):
+                    for part in value:
+                        if isinstance(part, dict):
+                            text = part.get("text") or part.get("content")
+                            if isinstance(text, str):
+                                total += estimator(text)
+                        elif isinstance(part, str):
+                            total += estimator(part)
+            role = message.get("role")
+            if isinstance(role, str):
+                total += estimator(role)
+            return total
+        # LangChain/BaseMessage 风格对象：尽量取 content。
+        content = getattr(message, "content", None)
+        if isinstance(content, str):
+            return estimator(content)
+        if isinstance(content, list):
+            total = 0
+            for part in content:
+                if isinstance(part, dict):
+                    text = part.get("text") or part.get("content")
+                    if isinstance(text, str):
+                        total += estimator(text)
+                elif isinstance(part, str):
+                    total += estimator(part)
+            return total
+        return estimator(str(message))
+
+
+_DEFAULT_COUNTER: HeuristicTokenCounter | None = None
+_PROVIDER_COUNTER: "TokenCounter | None" = None
+
+
+class _TiktokenTokenCounter:
+    """tiktoken 兼容 tokenizer（方案 §8.9 实现顺序 2）。
+
+    用于 OpenAI cl100k_base/o200k 系模型；非该系模型回退到 heuristic。``name`` 记录实际
+    tokenizer，供 ContextPlan ``tokenizer`` 字段如实标注。
+    """
+
+    def __init__(self, encoding_name: str = "cl100k_base") -> None:
+        try:
+            import tiktoken  # type: ignore
+
+            self._enc = tiktoken.get_encoding(encoding_name)
+            self._encoding_name = encoding_name
+        except Exception:  # noqa: BLE001
+            self._enc = None
+            self._encoding_name = encoding_name
+
+    @property
+    def name(self) -> str:
+        if self._enc is None:
+            return HEURISTIC_TOKENIZER_NAME
+        return f"tiktoken:{self._encoding_name}"
+
+    def count_text(self, text: str, *, model: str | None = None) -> int:
+        if self._enc is None:
+            return HeuristicTokenCounter().count_text(text)
+        try:
+            return len(self._enc.encode(str(text or "")))
+        except Exception:  # noqa: BLE001
+            return HeuristicTokenCounter().count_text(text)
+
+    def count_messages(self, messages: Sequence[Any], *, model: str | None = None) -> int:
+        total = 0
+        for message in messages:
+            total += HeuristicTokenCounter._count_message(message, self.count_text)
+        return total
+
+
+def _provider_counter_enabled() -> bool:
+    """是否启用 provider/兼容 tokenizer（方案 §8.9）。
+
+    默认 **关闭**（保持 heuristic baseline，不静默改变既有计数口径——方案 §8.9 "先观测后接管"）；
+    显式 ``KSADK_TOKENIZER_PROVIDER=auto|tiktoken`` 才尝试 tiktoken，不可用时回退 heuristic。
+    """
+    raw = str(os.environ.get("KSADK_TOKENIZER_PROVIDER", "") or "").strip().lower()
+    return raw in ("auto", "tiktoken")
+
+
+def get_default_token_counter() -> TokenCounter:
+    """返回进程级默认 TokenCounter（方案 §8.9）。
+
+    优先 provider/兼容 tokenizer（``KSADK_TOKENIZER_PROVIDER=auto`` 时尝试 tiktoken，不可用
+    回退 heuristic）；``auto`` 之外显式 ``heuristic`` 则只用启发式。名称如实记录，偏差监控由
+    调用方按 model 维度做（方案 §8.9 末）。
+    """
+    global _PROVIDER_COUNTER
+    if _provider_counter_enabled() and _PROVIDER_COUNTER is None:
+        _PROVIDER_COUNTER = _TiktokenTokenCounter()
+    if _PROVIDER_COUNTER is not None and _provider_counter_enabled():
+        return _PROVIDER_COUNTER
+    global _DEFAULT_COUNTER
+    if _DEFAULT_COUNTER is None:
+        _DEFAULT_COUNTER = HeuristicTokenCounter()
+    return _DEFAULT_COUNTER
+
+
+def set_default_token_counter(counter: TokenCounter | None) -> None:
+    """测试/注入用：覆盖默认 counter。``None`` 恢复自动解析。"""
+    global _PROVIDER_COUNTER, _DEFAULT_COUNTER
+    _PROVIDER_COUNTER = counter
+    if counter is None:
+        _DEFAULT_COUNTER = None

ksadk/conversations/context.py

@@ -6,6 +6,11 @@ from collections.abc import Mapping, Sequence
 from typing import Any, Dict, Iterable, List
 
 from ksadk.sessions.base import SessionEvent
+from ksadk.tools.result_budget import (
+    ToolResultBudget,
+    budget_tool_output,
+    default_tool_result_budget,
+)
 
 CANONICAL_EVENT_TYPES = {
     "user_message",
@@ -34,11 +39,23 @@ TRANSCRIPT_EVENT_TYPES = {
     "context_checkpoint",
 }
 
+_RUNTIME_PLACEHOLDER_EVENT_TYPES = {
+    "tool_call",
+    "tool_result",
+    "approval_request",
+    "approval_response",
+}
+
 DATA_URL_RE = re.compile(r"data:(?P<mime>[A-Za-z0-9.+-]+/[A-Za-z0-9.+-]+);base64,[A-Za-z0-9+/=_-]+")
 BASE64_FIELD_RE = re.compile(
     r"(?P<prefix>['\"](?P<field>file_data|data|bytes|base64)['\"]\s*:\s*['\"])(?P<value>[A-Za-z0-9+/=_-]{512,})(?P<suffix>['\"])",
     re.IGNORECASE,
 )
+_CORRECTION_MARKER_RE = re.compile(
+    r"(?:修正|更正|改为|更新为|最新(?:的)?|废弃|作废|不再使用|不是.+而是|不要|不得|禁止)"
+)
+_LATEST_USER_INSTRUCTION_MAX_CHARS = 8192
+_CORRECTION_SUMMARY_MAX_CHARS = 2048
 
 
 def sanitize_event_text_for_context(text: Any) -> str:
@@ -55,8 +72,7 @@ def sanitize_event_text_for_context(text: Any) -> str:
     value = DATA_URL_RE.sub(_replace_data_url, value)
     value = BASE64_FIELD_RE.sub(
         lambda match: (
-            f"{match.group('prefix')}[base64 {match.group('field')} omitted]"
-            f"{match.group('suffix')}"
+            f"{match.group('prefix')}[base64 {match.group('field')} omitted]{match.group('suffix')}"
         ),
         value,
     )
@@ -146,6 +162,70 @@ def _stringify_part_text(value: Any) -> str:
     return str(value)
 
 
+def budget_tool_result_for_event(
+    *,
+    tool_name: str,
+    tool_output: Any,
+    tool_call_id: str | None,
+    enabled: bool,
+    budget: ToolResultBudget | None = None,
+) -> tuple[str, dict[str, Any]]:
+    """PR C：tool_result 落 SessionEvent 前的单项预算（ksadk_hosted 门控）。
+
+    返回 ``(session_event_text, metadata_extras)``：
+
+    - ``enabled=False`` → ``(str(tool_output), {})``：与旧
+      ``text=str(tool_output)`` **字节级一致**，非ksadk_hosted / framework / native
+      路径零行为变更。
+    - ``enabled=True``：先 ``_stringify_part_text`` 干净渲染（已预算的 toolset dict →
+      ``"preview\\n[persisted-output] path (mime)"``；裸串 → 原串），再若仍超 ``max_chars`` 则
+      ``budget_tool_output`` 落盘+截断，``text = "preview\\n[persisted-output] path (mime)"``，
+      ``extras = {"tool_result_budget": {truncated, original_chars, preview_chars, persisted}}``。
+      未超阈值 → ``(rendered, {})``。
+
+    **不碰 ``metadata.tool_output``**：调用方保留原值（UI/Responses 读取方不受影响，
+    会话存储节省留后续）。
+    只 bound 进 ``content.parts[0].text``——即下一轮 ``extract_event_text`` → ``payload["history"]``
+    → 模型输入的那条 text。已预算 dict 经 ``_stringify_part_text`` 渲染后必小于阈值，不重复落盘。
+    """
+    if not enabled:
+        return str(tool_output), {}
+    active = budget or default_tool_result_budget()
+    rendered = _stringify_part_text(tool_output)
+    if len(rendered) <= active.max_chars:
+        return rendered, {}
+    budgeted = budget_tool_output(
+        tool_name=tool_name,
+        field_name="output",
+        value=tool_output,
+        metadata={"tool_call_id": tool_call_id or ""},
+        budget=active,
+    )
+    preview = str(budgeted.get("output") or "")
+    persisted = budgeted.get("persisted")
+    if not isinstance(persisted, Mapping) or not persisted.get("path"):
+        # 无落盘（不应发生，但兜底）→ 退回 rendered 截断标记，不谎报 persisted。
+        marker = f"\n[truncated {len(rendered) - active.max_chars} chars]"
+        return (rendered[: active.max_chars] + marker), {
+            "tool_result_budget": {
+                "truncated": True,
+                "original_chars": int(budgeted.get("original_chars") or len(rendered)),
+                "preview_chars": active.max_chars,
+            }
+        }
+    mime_type = persisted.get("mime_type") or "text/plain"
+    text = f"{preview}\n[persisted-output] {persisted['path']} ({mime_type})"
+    extras = {
+        "tool_result_budget": {
+            "truncated": bool(budgeted.get("truncated")),
+            "original_chars": int(budgeted.get("original_chars") or 0),
+            "preview_chars": int(budgeted.get("preview_chars") or len(preview)),
+            "persisted": dict(persisted),
+        }
+    }
+    return text, extras
+
+
 def build_request_history(messages: Iterable[Dict[str, Any]]) -> List[Dict[str, str]]:
     history: List[Dict[str, str]] = []
     for message in messages or []:
@@ -191,14 +271,16 @@ def summarize_event_groups(
 ) -> str:
     """把要折叠的旧轮次压成一段 checkpoint 文本。
 
-    这里没有直接照搬 Claude Code 的 LLM summarizer，而是先落一个可预测、
-    可恢复的结构化摘要骨架，后续再替换成真正的 summarize agent 也不需要改
-    event contract。
+    extractive fallback：结构化骨架 + 有界保留错误修正和最新长 user 指令。
+    无摘要模型时，关键修正可能位于长消息尾部，也可能不是最后一条 user 消息；
+    因此跨消息提取修正，并在预算上限内保留最新长指令首尾（方案 §9.4）。
     """
     lines: List[str] = []
     if previous_summary:
         lines.append(previous_summary)
     lines.append("Earlier conversation summary:")
+    last_user_text = ""
+    correction_snippets: list[str] = []
     for group in groups:
         snippets: List[str] = []
         for event in group:
@@ -216,9 +298,34 @@ def summarize_event_groups(
                 role = "assistant"
             else:
                 role = "user"
+                last_user_text = text
+                for sentence in re.split(r"[\n。；;]+", text):
+                    normalized = sentence.strip()
+                    if normalized and _CORRECTION_MARKER_RE.search(normalized):
+                        correction_snippets.append(normalized[:512])
             snippets.append(f"{role}: {text[:180]}")
         if snippets:
             lines.append(" | ".join(snippets))
+    # 修正不一定是 compact 范围内最后一条 user 消息；单独形成结构化段，供
+    # Working State 确定性解析。总量有界，避免用“保留完整”重新撑爆上下文。
+    if correction_snippets:
+        unique: list[str] = []
+        for snippet in correction_snippets:
+            if snippet not in unique:
+                unique.append(snippet)
+        correction_text = "；".join(unique[-8:])[-_CORRECTION_SUMMARY_MAX_CHARS:]
+        lines.append(f"错误修正：{correction_text}")
+    # 末尾追加最新 user 指令的有界首尾内容。短消息已在摘要骨架里，不重复。
+    if last_user_text and len(last_user_text) > 180:
+        preserved = last_user_text
+        if len(preserved) > _LATEST_USER_INSTRUCTION_MAX_CHARS:
+            half = _LATEST_USER_INSTRUCTION_MAX_CHARS // 2
+            preserved = (
+                preserved[:half]
+                + "\n...[中间内容因上下文预算省略]...\n"
+                + preserved[-half:]
+            )
+        lines.append(f"最新用户指令（有界保留）: {preserved}")
     return "\n".join(line for line in lines if line).strip()
 
 
@@ -235,6 +342,7 @@ def project_model_messages(
     3. tool/approval/attachment 仍保留成可解释的文本占位，避免状态丢失。
     """
     projected: List[Dict[str, str]] = []
+    placeholder_flags: list[bool] = []
     compacted_until = compacted_until_seq_id(events)
     checkpoint = next(
         (
@@ -253,6 +361,7 @@ def project_model_messages(
                     "content": summary_text,
                 }
             )
+            placeholder_flags.append(False)
 
     for event in events:
         event_type = canonical_event_type(
@@ -291,10 +400,17 @@ def project_model_messages(
         else:
             role = "user"
 
-        if projected and projected[-1]["role"] == role:
+        is_placeholder = event_type in _RUNTIME_PLACEHOLDER_EVENT_TYPES
+        if (
+            projected
+            and projected[-1]["role"] == role
+            and not placeholder_flags[-1]
+            and not is_placeholder
+        ):
             projected[-1]["content"] = f"{projected[-1]['content']}\n{text}".strip()
         else:
             projected.append({"role": role, "content": text})
+            placeholder_flags.append(is_placeholder)
 
     return projected
 
@@ -430,6 +546,10 @@ def project_responses_history(events: List[SessionEvent]) -> List[dict[str, Any]
     from a text summary. Events without a reliable call id fall back to the
     existing explanatory message representation instead of emitting an invalid
     ``function_call_output`` item.
+
+    公开承诺（契约声明见 ``ksadk/events/projections.py``）：仅 OpenAI Responses
+    input item 形态（``type``/``call_id``/``output``/role 消息）；内部不保证
+    compacted 前缀的重放方式与占位消息的具体措辞。
     """
     projected: List[dict[str, Any]] = []
     projected_call_ids: set[str] = set()