iCode-TUI 0.29.1__py3-none-any.whl

chrys/foundation/text/tool_output.py

@@ -0,0 +1,287 @@
+# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved.
+
+"""Shared cleaning and truncation helpers for tool output."""
+
+from __future__ import annotations
+
+import re
+
+from .tokenizer import MixedLanguageTokenizer
+
+_ANSI_RE = re.compile(
+    r"\x1b(?:"
+    r"\[[0-?]*[ -/]*[@-~]"  # CSI sequences (colors, cursor, etc.)
+    r"|\][^\x07\x1b]*(?:\x07|\x1b\\)"  # OSC sequences (title, hyperlinks)
+    r"|[()][0-2AB]"  # Charset designation
+    r"|[\x20-\x2F][\x30-\x7E]"  # Two-byte ESC sequences
+    r"|[=>NOcDEHMZ78\\]"  # Single-byte ESC sequences and a lone string terminator (ST)
+    r")"
+)
+
+_tokenizer = MixedLanguageTokenizer()
+
+
+def strip_ansi(text: str) -> str:
+    """Remove ANSI escape sequences from *text*."""
+    return _ANSI_RE.sub("", text)
+
+
+def process_carriage_returns(text: str) -> str:
+    r"""Collapse carriage-return overwrites while preserving normal newlines."""
+    lines = text.split("\n")
+    result: list[str] = []
+    for index, line in enumerate(lines):
+        # PTY ONLCR expands CRLF to CRCRLF. Strip all trailing CRs from
+        # completed lines; scanning by line stays linear even for long CR runs.
+        if index < len(lines) - 1:
+            line = line.rstrip("\r")
+        if "\r" in line:
+            line = line.rsplit("\r", 1)[-1]
+        result.append(line)
+    return "\n".join(result)
+
+
+def _join_sections(*sections: str) -> str:
+    return "\n".join(section for section in sections if section)
+
+
+def _line_marker(omitted: int, total_tokens: int) -> str:
+    return f"[...truncated {omitted} lines (estimated {total_tokens} tokens total)...]"
+
+
+def _char_marker(omitted: int, total_tokens: int) -> str:
+    return f"[...truncated ~{omitted} chars (estimated {total_tokens} tokens total)...]"
+
+
+def _largest_prefix_within_budget(text: str, budget: int) -> str:
+    """Return the longest prefix whose full estimate fits *budget*."""
+    low = 0
+    high = len(text)
+    while low < high:
+        mid = (low + high + 1) // 2
+        if _tokenizer.count_tokens(text[:mid]) <= budget:
+            low = mid
+        else:
+            high = mid - 1
+    return text[:low]
+
+
+def _char_truncate(
+    text: str,
+    max_tokens: int,
+    *,
+    head_ratio: float,
+    truncation_suffix: str,
+    total_tokens: int,
+) -> str:
+    """Truncate at character granularity, measuring every complete candidate."""
+
+    def candidate(kept: int, suffix: str) -> str:
+        head_count = int(kept * head_ratio)
+        tail_count = kept - head_count
+        head = text[:head_count]
+        tail = text[len(text) - tail_count :] if tail_count else ""
+        marker = _char_marker(len(text) - kept, total_tokens)
+        return _join_sections(head, marker, suffix, tail)
+
+    # A suffix is useful only when the marker and suffix fit together.  If
+    # they do not, degrade to an ordinary bounded truncation without it.
+    suffix = truncation_suffix
+    if suffix and _tokenizer.count_tokens(candidate(0, suffix)) > max_tokens:
+        suffix = ""
+
+    if _tokenizer.count_tokens(candidate(0, suffix)) > max_tokens:
+        marker = _char_marker(len(text), total_tokens)
+        return _largest_prefix_within_budget(marker, max_tokens)
+
+    low = 0
+    high = max(0, len(text) - 1)
+    while low < high:
+        mid = (low + high + 1) // 2
+        if _tokenizer.count_tokens(candidate(mid, suffix)) <= max_tokens:
+            low = mid
+        else:
+            high = mid - 1
+
+    result = candidate(low, suffix)
+    while low > 0 and _tokenizer.count_tokens(result) > max_tokens:
+        low -= 1
+        result = candidate(low, suffix)
+    return result
+
+
+def truncate_output(
+    text: str,
+    max_tokens: int,
+    *,
+    head_ratio: float = 1 / 3,
+    truncation_suffix: str = "",
+    force_truncation: bool = False,
+) -> str:
+    """Return a head/tail view of *text* bounded by *max_tokens*.
+
+    The complete returned string is measured after assembly, including the
+    marker and optional suffix.  The suffix is ignored when the original text
+    fits unless ``force_truncation`` is set; multi-item callers use that mode
+    when later items must be represented by an omission footer.  ``max_tokens``
+    must be at least one; zero is a caller-level disable value and is
+    intentionally outside this helper's domain.
+    """
+    if max_tokens < 1:
+        raise ValueError("max_tokens must be at least 1")
+    if not 0 <= head_ratio <= 1:
+        raise ValueError("head_ratio must be between 0 and 1")
+
+    total_tokens = _tokenizer.count_tokens(text)
+    if total_tokens <= max_tokens and not force_truncation:
+        return text
+
+    lines = text.splitlines()
+    if len(lines) <= 1:
+        return _char_truncate(
+            text,
+            max_tokens,
+            head_ratio=head_ratio,
+            truncation_suffix=truncation_suffix,
+            total_tokens=total_tokens,
+        )
+
+    initial_marker = _line_marker(len(lines), total_tokens)
+    footer = _join_sections(initial_marker, truncation_suffix)
+    if _tokenizer.count_tokens(footer) > max_tokens:
+        return _char_truncate(
+            text,
+            max_tokens,
+            head_ratio=head_ratio,
+            truncation_suffix=truncation_suffix,
+            total_tokens=total_tokens,
+        )
+
+    approximate_payload_budget = max_tokens - _tokenizer.count_tokens(footer)
+    head_budget = int(approximate_payload_budget * head_ratio)
+    tail_budget = approximate_payload_budget - head_budget
+
+    head_end = 0
+    head_used = 0
+    while head_end < len(lines):
+        line_cost = _tokenizer.count_tokens(lines[head_end]) + 1
+        if head_used + line_cost > head_budget:
+            break
+        head_used += line_cost
+        head_end += 1
+
+    tail_start = len(lines)
+    tail_used = 0
+    while tail_start > head_end:
+        line_cost = _tokenizer.count_tokens(lines[tail_start - 1]) + 1
+        if tail_used + line_cost > tail_budget:
+            break
+        tail_used += line_cost
+        tail_start -= 1
+
+    # Forced multi-item truncation must actually omit payload; otherwise a
+    # zero-line marker could claim truncation while all lines survive.
+    if force_truncation and tail_start <= head_end:
+        return _char_truncate(
+            text,
+            max_tokens,
+            head_ratio=head_ratio,
+            truncation_suffix=truncation_suffix,
+            total_tokens=total_tokens,
+        )
+
+    # A giant boundary line would otherwise discard all usable payload.
+    if head_end == 0 and tail_start == len(lines):
+        return _char_truncate(
+            text,
+            max_tokens,
+            head_ratio=head_ratio,
+            truncation_suffix=truncation_suffix,
+            total_tokens=total_tokens,
+        )
+
+    def assemble() -> str:
+        omitted = tail_start - head_end
+        marker = _line_marker(omitted, total_tokens)
+        return _join_sections("\n".join(lines[:head_end]), marker, truncation_suffix, "\n".join(lines[tail_start:]))
+
+    result = assemble()
+    while _tokenizer.count_tokens(result) > max_tokens and (head_end > 0 or tail_start < len(lines)):
+        head_share = head_used / head_budget if head_budget else (1.0 if head_end else 0.0)
+        tail_share = tail_used / tail_budget if tail_budget else (1.0 if tail_start < len(lines) else 0.0)
+        if head_end > 0 and (tail_start == len(lines) or head_share >= tail_share):
+            head_end -= 1
+            head_used -= _tokenizer.count_tokens(lines[head_end]) + 1
+        else:
+            tail_used -= _tokenizer.count_tokens(lines[tail_start]) + 1
+            tail_start += 1
+        result = assemble()
+
+    # The complete-string recount is authoritative.  If the line form cannot
+    # satisfy it (usually only at tiny budgets), the character path can.
+    if _tokenizer.count_tokens(result) > max_tokens:
+        return _char_truncate(
+            text,
+            max_tokens,
+            head_ratio=head_ratio,
+            truncation_suffix=truncation_suffix,
+            total_tokens=total_tokens,
+        )
+    return result
+
+
+def truncate_content_texts(
+    texts: list[str],
+    budget: int,
+    *,
+    truncation_suffix: str = "",
+) -> list[str | None] | None:
+    """Bound ordered text payloads while preserving their positional shape."""
+    if budget < 1:
+        raise ValueError("budget must be at least 1")
+    if _tokenizer.count_tokens("\n".join(texts)) <= budget:
+        return None
+
+    kept: list[str] = []
+    for index, text in enumerate(texts):
+        dropped_count = len(texts) - index - 1
+        dropped_line = f"[{dropped_count} further text items omitted.]" if dropped_count else ""
+        footer_suffix = _join_sections(dropped_line, truncation_suffix)
+        marker = _char_marker(len(text), _tokenizer.count_tokens(text))
+        fixed_footer = _join_sections(marker, footer_suffix)
+        candidate_prefix = "\n".join([*kept, text])
+        keep_whole_candidate = _join_sections(candidate_prefix, fixed_footer)
+        if _tokenizer.count_tokens(keep_whole_candidate) <= budget:
+            kept.append(text)
+            continue
+
+        prefix_with_sep = f"{'\n'.join(kept)}\n" if kept else ""
+        prefix_cost = _tokenizer.count_tokens(prefix_with_sep) if prefix_with_sep else 0
+        slot = budget - prefix_cost
+        result: list[str | None] = [*kept]
+        if slot < 1:
+            result.extend([None] * (len(texts) - len(result)))
+            return result
+
+        truncated = truncate_output(text, slot, truncation_suffix=footer_suffix, force_truncation=True)
+        result.append(truncated)
+        result.extend([None] * (len(texts) - len(result)))
+
+        joined = "\n".join(value for value in result if value is not None)
+        while slot > 1 and _tokenizer.count_tokens(joined) > budget:
+            overflow = _tokenizer.count_tokens(joined) - budget
+            slot = max(1, slot - max(1, overflow))
+            result[index] = truncate_output(
+                text,
+                slot,
+                truncation_suffix=footer_suffix,
+                force_truncation=True,
+            )
+            joined = "\n".join(value for value in result if value is not None)
+        if _tokenizer.count_tokens(joined) > budget:
+            result[index] = None
+        return result
+
+    # The aggregate pre-check guarantees the loop normally truncates.  Keep a
+    # defensive shape-preserving fallback for estimator boundary oddities.
+    return [*kept, *([None] * (len(texts) - len(kept)))]

chrys/foundation/text/yaml_io.py

@@ -0,0 +1,62 @@
+# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved.
+
+"""Shared YAML dump helper — readable output for human-edited profile files.
+
+``yaml.dump``'s default behaviour wraps long / multi-line strings into
+double-quoted scalars with ``\\n`` escapes.  Agent and model profile YAMLs
+are meant to be opened and hand-edited by users, so we override string
+serialisation to use the literal block style (``|``) whenever a string
+contains a newline.  The result is natural, line-broken text with no
+escape noise.
+
+Scoped to a custom ``Dumper`` subclass so the representer does not leak
+into other ``yaml.dump`` call sites in the process.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import yaml
+
+
+class _BlockStyleDumper(yaml.SafeDumper):
+    """SafeDumper variant that emits multi-line strings in block style."""
+
+
+def _str_representer(dumper: yaml.SafeDumper, data: str) -> yaml.ScalarNode:
+    """Use ``|`` for multi-line strings; leave single-line strings to the default."""
+    if "\n" in data:
+        # ``|`` preserves line breaks verbatim and is trivial for users to
+        # edit.  Strip trailing whitespace per line to avoid YAML's
+        # "chomping"-style surprises where trailing spaces change meaning.
+        cleaned = "\n".join(line.rstrip() for line in data.splitlines())
+        # Preserve a final trailing newline if the original had one so
+        # round-tripping stays faithful.
+        if data.endswith("\n"):
+            cleaned += "\n"
+        return dumper.represent_scalar("tag:yaml.org,2002:str", cleaned, style="|")
+    return dumper.represent_scalar("tag:yaml.org,2002:str", data)
+
+
+_BlockStyleDumper.add_representer(str, _str_representer)
+
+
+def dump_yaml(data: Any) -> str:
+    """Serialise *data* to YAML using human-friendly block-style strings.
+
+    Args:
+        data: Any ``yaml.dump``-compatible mapping / sequence / scalar.
+
+    Returns:
+        YAML text with ``default_flow_style=False``, keys in insertion
+        order, unicode preserved, and multi-line strings rendered with
+        ``|`` block scalars.
+    """
+    return yaml.dump(
+        data,
+        Dumper=_BlockStyleDumper,
+        default_flow_style=False,
+        allow_unicode=True,
+        sort_keys=False,
+    )

chrys/foundation/tool_call_context.py

@@ -0,0 +1,202 @@
+# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved.
+
+"""Out-of-band tool-call provenance context channel.
+
+Mirrors the ``chrys_kind`` attribute channel (:mod:`chrys.foundation.tool_kinds`):
+producers attach a small JSON-safe provenance dict to a tool object, and the
+persistence pipeline copies it onto the assistant ``function_call`` content as
+``additional_properties["_chrys_tool_context"]`` so offline session.json
+readers can answer "which skill? which MCP server?" without name guessing.
+
+Two segments with different lifecycles:
+
+- **static** (:func:`set_tool_context`) — fixed at tool creation
+  (e.g. MCP server/remote identity); sanitized once at registration and read
+  back as-is.
+- **builder** (:func:`set_tool_context_builder`) — computed per invocation
+  from the final call args (e.g. canonical skill names); sanitized at
+  resolve time by the same segment sanitizer.
+
+Sanitization is exact-or-absent per segment: values are identifiers, so a
+non-conforming or over-cap value drops its key (never truncated — a mangled
+identifier is worse than a missing one), and an over-budget segment drops
+whole. Provenance must never break tool creation or execution: nothing in
+this module raises on bad producer input; problems are debug-logged.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import math
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any, Final
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+logger = logging.getLogger(__name__)
+
+_CONTEXT_ATTR: Final[str] = "chrys_tool_context"
+_CONTEXT_BUILDER_ATTR: Final[str] = "chrys_tool_context_builder"
+
+TOOL_CALL_CONTEXT_METADATA_KEY: Final[str] = "_chrys_tool_context"
+"""Kind-specific provenance persisted on assistant ``function_call`` contents."""
+
+MAX_CONTEXT_VALUE_CHARS: Final[int] = 512
+"""Per-value cap; a longer string value drops its key (exact-or-absent)."""
+
+MAX_CONTEXT_SEGMENT_KEYS: Final[int] = 16
+"""Per-segment key-count bound; exceeding it drops the whole segment."""
+
+MAX_CONTEXT_SEGMENT_SERIALIZED_CHARS: Final[int] = 1024
+"""Per-segment serialized budget; exceeding it drops the whole segment."""
+
+_SCALAR_TYPES: Final[tuple[type, ...]] = (str, int, float, bool)
+
+
+def sanitize_context_segment(segment: Mapping[str, Any], *, source: str = "") -> dict[str, Any]:
+    """Return the sanitized form of one context segment (exact-or-absent).
+
+    The single sanitization authority for both write points: the static
+    segment at :func:`set_tool_context` registration and the builder segment
+    inside :func:`resolve_tool_call_context`. Never raises.
+    """
+    kept: dict[str, Any] = {}
+    for key, value in segment.items():
+        if not isinstance(key, str) or not key:
+            logger.debug("Dropping tool-context key %r from %s: keys must be non-empty strings.", key, source)
+            continue
+        if not isinstance(value, _SCALAR_TYPES):
+            logger.debug("Dropping tool-context key %r from %s: non-scalar value %r.", key, source, type(value))
+            continue
+        if isinstance(value, float) and not math.isfinite(value):
+            # json.dumps would emit non-standard JSON (NaN/Infinity) for these,
+            # and the session store serializes with the same default.
+            logger.debug("Dropping tool-context key %r from %s: non-finite float %r.", key, source, value)
+            continue
+        if isinstance(value, str) and len(value) > MAX_CONTEXT_VALUE_CHARS:
+            # Never truncate: context values are identifiers, and a mangled
+            # identifier is worse than a missing one.
+            logger.debug(
+                "Dropping tool-context key %r from %s: value length %d exceeds %d.",
+                key,
+                source,
+                len(value),
+                MAX_CONTEXT_VALUE_CHARS,
+            )
+            continue
+        kept[key] = value
+    if len(kept) > MAX_CONTEXT_SEGMENT_KEYS:
+        logger.debug(
+            "Dropping tool-context segment from %s: %d keys exceed the %d-key bound.",
+            source,
+            len(kept),
+            MAX_CONTEXT_SEGMENT_KEYS,
+        )
+        return {}
+    if kept:
+        try:
+            # allow_nan=False backstops the per-value filter above: if a
+            # non-finite float ever slips through, the segment drops whole
+            # here instead of writing non-standard JSON into session.json.
+            serialized_len = len(json.dumps(kept, ensure_ascii=False, allow_nan=False))
+        except TypeError, ValueError:  # pragma: no cover - filtered scalars always serialize
+            logger.debug("Dropping tool-context segment from %s: not JSON-serializable.", source)
+            return {}
+        if serialized_len > MAX_CONTEXT_SEGMENT_SERIALIZED_CHARS:
+            logger.debug(
+                "Dropping tool-context segment from %s: serialized size %d exceeds %d.",
+                source,
+                serialized_len,
+                MAX_CONTEXT_SEGMENT_SERIALIZED_CHARS,
+            )
+            return {}
+    return kept
+
+
+def set_tool_context(tool: Any, context: Mapping[str, Any]) -> None:
+    """Sanitize the static context segment and store it on *tool*. Never raises.
+
+    Non-conforming keys are dropped and an over-budget segment is stored
+    empty — each with a debug log. Provenance must not break tool creation
+    any more than tool execution: MCP server/remote names are externally
+    controlled, and a provenance cap must never prevent a tool from loading.
+    """
+    setattr(tool, _CONTEXT_ATTR, sanitize_context_segment(context, source=f"static context of {_tool_label(tool)}"))
+
+
+def get_tool_context(tool: Any) -> Mapping[str, Any] | None:
+    """Return the stored, already-sanitized static context of *tool*, if any."""
+    context = getattr(tool, _CONTEXT_ATTR, None)
+    if isinstance(context, Mapping) and context:
+        return context
+    return None
+
+
+def set_tool_context_builder(tool: Any, builder: Callable[[Mapping[str, Any]], Mapping[str, Any] | None]) -> None:
+    """Register a per-call context builder invoked with the final call args."""
+    setattr(tool, _CONTEXT_BUILDER_ATTR, builder)
+
+
+def resolve_tool_call_context(tool: Any, args: Mapping[str, Any]) -> dict[str, Any]:
+    """Merge the static context with the per-call builder output (static wins).
+
+    A builder key that collides with a static key is dropped and debug-logged
+    — collisions are producer bugs (static and builder key sets must be
+    disjoint), and static-wins keeps this resolve-time output identical to
+    what the kernel-stamp + subkey-backfill persistence sequence produces.
+    Builder exceptions are swallowed (debug log): provenance must never break
+    tool execution. The static segment is used as-is — already sanitized at
+    :func:`set_tool_context`, never re-sanitized.
+    """
+    static = get_tool_context(tool)
+    merged: dict[str, Any] = dict(static) if static else {}
+
+    builder = getattr(tool, _CONTEXT_BUILDER_ATTR, None)
+    if not callable(builder):
+        return merged
+    label = _tool_label(tool)
+    try:
+        built = builder(args)
+    except Exception:
+        logger.debug("Tool-context builder for %s raised; keeping static context only.", label, exc_info=True)
+        return merged
+    if not isinstance(built, Mapping) or not built:
+        return merged
+    for key, value in sanitize_context_segment(built, source=f"context builder of {label}").items():
+        if key in merged:
+            logger.debug(
+                "Dropping builder tool-context key %r for %s: collides with static context (producer bug).",
+                key,
+                label,
+            )
+            continue
+        merged[key] = value
+    return merged
+
+
+def merge_tool_call_context_property(props: Any, tool_context: Mapping[str, Any]) -> None:
+    """Backfill call provenance context at subkey granularity.
+
+    Unlike the kind (an atomic string, whole-key first-write-wins), the
+    context merges per subkey: a pre-execution static context gets the
+    builder subkeys it couldn't know filled in, while existing subkeys are
+    never overwritten (``setdefault``).
+    """
+    if not tool_context:
+        return
+    existing = props.get(TOOL_CALL_CONTEXT_METADATA_KEY)
+    if TOOL_CALL_CONTEXT_METADATA_KEY not in props:
+        props[TOOL_CALL_CONTEXT_METADATA_KEY] = dict(tool_context)
+        return
+    if not isinstance(existing, dict):
+        # Corrupt/foreign value — first-write-wins, leave it alone.
+        return
+    for key, value in tool_context.items():
+        existing.setdefault(key, value)
+
+
+def _tool_label(tool: Any) -> str:
+    name = getattr(tool, "name", None)
+    return name if isinstance(name, str) and name else repr(tool)