toolpatch 0.4.0__py3-none-any.whl

toolpatch/__init__.py

@@ -0,0 +1,51 @@
+"""toolpatch: LLM tool-calling template-mismatch detector and repairer."""
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _metadata_version
+
+try:
+    __version__: str = _metadata_version("toolpatch")
+except PackageNotFoundError:  # unbuilt dev checkout
+    __version__ = "0.0.0.dev0+local"
+
+from toolpatch.extractor import ExtractedToolCalls, extract_tool_calls
+from toolpatch.fixer import heal_json
+from toolpatch.logging import NullLogger, configure_logger, get_logger
+from toolpatch.request import normalize_chat_request
+from toolpatch.stream import StreamHealer
+
+
+def _configure_logging(*, level: str | None = None, quiet: bool = False) -> None:
+    """Thin wrapper so proxytool can drive toolpatch logging without importing
+    the stdlib logger setup inside toolpatch itself.
+
+    ``level`` uses the same tier strings as proxytool: ``debug``, ``info``,
+    ``warning``, or ``error``. When ``level`` is ``None``, toolpatch reverts
+    to its internal no-op logger.
+    """
+    from toolpatch.logging import LogLevels
+    from toolpatch.logging import configure_logger as _cfg
+
+    _valid: dict[str, LogLevels] = {
+        "debug": "debug", "info": "info", "warning": "warning", "error": "error",
+    }
+    _level: LogLevels | None = None
+    if quiet:
+        _level = "error"
+    elif level:
+        _level = _valid.get(level)
+    _cfg("toolpatch", _level)
+
+__all__ = [
+    "ExtractedToolCalls",
+    "NullLogger",
+    "StreamHealer",
+    "__version__",
+    "configure_logger",
+    "extract_tool_calls",
+    "get_logger",
+    "heal_json",
+    "normalize_chat_request",
+]
+
+

toolpatch/extractor.py

@@ -0,0 +1,206 @@
+"""Detect and extract tool-call payloads leaked into assistant ``content``.
+
+Qwen (with a mismatched chat template) writes its tool call as text inside
+``choices[*].message.content`` instead of the structured ``tool_calls`` field.
+Recognition families (all merged into one result, document order):
+
+1. Hermes XML:  <tool_call>{...}</tool_call>
+2. Fenced JSON:  ```json {...} ```
+3. Bare object:  {"name": ..., "arguments": ...} inside prose
+
+Families 2 and 3 can sit inside a template-specific wrapper tag — the live
+Qwen2.5-Coder GGUF run emits ``<tools>{"name": …}</tools>`` or
+``<function>{…}</function>`` depending on the template it loaded with, and
+the MLX build ``<function-call>…</function-call>``.  A wrapper around the
+leak envelope is part of the leak: leaving ``<tools></tools>`` behind in
+``content`` makes clients render stray markup next to the parsed call.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import uuid
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+from toolpatch.fixer import heal_json
+from toolpatch.logging import get_logger
+
+log = get_logger("toolpatch.extractor")
+
+_HERMES_OPEN = "<tool_call>"
+_HERMES_CLOSE = "</tool_call>"
+
+# Group 1 captures the JSON payload.
+_HERMES_RE = re.compile(r"<tool_call>\s*(.*?)\s*</tool_call>", re.DOTALL)
+_FENCED_RE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.DOTALL)
+_RAW_RE = re.compile(
+    r'\{\s*"name"\s*:\s*"[^"]+"\s*,\s*"arguments"\s*:\s*(\{[^{}]*\}|\{.*?\}|"[^"]*")\s*\}',
+    re.DOTALL,
+)
+
+CallSource = Literal["hermes", "fenced", "raw_json", "mixed"]
+
+# Template-specific wrappers around a leaked call (see module docstring):
+# the same Qwen2.5-Coder build writes `<tools>{...}</tools>` or
+# `<function>{...}</function>` depending on the chat template it was loaded
+# with; MLX builds use `<function-call>`. All of them are leak syntax.
+WRAPPER_TAGS = (
+    "tools", "functions", "function",
+    # singular/plural, hyphen/underscore: the same model has been seen writing
+    # `<function>`, `<function-call>` and `<function-calls>` on successive
+    # prompts with one chat template loaded.
+    "tool_call", "tool_calls", "tool-call", "tool-calls",
+    "function_call", "function_calls", "function-call", "function-calls",
+)
+_WRAPPER_OPEN_RE = re.compile(
+    r"<\s*(" + "|".join(re.escape(t) for t in WRAPPER_TAGS) + r")\s*>\s*$"
+)
+_WRAPPER_CLOSE = r"\s*</\s*{tag}\s*>"
+
+
+@dataclass(frozen=True)
+class ExtractedToolCalls:
+    content: str | None                 # cleaned content; None when nothing remains
+    tool_calls: list[dict] = field(default_factory=list)
+    source: CallSource | None = None
+
+
+def _to_tool_call(parsed: dict) -> dict:
+    args = parsed.get("arguments", {})
+    return {
+        "id": f"call_{uuid.uuid4().hex[:24]}",
+        "type": "function",
+        "function": {
+            "name": parsed["name"],
+            "arguments": args if isinstance(args, str) else json.dumps(args),
+        },
+    }
+
+
+def _parse_candidate(regex: re.Pattern[str], m: re.Match[str]) -> dict[str, Any] | None:
+    """Heal and parse one regex match; return the {"name": ...} dict or None.
+
+    The captured payload may be arguments-only (hermes/raw call): in that
+    case fall back to the full match, which is the call object itself.
+    """
+    payload = m.group(1) if (regex.groups and m.group(1) is not None) else m.group(0)
+    healed = heal_json(payload)
+    if not healed:
+        return None
+    try:
+        parsed = json.loads(healed)
+    except json.JSONDecodeError:
+        return None
+    if isinstance(parsed, dict) and "name" in parsed:
+        return parsed
+    # The group-1 payload was arguments-only (hermes/raw call); the full
+    # match is the JSON object to inspect.
+    full = heal_json(m.group(0))
+    if not full:
+        return None
+    try:
+        parsed2 = json.loads(full)
+    except json.JSONDecodeError:
+        return None
+    return parsed2 if isinstance(parsed2, dict) and "name" in parsed2 else None
+
+
+def _absorb_wrapper(content: str, lo: int, hi: int) -> tuple[int, int]:
+    """Extend a leak span over a template wrapper that encloses the call.
+
+    The wrapper is leak syntax, not prose: strip it with the payload instead
+    of leaving `<tools></tools>` in the repaired content. An opener with no
+    close yet (mid-stream) is absorbed too — only the tag is dropped.
+    """
+    m = _WRAPPER_OPEN_RE.search(content[:lo])
+    if m is None:
+        return lo, hi
+    close = re.compile(_WRAPPER_CLOSE.format(tag=re.escape(m.group(1))))
+    cm = close.match(content, hi)
+    return m.start(), (cm.end() if cm else hi)
+
+
+def _collect_all(content: str,
+                 tool_names: set[str] | None) -> ExtractedToolCalls | None:
+    """Run every detector family and merge the distinct calls found.
+
+    Multi-call turns — several Hermes blocks, parallel fenced blocks, or a
+    mix of families — all land in ONE promotion frame, in document order.
+    Overlapping/nested matches (e.g. a raw {"name"…} object inside a fenced
+    block) dedupe to the outermost family, which is probed first.
+    """
+    found: list[tuple[int, int, dict[str, Any], CallSource]] = []
+    detectors: tuple[tuple[CallSource, re.Pattern[str]], ...] = (
+        ("hermes", _HERMES_RE),
+        ("fenced", _FENCED_RE),
+        ("raw_json", _RAW_RE),
+    )
+    for source, regex in detectors:
+        for m in regex.finditer(content):
+            span = m.span()
+            if any(span[0] < e and s < span[1] for s, e, _, _ in found):
+                continue  # nested/overlapping: an outer family already claimed it
+            parsed = _parse_candidate(regex, m)
+            if parsed is None or (
+                tool_names is not None and parsed["name"] not in tool_names
+            ):
+                continue
+            found.append((span[0], span[1], parsed, source))
+    if not found:
+        return None
+    found.sort(key=lambda c: (c[0], c[1]))
+    lo = found[0][0]
+    hi = found[-1][1]  # nested/overlapping spans are deduped, so ends ascend
+    env_lo, env_hi = _absorb_wrapper(content, lo, hi)
+    # Whitespace OUTSIDE the leak envelope (pre-first / post-last span) is
+    # model padding around the leak — strip it. Whitespace INSIDE the
+    # envelope (between two leaked calls) is genuine inter-call prose —
+    # keep it verbatim, including its edge spaces. The envelope can reach
+    # wider than the calls themselves (template wrapper tags), so head and
+    # tail are cut at env_* while the call slices still come from lo/hi.
+    head = content[:env_lo].rstrip()
+    tail = content[env_hi:].lstrip()
+    cleaned = head
+    pos = lo
+    for s, e, _, _ in found:
+        cleaned += content[pos:s]
+        pos = e
+    cleaned += tail
+    calls = [_to_tool_call(parsed) for _, _, parsed, _ in found]
+    families_src: list[CallSource] = list(dict.fromkeys(
+        src for _, _, _, src in found))
+    source: CallSource = families_src[0] if len(families_src) == 1 else "mixed"
+    log.debug("extract probed: %d call(s) from %s", len(calls),
+              "+".join(families_src))
+    return ExtractedToolCalls(content=cleaned or None, tool_calls=calls, source=source)
+
+
+def extract_tool_calls(content: str,
+                       tool_names: set[str] | None = None,
+                       allow_unterminated_hermes: bool = False) -> ExtractedToolCalls | None:
+    """Extract leaked tool calls; None when the content is clean (passthrough).
+
+    Multiple distinct calls — Hermes, fenced, raw-JSON, in any mix — are
+    all returned in a single promotion (document order), so a parallel or
+    rapid-succession multi-tool turn promotes as one ``tool_calls`` array.
+
+    A hermes open tag with no close tag yet means the call is still forming
+    (mid-stream); the bare-JSON rule must not fire early or the open tag
+    would leak out as leftover prose.
+    """
+    if _open_hermes_only(content):
+        if not allow_unterminated_hermes:
+            return None  # still forming mid-stream: hold, don't half-extract
+        content = content + _HERMES_CLOSE  # tolerate the missing close at end
+    return _collect_all(content, tool_names)
+
+
+def _open_hermes_only(content: str) -> bool:
+    """True when a hermes open tag is present but not yet closed."""
+    first_open = content.find(_HERMES_OPEN)
+    if first_open == -1:
+        return False
+    first_close = content.find(_HERMES_CLOSE)
+    return first_close == -1 or first_close < first_open

toolpatch/fixer.py

@@ -0,0 +1,63 @@
+"""Heal malformed JSON emitted by Qwen inside tool-call payloads.
+
+Repairs, in escalating order:
+1. as-is (valid JSON passes through unchanged)
+2. code-fence / smart-quote cleanup, then trailing-comma removal
+3. trailing commas *and* raw newlines inside string literals
+"""
+
+from __future__ import annotations
+
+import json
+import re
+
+from toolpatch.logging import get_logger
+
+log = get_logger("toolpatch.fixer")
+
+_TRAILING_RE = re.compile(r",\s*([}\]])")
+_FENCE_START = re.compile(r"^\s*```(?:json)?\s*")
+_FENCE_END = re.compile(r"\s*```\s*$")
+_SMART = str.maketrans({
+    "\u201c": '"', "\u201d": '"',
+    "\u2018": "'", "\u2019": "'",
+})
+
+
+def _unescape_stray_newlines_in_strings(s: str) -> str:
+    """Escape raw newlines that appear inside JSON string literals."""
+    out: list[str] = []
+    in_str = False
+    esc = False
+    for ch in s:
+        if in_str and not esc and ch == "\n":
+            out.append("\\n")
+            continue
+        out.append(ch)
+        if ch == '"' and not esc:
+            in_str = not in_str
+        esc = in_str and ch == "\\" and not esc
+    return "".join(out)
+
+
+def heal_json(text: str) -> str | None:
+    """Return a repaired JSON string, or None when unrecoverable."""
+    if not text:
+        return None
+    log.debug("fixer.heal_json: attempt on %d chars", len(text))
+    c = _FENCE_START.sub("", text.strip())
+    c = _FENCE_END.sub("", c)
+    c = c.translate(_SMART)
+
+    candidates = (
+        c,
+        _TRAILING_RE.sub(r"\1", c),
+        _TRAILING_RE.sub(r"\1", _unescape_stray_newlines_in_strings(c)),
+    )
+    for attempt in candidates:
+        try:
+            json.loads(attempt)
+            return attempt
+        except json.JSONDecodeError:
+            continue
+    return None

toolpatch/logging.py

@@ -0,0 +1,129 @@
+"""Lightweight logging support for toolpatch.
+
+toolpatch is a pure protocol-repair library. It does not configure the
+Python logging system itself; consumers (proxytool) do that at app startup.
+
+This module provides:
+
+* ``configure_logger(name, level)`` — create/return a logger for ``name`` at
+  an explicit level, or a no-op logger when ``level`` is None.
+* ``NullLogger`` — a drop-in logger whose methods are no-ops.
+* ``LogLevels`` — the accepted tier strings shared by proxytool and toolpatch.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+LogLevels = Literal["debug", "info", "warning", "error"]
+
+
+def configure_logger(name: str, level: LogLevels | None) -> Any:
+    """Return a logger for ``name`` when ``level`` is set, else a no-op logger.
+
+    When a level is supplied, the underlying stdlib logger is created and its
+    threshold is set. It is intentionally not attached to any handler here —
+    the host app (proxytool) owns handler/transport configuration so that
+    logging can be routed alongside Uvicorn/FastAPI output whichever way the
+    app wants.
+    """
+    if level is None:
+        return NullLogger()
+
+    import logging
+
+    logger = logging.getLogger(name)
+    logger.setLevel(_resolve_level(level))
+    return logger
+
+
+def get_logger(name: str = "toolpatch") -> Any:
+    """Shared logger for toolpatch internals.
+
+    Always returns the real stdlib logger for ``name``. Before the host app
+    calls ``configure_logger``, it has no handlers and the default root
+    level drops debug/info lines silently; afterwards, the app-configured
+    level and handlers take over.
+    """
+    import logging
+
+    return logging.getLogger(name)
+
+
+def _configure_app_handler(*, level: int) -> None:
+    """Best-effort handler setup for the shared ``toolpatch`` logger.
+
+    Called by proxytool's ``configure_logging`` so toolpatch debug lines are
+    routed through the same stderr stream/progress as app output. When
+    logging is already configured elsewhere, this is a no-op.
+    """
+    import logging as _logging
+
+    logger = _logging.getLogger("toolpatch")
+    if logger.handlers:
+        return
+    _configure_handler(logger, level)
+
+
+def _configure_handler(logger: Any, level: int) -> None:
+    """Attach a stderr handler to ``logger`` when one is not already present."""
+    import logging as _logging
+    import sys as _sys
+
+    has_target = any(
+        isinstance(h, _logging.StreamHandler) and getattr(h, "stream", None) is _sys.stderr
+        for h in logger.handlers
+    )
+    if has_target:
+        return
+    handler = _logging.StreamHandler(_sys.stderr)
+    handler.setLevel(level)
+    handler.setFormatter(_logging.Formatter(
+        "%(levelname)s [%(name)s] %(message)s",
+    ))
+    logger.addHandler(handler)
+
+
+def _emit(record: Any) -> None:
+    """Best-effort emit through the real stdlib logger when it has handlers."""
+    import logging as _logging
+
+    logger = _logging.getLogger("toolpatch.emit")
+    if not logger.handlers:
+        return
+    logger.handle(record)
+
+
+class NullLogger:
+    """No-op logger used by toolpatch when no app-level logger has been injected.
+
+    Callers can freely call ``debug``/``info``/``warning``/``error`` without
+    guarding on ``isEnabledFor``.
+    """
+
+    def debug(self, *args: Any, **kwargs: Any) -> None:
+        return None
+
+    def info(self, *args: Any, **kwargs: Any) -> None:
+        return None
+
+    def warning(self, *args: Any, **kwargs: Any) -> None:
+        return None
+
+    def error(self, *args: Any, **kwargs: Any) -> None:
+        return None
+
+    def exception(self, *args: Any, **kwargs: Any) -> None:
+        return None
+
+
+def _resolve_level(level: LogLevels) -> int:
+    import logging
+
+    mapping = {
+        "debug": logging.DEBUG,
+        "info": logging.INFO,
+        "warning": logging.WARNING,
+        "error": logging.ERROR,
+    }
+    return mapping[level]

toolpatch/request.py

@@ -0,0 +1,197 @@
+"""Outbound request normalization: give the backend its best shot at matching
+its chat template, without mutating the caller's payload.
+
+Backend-template history rules (from live opencode traces):
+
+- Llama-server rejects histories ending in 2+ assistant messages
+  ("Cannot have 2 or more assistant messages at the end of the list.").
+- When the extractor repairs a leaked tool call, the client records BOTH the
+  leaked text turn and the repaired ``tool_calls`` turn — the text twin must be
+  re-attached to the structured turn to form one assistant message.
+- Identical consecutive assistant turns (client double-append) are deduped.
+"""
+
+from __future__ import annotations
+
+import copy
+from typing import Any, Literal
+
+Backend = Literal["gguf", "mlx"]
+
+
+def normalize_chat_request(
+    payload: dict, backend: Backend = "gguf"
+) -> tuple[dict, list[str]]:
+    """Return (sanitized_payload, operator_notes). Input is deep-copied, never mutated."""
+    out = copy.deepcopy(payload)
+    notes: list[str] = []
+
+    # Rule 1: tool_choice unsupported by mlx_lm.server — strip for MLX.
+    if backend == "mlx" and "tool_choice" in out:
+        del out["tool_choice"]
+        notes.append("tool_choice stripped for mlx backend")
+
+    # Rule 2: empty/null tools drop the key entirely.
+    tools = out.get("tools")
+    if tools is not None and not tools:
+        del out["tools"]
+        notes.append("empty tools array dropped")
+
+    # Rule 3: parameters must be an object schema; drop offending tools.
+    if isinstance(tools, list):
+        good, for_name = [], {t.get("function", {}).get("name", "?") for t in tools}
+        for t in tools:
+            fn = t.get("function") or {}
+            params = fn.get("parameters")
+            if params is not None and not isinstance(params, dict):
+                notes.append(f"tool '{fn.get('name', '?')}' parameters not object-schema; dropped")
+                continue
+            good.append(t)
+        if tools and len(good) != len(tools):
+            out["tools"] = good
+        elif not tools:
+            for_name = set()
+        _ = for_name  # silence ruff if unused in future edits
+
+    _sanitize_history(out["messages"], notes)
+    return out, notes
+
+
+def _sanitize_history(messages: list[dict[str, Any]], notes: list[str]) -> None:
+    """Fix assistant-message history shapes llama-server rejects, in place.
+
+    1. Merge an assistant *text* turn that is a leaked tool call (matches the
+       raw-JSON shape the extractor targets) into the following assistant
+       ``tool_calls`` turn when the tool name + arguments agree.
+    2. Merge consecutive assistant messages into one (template allows only one
+       assistant turn per generation).
+    """
+    if not isinstance(messages, list):
+        return
+
+    import json
+    import re
+
+    leaked = re.compile(r'\{\s*"name"\s*:')
+
+    def _stem(m: dict[str, Any]) -> tuple | None:
+        """(name, arguments-json) of a message's first tool call, else None."""
+        if m.get("role") != "assistant":
+            return None
+        tcs = m.get("tool_calls") or []
+        if not tcs:
+            return None
+        fn = (tcs[0] or {}).get("function") or {}
+        return fn.get("name"), fn.get("arguments")
+
+    merged = 0
+    deduped_ids = 0
+    i = 1  # keep index 0 (system) untouched
+    while i < len(messages):
+        m = messages[i]
+        if m.get("role") != "assistant":
+            i += 1
+            continue
+
+        # 0. Duplicate tool_call id accumulation (LangChain4j-style streaming
+        #    bug from the live trace): the same call id repeated within one
+        #    assistant turn must collapse — llama-server rejects duplicated
+        #    ids with `tool_call_id mismatch` on the reply turn.
+        tcs = m.get("tool_calls") or []
+        if tcs:
+            seen: set[str] = set()
+            fresh: list[dict[str, Any]] = []
+            for tc in tcs:
+                tid = (tc or {}).get("id")
+                if tid and tid in seen:
+                    deduped_ids += 1
+                    continue
+                if tid:
+                    seen.add(tid)
+                fresh.append(tc)
+            if len(fresh) != len(tcs):
+                m["tool_calls"] = fresh
+                notes.append(
+                    f"duplicate tool_call ids deduped ({len(tcs) - len(fresh)} removed)"
+                )
+
+        # 1. text-twins of a repaired tool_calls turn: `leaked` text immediately
+        #    before an assistant tool_calls turn whose call matches.
+        content = m.get("content")
+        if (
+            isinstance(content, str)
+            and leaked.search(content)
+            and i + 1 < len(messages)
+        ):
+            nxt = messages[i + 1]
+            nxt_stem = _stem(nxt)
+            if nxt_stem and not m.get("tool_calls"):
+                try:
+                    parsed = json.loads(content)
+                    name, args_json = nxt_stem
+                    args_norm = (
+                        json.dumps(json.loads(args_json), sort_keys=True)
+                        if isinstance(args_json, str)
+                        else json.dumps(args_json, sort_keys=True)
+                    )
+                    parsed_args_norm = json.dumps(parsed.get("arguments"), sort_keys=True)
+                    if parsed.get("name") == name and parsed_args_norm == args_norm:
+                        # drop the text twin; the tool_calls turn already carries it.
+                        messages.pop(i)
+                        merged += 1
+                        notes.append("leaked tool-call text turn merged into tool_calls turn")
+                        continue
+                except (json.JSONDecodeError, KeyError, TypeError, ValueError):
+                    pass
+
+        # 2. two assistant messages in a row: try to merge
+        if i + 1 < len(messages) and messages[i + 1].get("role") == "assistant":
+            nxt = messages[i + 1]
+            a_calls = m.get("tool_calls") or []
+            b_calls = nxt.get("tool_calls") or []
+            if a_calls or b_calls:
+                if not b_calls:
+                    # m carries the structured call; keep it, adopt nxt's text
+                    # as the turn content when m has none.
+                    if not (m.get("content") or "").strip():
+                        m["content"] = (nxt.get("content") or "").strip()
+                    messages.pop(i + 1)
+                    merged += 1
+                    notes.append("assistant turns merged (text into tool_calls)")
+                    continue
+                if not a_calls:
+                    # nxt carries the structured call; adopt m's text as content
+                    # when nxt has none.
+                    if not (nxt.get("content") or "").strip():
+                        nxt["content"] = (m.get("content") or "").strip()
+                    messages.pop(i)
+                    merged += 1
+                    notes.append("assistant turns merged (text into tool_calls)")
+                    continue
+                # both have tool_calls: merge into the second (tool results follow)
+                nxt["tool_calls"] = a_calls + b_calls
+                messages.pop(i)
+                merged += 1
+                notes.append("assistant turns merged (tool_calls + tool_calls)")
+                continue
+            # plain text duplicates
+            if (m.get("content") or "") == (nxt.get("content") or ""):
+                messages.pop(i)
+                merged += 1
+                notes.append("duplicate adjacent assistant turns deduped")
+                continue
+            # distinct assistant texts: keep the LAST one (the final generation)
+            nxt["content"] = (
+                (m.get("content") or "") + "\n" + (nxt.get("content") or "")
+            ).strip()
+            messages.pop(i)
+            merged += 1
+            notes.append("assistant turns merged (text + text)")
+            continue
+
+        i += 1
+
+    if deduped_ids:
+        notes.append(f"tool_call ids deduped: {deduped_ids}")
+    if merged:
+        notes.append(f"assistant history sanitized: {merged} merge(s)")

toolpatch/schemas.py

@@ -0,0 +1,52 @@
+"""OpenAI-compatible Pydantic schemas shared by the proxytool pipeline."""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict
+
+Role = Literal["system", "user", "assistant", "tool"]
+
+
+class ToolFunction(BaseModel):
+    """A function definition inside a `tools` entry."""
+
+    name: str
+    description: str | None = None
+    parameters: dict[str, Any] | None = None
+
+
+class Tool(BaseModel):
+    type: Literal["function"] = "function"
+    function: ToolFunction
+
+
+class ToolCallFn(BaseModel):
+    name: str
+    arguments: str  # JSON-encoded string, per OpenAI spec
+
+
+class ToolCall(BaseModel):
+    id: str
+    type: Literal["function"] = "function"
+    function: ToolCallFn
+
+
+class Message(BaseModel):
+    role: Role
+    content: str | list[dict[str, Any]] | None = None
+    name: str | None = None
+    tool_calls: list[ToolCall] | None = None
+    tool_call_id: str | None = None
+
+
+class ChatRequest(BaseModel):
+    """Partial OpenAI chat-completion request; unknown fields pass through."""
+
+    model: str
+    messages: list[Message]
+    tools: list[Tool] | None = None
+    stream: bool = False
+
+    model_config = ConfigDict(extra="allow")

toolpatch/stream.py

@@ -0,0 +1,214 @@
+"""StreamHealer: repair tool-call leaks inside SSE content deltas.
+
+Bounded streaming repair: deltas accumulate in a small buffer. Clean prose is
+released quickly; text that *might* be forming a tool call is held until the
+extractor can decide. Emits plain-dict events consumed by the proxy without
+further translation.
+
+The hold rule is NOT tail-only: models like Qwen2.5 stream a reasoning block
+(<tool_call>-style tag) that appears BEFORE the tool-call payload in the same
+buffer, and a tail-prefix-only check releases the whole buffer as "prose"
+because the leak prefix sits mid-buffer. The hold rule therefore also fires
+when a leak prefix appears **inside** the buffer, not just at the end.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from toolpatch.extractor import WRAPPER_TAGS, extract_tool_calls
+from toolpatch.logging import get_logger
+
+log = get_logger("toolpatch.stream")
+
+# Markers that suggest a leak may be forming; while any prefix-match lingers we hold.
+# ALL of these are prefix-triggered hold signals — a tail that could still grow
+# into one of them keeps the buffer held. This covers reasoning/thinking tags
+# (Qwen gpt-think, provider variations) that stream RIGHT BEFORE a leak: the
+# whole 'reasoning + Hermes' sequence holds until the extractor resolves it.
+_HERMES_OPEN = "<tool_call>"
+_FENCE_MARK = "```"
+_JSON_BRACE = '{"name"'
+_BRACE = "{"
+# Template-specific wrappers the extractor strips along with the payload.
+# Without holding on these, `<tools>` is released as prose before the call
+# inside it resolves, and the client renders the tag next to the tool call.
+_WRAPPER_OPENS = tuple(f"<{tag}>" for tag in WRAPPER_TAGS)
+
+
+class StreamHealer:
+    """Buffers streamed content until a tool-call leak is decided; emits events.
+
+    Event shapes (plain dicts, consumed verbatim by the proxy):
+
+    - ``{"type": "content", "text": str}``                       safe to forward
+    - ``{"type": "tool_calls", "tool_calls": [...], "stop": True}``  replace stream
+    - ``{"type": "content", "text": <remaining buffer>}``        flush at finish
+
+    Multi-call aggregation: once any tool-call leak has been seen, the
+    healer enters *collecting* mode — no further prose is released and the
+    promotion is deferred to the stream end (or a 4x``max_buffer`` cap), so
+    rapid-succession or parallel blocks (Hermes/fenced/raw, in any mix) all
+    land in ONE ``tool_calls`` frame instead of promoting the first call
+    and suppressing the rest.
+    """
+
+    def __init__(self, max_buffer: int = 512,
+                 tool_names: set[str] | None = None,
+                 logger: Any | None = None) -> None:
+        self._max = max_buffer
+        self._tool_names = tool_names
+        self._buf = ""
+        self._holding = False
+        self._leak_open = False  # a partial leak prefix is in flight
+        self._done = False  # a leak was extracted; suppress plain content after
+        self._seen_call = False  # collecting mode: aggregate until stream end
+        self._cap = max_buffer * 4  # collecting-mode bound
+        self._log = logger or log
+
+    # -- public ------------------------------------------------------------
+    def feed(self, content_delta: str) -> list[dict]:
+        if self._done or not content_delta:
+            return []
+        self._buf += content_delta
+        events: list[dict] = []
+
+        # Try to decide with what we have (cheap probe: full extractor on buffer).
+        ex = extract_tool_calls(self._buf, self._tool_names)
+        if ex is not None:
+            self._seen_call = True
+        if self._seen_call:
+            # Collecting mode: more calls (rapid succession, parallel
+            # blocks, mixed families) may still be streaming downstream, so
+            # the promotion waits for the stream end — they all land in ONE
+            # ``tool_calls`` frame. Bounded by ``_cap`` for streams that
+            # rack up endless trailing content after a call.
+            if len(self._buf) >= self._cap:
+                self._log.debug("StreamHealer cap reached; promoting buffer")
+                return self._promote(allow_unterminated=False)
+            return events
+
+        if self._should_release():
+            # Retain a small suffix window, NOT an empty buffer: a leak opener
+            # straddling two releases (think>… <tool_ca | ll>{"name") must be
+            # seen whole once the next delta completes it. Everything BEFORE
+            # the window is emitted; the window itself stays buffered and is
+            # re-judged with the next delta (so it is never emitted twice).
+            keep = max(len(_HERMES_OPEN), len(_JSON_BRACE)) - 1
+            emit, self._buf = self._buf[:-keep], self._buf[-keep:]
+            if emit:
+                # A leak callee can start inside the release slice itself:
+                # with a partial open tag at the slice boundary the extractor
+                # cannot yet resolve a call, so re-judge the WHOLE slice
+                # before emitting it as prose. If an open/unresolvable tag is
+                # now in the slice, un-release: re-buffer and hold.
+                if _HERMES_OPEN in emit or self._midbuf_leak(emit):
+                    self._buf = emit + self._buf
+                    self._leak_open = True
+                    self._log.debug("StreamHealer un-releasing slice with leak shape")
+                else:
+                    events.append({"type": "content", "text": emit})
+        return events
+
+    def finish(self) -> list[dict]:
+        """Flush the stream end: decide on whatever buffer remains."""
+        if self._done:
+            return []
+        if not self._buf:
+            return []
+        return self._promote(allow_unterminated=True)
+
+    def _promote(self, allow_unterminated: bool) -> list[dict]:
+        """Resolve the buffer: emit residual content once, then the calls.
+
+        Single promotion point for both feed-time cap overflow and the
+        stream end. An unterminated Hermes tag is only tolerated at the
+        very end of the stream; mid-stream caps flush what resolved.
+        """
+        ex = extract_tool_calls(self._buf, self._tool_names,
+                                allow_unterminated_hermes=allow_unterminated)
+        buf = self._buf
+        self._buf = ""
+        self._done = True
+        self._seen_call = False
+        if ex is None:
+            if buf:
+                return [{"type": "content", "text": buf}]
+            return []
+        events: list[dict] = []
+        # Residual prose (retained window, reasoning tags' text,
+        # pre/post-object preamble) must survive — emitted BEFORE the
+        # promotion frame so a client renders it around the same turn.
+        if ex.content:
+            events.append({"type": "content", "text": ex.content})
+        events.append({
+            "type": "tool_calls",
+            "tool_calls": ex.tool_calls,
+            "stop": True,
+        })
+        self._log.debug("StreamHealer extracted tool calls: %d call(s)",
+                        len(ex.tool_calls))
+        return events
+
+    # -- internals ---------------------------------------------------------
+    def _should_release(self) -> bool:
+        """Release only when we're sure no leak prefix is in flight."""
+        if self._leak_open:
+            # A leak candidate started; hold until it resolves (or max size).
+            if len(self._buf) >= self._max:
+                self._log.debug("StreamHealer releasing held buffer at max size")
+                return True
+            return False
+        if self._tail_holds():
+            # Mark that a leak candidate is now in flight.
+            self._leak_open = True
+            self._log.debug("StreamHealer holding: possible leak opener detected")
+            return False
+        return True
+
+    def _tail_holds(self) -> bool:
+        buf = self._buf
+        tail = buf[-len(_HERMES_OPEN):] if len(buf) >= len(_HERMES_OPEN) else buf
+        if _HERMES_OPEN.startswith(tail) or _HERMES_OPEN in buf[-32:]:
+            return True
+        fence = buf[-len(_FENCE_MARK):] if len(buf) >= len(_FENCE_MARK) else buf
+        if _FENCE_MARK.startswith(fence) or buf.count(_FENCE_MARK) % 2 == 1:
+            return True
+        for opener in _WRAPPER_OPENS:
+            tail = buf[-len(opener):] if len(buf) >= len(opener) else buf
+            if opener.startswith(tail):
+                return True
+        # Any leading '{' opens a candidate tool-call object until either the
+        # extractor resolves it (feed re-checks each delta) or max_buffer
+        # forces release. The replayed mlx stream showed a bare {"name"...}
+        # object released before its arguments were complete.
+        stripped = buf.lstrip()
+        # Hold while the buffer is *only* a candidate JSON object (nothing but
+        # whitespace before the opening brace) and still under max_buffer.
+        if (
+            stripped.startswith(_BRACE)
+            and len(stripped) < self._max
+            and not buf[: len(buf) - len(stripped)].strip()
+        ):
+            return True
+        # Mid-buffer leak start (reasoning-tag-wrapped leaks, live Qwen2.5
+        # mlx trace): a Hermes open / bare-JSON object that BEGINS inside
+        # already-released prose, not at the tail. Tail-only rules let the
+        # whole reasoning+leak sequence release as prose; hold until the
+        # extractor resolves (bounded by max_buffer, as above).
+        return self._midbuf_leak(buf)
+
+    @staticmethod
+    def _midbuf_leak(buf: str) -> bool:
+        """True when a leak-shape opener sits anywhere inside the buffer.
+
+        Cheaper and more conservative than running the full extractor: any
+        Hermes open tag or bare `{"name"` object beginning mid-buffer is a
+        hold signal. Fences are already covered by the tail rule's parity
+        check.
+        """
+        return (
+            _HERMES_OPEN in buf
+            or _JSON_BRACE in buf
+            or any(opener in buf for opener in _WRAPPER_OPENS)
+        )

toolpatch-0.4.0.dist-info/METADATA

@@ -0,0 +1,254 @@
+Metadata-Version: 2.3
+Name: toolpatch
+Version: 0.4.0
+Summary: LLM tool-calling protocol repairer (pure logic, offline-testable)
+Author: Dave Holden
+Author-email: Dave Holden <holden.dave@gmail.com>
+Requires-Dist: pydantic>=2.9
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <img src="public/assets/images/toolpatch.webp" alt="toolpatch" width="200">
+</p>
+
+# toolpatch
+
+[![test & checks](https://github.com/dave-holden/proxytoolpatch/actions/workflows/test.yml/badge.svg)](https://github.com/dave-holden/proxytoolpatch/actions/workflows/test.yml)
+[![PyPI toolpatch](https://img.shields.io/badge/PyPI-toolpatch-blue)](https://pypi.org/project/toolpatch/)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue)](https://www.python.org)
+[![code style: ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+
+**In plain terms:** this is the *fix-it library*. Local AI models sometimes
+send their tool commands as garbled text inside the chat message instead of
+the structured format your software can execute. Feed the mumbled reply to
+toolpatch and get back a clean, structured tool call. It doesn't serve
+anything by itself — you embed it in your own Python code, or let
+**proxytool** use it for you.
+
+A **pure, zero-I/O Python library** for repairing LLM tool-calling
+protocol breakage: no sockets, no event loop, no CLI, nothing runs on its
+own — it is meant to be embedded by any server, proxy, or script that talks
+to local LLMs over an OpenAI-compatible interface.
+
+This is the repair "brick" that **proxytool** (`apps/proxytool`) embeds for all
+of its protocol-repair logic:
+
+```
+proxytool  (the runnable app: HTTP serving, routing, supervision)
+    └─ imports
+toolpatch (this library: pure text repair, no I/O)
+```
+
+Any Python program can do the same `import` — see [Embedding](#embedding).
+
+## Install
+
+```bash
+pip install toolpatch          # once published to PyPI
+# or
+uv add toolpatch               # into an existing uv project
+```
+
+In this workspace (development install, editable):
+
+```bash
+uv sync --all-packages         # one shared root .venv, both modules included
+```
+
+## Quick use
+
+One completed (non-streaming) model reply in, clean result out:
+
+```python
+from toolpatch import extract_tool_calls
+
+reply = 'Sure. \u003ctool_call\u003e{"name": "list_dir", "arguments": {"path": "."}}\u003c/tool_call\u003e'
+
+ex = extract_tool_calls(reply, tool_names={"list_dir"})
+if ex:
+    ex.content      # "Sure."  (clean prose, leak gone)
+    ex.tool_calls   # [{'id': 'call_…', 'type': 'function',
+                    #   'function': {'name': 'list_dir', 'arguments': '{"path": "."}'}}]
+```
+
+That is the whole loop for a completed reply. For streaming replies use
+`StreamHealer.feed()`/`.finish()`, and for outbound request sanitization use
+`normalize_chat_request` — all three below in [API](#api).
+
+## What it repairs
+
+When a model's chat template is missing or mismatched, some local servers
+emit the tool call as **text inside** `choices[*].message.content` instead
+of the structured `tool_calls` field. The library recognizes three leaked
+payload shapes (all merged into one result, in document order):
+
+1. **Hermes-style XML function tags** — an XML tag block wrapping a JSON
+   payload, the form Qwen and several other open models emit when they leak.
+2. **Fenced JSON** — ```json … ``` code blocks.
+3. **Bare JSON object** — a raw `{"name": …, "arguments": …}` embedded in
+   prose (the extractor checks the keys and heals the payload before
+   accepting it).
+
+Plus the supporting repairs:
+
+- **JSON healing** — trailing commas, raw newlines inside string literals,
+  smart quotes, code fences.
+- **History normalization** — merges assistant-turn shapes that
+  llama-server-style chat templates reject with 4xx (leaked-JSON "twins" of
+  repaired tool calls, adjacent assistant turns).
+- **Streaming** — `StreamHealer` does the same work on SSE content deltas
+  with bounded buffering (time-to-first-token is preserved). It is an
+  in-flight accumulator, not "buffer until the first complete match": clean
+  prose is released as soon as no leak-shape opener is in flight, with a
+  10-char suffix window so an opener straddling two deltas is re-judged
+  whole. Reasoning-tag-wrapped leaks (`gpt-think … <tool_call>` mid-buffer) are held
+  by a mid-buffer opener check, not just tail rules — the Qwen2.5 mlx
+  pattern that motivated it. Once a leak resolves, the healer switches to
+  *collecting* mode and defers promotion to stream end (bounded by a
+  4×`max_buffer` cap), so rapid-succession or parallel blocks — mixed
+  families included — all land in **one** `tool_calls` frame instead of
+  promoting the first call and suppressing the rest. Whitespace between two
+  leaked calls survives as real inter-call prose; only padding outside the
+  leak envelope is stripped.
+
+## API
+
+The public surface is deliberately small — three callables plus the stream
+helper and one dataclass, all re-exported from `toolpatch`:
+
+```python
+from toolpatch import extract_tool_calls, heal_json, StreamHealer, normalize_chat_request
+from toolpatch import ExtractedToolCalls  # dataclass: content, tool_calls, source
+```
+
+### `extract_tool_calls(content, tool_names=None,
+                        allow_unterminated_hermes=False)`
+
+`-> ExtractedToolCalls | None`
+
+Runs all three detectors over the text and merges every distinct call into
+one result, in document order; overlapping/nested matches dedupe to the
+outer family. Returns `None` for clean prose (passthrough); on a hit
+returns the cleaned content plus the structured tool calls. Pass
+`tool_names={...}` to only accept calls whose `name` is in the set
+(anything else stays prose). `allow_unterminated_hermes=True` is the
+streaming path's opt-in: it tolerates one still-forming Hermes block at the
+end of the text.
+
+`ExtractedToolCalls.source` names the matching family (`hermes` / `fenced`
+/ `raw_json`) or `mixed` when families differ across the calls.
+
+```python
+ex = extract_tool_calls('Sure. ' + hermes_block)
+ex.content                                  # 'Sure.' (leak text removed)
+ex.source                                  # 'hermes'
+ex.tool_calls[0]['function']['name']        # 'list_dir'
+```
+
+### `heal_json(text) -> str | None`
+
+Repairs malformed JSON from a model payload: strips code fences, un-smarts
+quotes, drops trailing commas, escapes raw newlines that broke a string
+literal. Returns the repaired JSON string, or `None` when unrecoverable.
+
+### `StreamHealer(max_buffer=512, tool_names=None)`
+
+Streaming variant, for SSE content deltas. Feed it partial text as it
+arrives; it buffers only while a tool-call leak might be forming (bounded by
+`max_buffer`) and yields plain-dict events. Boundary safety is built in:
+fragments are re-judged over the whole buffer each delta, with the last
+`max(11, 8)−1 = 10` trailing chars held back (the 11-char `<tool_call>` opener
+vs the 8-char `{"name"` probe, minus one), so a leak opener split across
+two deltas is seen whole once the next delta completes it. A release slice
+ containing a leak shape is re-held instead of emitted — the raw leak text
+is never forwarded, so clients can't render duplicate content. After the
+first call resolves, promotion defers to stream end (4×`max_buffer` cap so
+an endless trailing ramble still promotes): all subsequent calls land in
+the one frame; residual prose between them comes out in the same batch,
+before the `tool_calls` event.
+
+```python
+healer = StreamHealer()
+
+for delta in sse_content_deltas:
+    for event in healer.feed(delta):
+        match event["type"]:
+            case "content":
+                pass  # safe to forward to the client as-is
+            case "tool_calls":
+                ...   # event["stop"] is True — one frame per turn;
+                      # contains ALL calls seen (file, webfetch, …)
+
+for event in healer.finish():   # flush at stream end
+    ...
+```
+
+Event shapes: `{"type": "content", "text": str}` and
+`{"type": "tool_calls", "tool_calls": [...], "stop": True}`.
+
+### `normalize_chat_request(payload, backend="gguf") -> (dict, list[str])`
+
+Sanitizes an outbound chat-completion request without mutating the input
+(deep-copies). `backend="mlx"` additionally strips `tool_choice`
+(unsupported by `mlx_lm.server`). Returns the sanitized payload plus
+operator notes describing each fix. History fixes: leaked tool-call text
+turns merged into the following `tool_calls` turn, adjacent assistant turns
+merged or deduped — the shapes llama-server templates reject with `400`s.
+
+```python
+clean, notes = normalize_chat_request(payload, backend="gguf")
+if notes:
+    log.info("sanitized request", extra={"notes": notes})
+```
+
+## Embedding
+
+Concretely, other Python consumers that can embed this today:
+
+- **Open WebUI Pipelines** (Python) — run `extract_tool_calls` on the model
+  response before it reaches the client.
+- **LiteLLM custom callbacks** (Python) — hook extraction into the response
+  path.
+- **FastAPI / FastMCP / custom servers** fronting a local model — call
+  `normalize_chat_request` before forwarding upstream.
+- **Plain OpenAI-SDK scripts** — wrap every `chat.completions.create` with
+  `normalize_chat_request` on send and `extract_tool_calls` on receive.
+
+Non-Python tools (Ollama, llama-server's C++ internals, llama-swap, etc.)
+cannot import a Python library — for those, the recommended integration is
+**proxytool** in front of the backend over HTTP, which applies this same
+repair logic.
+
+## Relationship to the app
+
+One-way dependency:
+
+- `apps/proxytool` (proxytool) imports this library in-process.
+- This library never imports the app, never opens sockets, and its test
+  suite is fully offline (`packages/toolpatch/tests`) — it does not need
+  the proxy or a model to run.
+
+Real-world misformattings captured from live models are kept as regression
+fixture pairs — see `tests/regression/fixtures/README.md`.
+
+## Logging
+
+The library is **silent by default** — it logs through the stdlib `logging`
+system (logger names `toolpatch.extractor`, `toolpatch.stream`, …) but never
+configures handlers or levels itself; a library shouldn't. In a bare script
+nothing is emitted. To turn repair diagnostics on:
+
+```python
+from toolpatch import configure_logger
+
+configure_logger("toolpatch", "debug")   # or "info"/"warning"/"error"
+```
+
+(Host apps like proxytool own this: proxytool's `--log-level debug` also
+raises the library's logger — one knob for everything.)
+
+## Status
+
+`v2.0.0`. Depends only on `pydantic`
+(a candidate for stdlib-only before publishing).

toolpatch-0.4.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+toolpatch/__init__.py,sha256=UbUXCpL78PWFuPB6h9nXklEqqavFdLCCkQ1BMmtSsys,1626
+toolpatch/extractor.py,sha256=SppSIz8PLP2lwgFErB2-6B0VHGjoHz_nK-Bu62fkNQc,8436
+toolpatch/fixer.py,sha256=dLqhP2QdMd1Sf6E9dcQzxhvSRZB8at8MAwSqztJIrhs,1740
+toolpatch/logging.py,sha256=RbK68OVxEJmG4IK-TBy5nLK0EfkpF50jZysSigQjo7U,3909
+toolpatch/request.py,sha256=KtQZGUqdJ0FLB1BbhMYhiWL5X3f7zl8Vf-EWGwOmU9g,8110
+toolpatch/schemas.py,sha256=wMTkwcttzYGeetw3wzPmlGzPcFidxaa_cu37KX1vldY,1203
+toolpatch/stream.py,sha256=B0Wtg-g9C7fL6rJT6qkpO7dv2RpeDZkTAunAA1PPKzw,10005
+toolpatch-0.4.0.dist-info/WHEEL,sha256=si2p8vW03RnhyBnhWl4tj-ObJa78SpE-c9HsEtblGR0,80
+toolpatch-0.4.0.dist-info/METADATA,sha256=22xHJnV5u6uAeK5vYLqYINYRIyuWNWu1xb7J4toaGiY,10698
+toolpatch-0.4.0.dist-info/RECORD,,

toolpatch-0.4.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.13.5
+Root-Is-Purelib: true
+Tag: py3-none-any