coderouter-cli 2.5.3__tar.gz → 2.5.4__tar.gz

{coderouter_cli-2.5.3 → coderouter_cli-2.5.4}/CHANGELOG.md

@@ -6,6 +6,50 @@ versioning follows [SemVer](https://semver.org/).
 
 ---
 
+## [v2.5.4] — 2026-06-05 (Gemma `<0xNN>` byte-fallback repair filter)
+
+Patch release: a new opt-in output filter that repairs Japanese (and other
+multi-byte) text corrupted by the Ollama 0.30 / llama.cpp detokenizer change.
+
+### Added
+
+- **`repair_byte_fallback` output filter.** Ollama 0.30 unified its GGUF
+  runtime onto llama.cpp (`ollama/ollama#16031`). For gemma4 the detokenizer
+  changed, and multi-byte characters it cannot assemble now leak as
+  llama.cpp's byte-fallback notation `<0xNN>`:
+  - full-width space `　` → `<0xE3><0x80><0x80>`
+  - rare kanji `躙` → `<0xE8><0xBA><0x99>`
+
+  These corrupt Japanese prose **and** tool-call JSON argument strings routed
+  through CodeRouter (a stray `<0xNN>` inside an argument breaks JSON parsing).
+  The new filter reassembles consecutive `<0xNN>` runs back into UTF-8.
+
+  - **Opt-in** per provider: `output_filters: [repair_byte_fallback]`
+    (disabled by default). Place it **before** `tool_repair` / tool-call XML
+    strip so byte-fallback inside tool-call arguments is restored before JSON
+    extraction.
+  - **Streaming-safe**: handles chunk boundaries inside a single token
+    (`<0x` | `E3>`) and inside a multi-byte run (`<0xE3>` | `<0x80><0x80>`).
+    A pending byte run is only flushed once it has definitively ended.
+  - **Lossless**: bytes that cannot form valid UTF-8 are re-emitted verbatim
+    as `<0xNN>`, so output is never made worse than llama.cpp already left it.
+  - **No new runtime dependencies** (stdlib `re` only).
+
+  Verified with 22 new unit tests (61 filter tests total pass on py3.12),
+  ruff clean, a 20,000-iteration streaming chunk-boundary fuzz (0 mismatches),
+  and a Japanese/emoji round-trip.
+
+  Ref: <https://note.com/akb428/n/n737e786f32ce>
+
+### Known mitigations (documented)
+
+- For gemma4, staying on / downgrading to Ollama 0.24 is the most reliable
+  fix (the root cause is the 0.30 llama.cpp detokenizer swap).
+- A larger `num_ctx` increases the leak rate; consider not auto-raising it for
+  gemma4 on Ollama 0.30.
+
+---
+
 ## [v2.5.2] — 2026-05-22 (Backend-aware Launcher suggestions + backend install guide)
 
 Patch release: a Launcher bug fix and documentation improvements.

{coderouter_cli-2.5.3 → coderouter_cli-2.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: coderouter-cli
-Version: 2.5.3
+Version: 2.5.4
 Summary: Local-first, free-first, fallback-built-in LLM router. Claude Code / OpenAI compatible.
 Project-URL: Homepage, https://github.com/zephel01/CodeRouter
 Project-URL: Repository, https://github.com/zephel01/CodeRouter

{coderouter_cli-2.5.3 → coderouter_cli-2.5.4}/coderouter/output_filters.py

@@ -43,6 +43,7 @@ Reference: plan.md §10.2 "出力クリーニング" / docs/retrospectives/v0.7.
 
 from __future__ import annotations
 
+import re
 from typing import Protocol
 
 __all__ = [
@@ -50,6 +51,7 @@ __all__ = [
     "KNOWN_FILTERS",
     "OutputFilter",
     "OutputFilterChain",
+    "RepairByteFallbackFilter",
     "StripStopMarkersFilter",
     "StripThinkingFilter",
     "StripToolCallXmlFilter",
@@ -382,6 +384,151 @@ class StripToolCallXmlFilter:
         return "".join(out_parts)
 
 
+# ---------------------------------------------------------------------------
+# repair_byte_fallback (v2.x)
+# ---------------------------------------------------------------------------
+
+
+# A complete byte-fallback token: ``<0x`` + exactly two hex digits + ``>``.
+_BYTE_RE = re.compile(r"<0x([0-9A-Fa-f]{2})>")
+
+# The whole remaining buffer is a *proper prefix* of some ``<0xHH>`` token,
+# i.e. it could still complete (and continue a run) on the next feed:
+# ``<`` / ``<0`` / ``<0x`` / ``<0xH`` / ``<0xHH`` (closing ``>`` not yet seen).
+_PREFIX_RE = re.compile(r"<(0(x[0-9A-Fa-f]{0,2})?)?")
+
+_BYTE_TOKEN_START = "<0x"
+
+
+def _decode_byte_run(buf: bytes) -> str:
+    """Decode a run of fallback bytes to text, losslessly.
+
+    Decodes the maximal valid UTF-8 prefix; any byte that cannot start or
+    continue a valid sequence is re-emitted as its ``<0xHH>`` token and
+    decoding resumes after it. So ``b"\\xe3\\x80\\x80"`` -> ``"　"`` while a
+    stray ``b"\\xff"`` round-trips to ``"<0xFF>"`` — we never make the stream
+    worse than llama.cpp already did.
+    """
+    parts: list[str] = []
+    i = 0
+    n = len(buf)
+    while i < n:
+        try:
+            parts.append(buf[i:].decode("utf-8"))
+            break
+        except UnicodeDecodeError as exc:
+            good_end = i + exc.start
+            if good_end > i:
+                parts.append(buf[i:good_end].decode("utf-8"))
+            parts.append(f"<0x{buf[good_end]:02X}>")
+            i = good_end + 1
+    return "".join(parts)
+
+
+class RepairByteFallbackFilter:
+    """Reassemble llama.cpp ``<0xNN>`` byte-fallback leaks into UTF-8 text.
+
+    Ollama 0.30 unified its GGUF runtime onto llama.cpp
+    (``ollama/ollama#16031``). For gemma4 the detokenizer changed, and
+    multi-byte characters it cannot assemble now leak as llama.cpp's
+    byte-fallback notation::
+
+        full-width space ``　``  -> ``<0xE3><0x80><0x80>``
+        rare kanji      ``躙``  -> ``<0xE8><0xBA><0x99>``
+
+    These corrupt Japanese prose AND tool-call JSON arguments (a stray
+    ``<0xNN>`` inside an argument string breaks JSON parsing). This filter
+    reassembles runs of consecutive ``<0xNN>`` tokens back into UTF-8.
+
+    Stateful across ``feed`` calls so a token split across SSE deltas
+    (``<0x`` | ``E3>``) and a multi-byte run split across deltas
+    (``<0xE3>`` | ``<0x80><0x80>``) both reassemble correctly. A pending byte
+    run is only flushed once we are certain it has ended (confirmed normal
+    text follows, or ``eof``) — never at a bare chunk boundary, where the run
+    might continue in the next delta. Bytes that cannot form valid UTF-8 are
+    re-emitted verbatim (lossless).
+
+    ``modified`` flips True the first time any ``<0xNN>`` token is consumed —
+    the adapter uses it to gate the log-once "output-filter-applied" line.
+
+    Ordering note: place this BEFORE ``tool_repair`` / the tool-call XML
+    strip so byte-fallback inside tool-call argument strings is restored
+    before JSON extraction.
+    """
+
+    name = "repair_byte_fallback"
+
+    def __init__(self) -> None:
+        """Initialize per-request buffer, pending byte run and state."""
+        self.modified: bool = False
+        self._buffer: str = ""
+        self._pending = bytearray()
+
+    def _flush_pending(self, out: list[str]) -> None:
+        """Decode and emit the accumulated byte run, then clear it."""
+        if self._pending:
+            out.append(_decode_byte_run(bytes(self._pending)))
+            self._pending.clear()
+
+    def feed(self, text: str, *, eof: bool = False) -> str:
+        """Consume ``text``; return the portion safe to emit now."""
+        self._buffer += text
+        out: list[str] = []
+
+        while self._buffer:
+            m = _BYTE_RE.match(self._buffer)
+            if m is not None:
+                # Complete byte token at position 0 — extend the run.
+                self._pending.append(int(m.group(1), 16))
+                self._buffer = self._buffer[m.end() :]
+                self.modified = True
+                continue
+
+            idx = self._buffer.find(_BYTE_TOKEN_START)
+            if idx == -1:
+                # No complete/started token in the buffer. Hold a trailing
+                # partial of ``<0x`` (it may complete — and CONTINUE the run —
+                # on the next feed); treat anything before it as confirmed
+                # normal text that ends the run.
+                hold = (
+                    0 if eof else _max_suffix_overlap(self._buffer, _BYTE_TOKEN_START)
+                )
+                safe = self._buffer[:-hold] if hold else self._buffer
+                if safe:
+                    self._flush_pending(out)
+                    out.append(safe)
+                    self._buffer = self._buffer[len(safe) :]
+                # else: whole buffer is a token-start prefix; keep pending
+                # (the run might continue) and wait for more input.
+                break
+
+            if idx > 0:
+                # Normal text precedes the next token start — run ended.
+                self._flush_pending(out)
+                out.append(self._buffer[:idx])
+                self._buffer = self._buffer[idx:]
+                continue
+
+            # idx == 0: buffer starts with ``<0x`` but is not a complete token.
+            if not eof and _PREFIX_RE.fullmatch(self._buffer):
+                # Could still complete next feed — hold token AND pending run.
+                break
+
+            # Malformed ``<0x..`` (non-hex, or stuck at eof). The ``<`` is
+            # ordinary text; the run (if any) has ended.
+            self._flush_pending(out)
+            out.append("<")
+            self._buffer = self._buffer[1:]
+
+        if eof:
+            self._flush_pending(out)
+            if self._buffer:
+                out.append(self._buffer)
+            self._buffer = ""
+
+        return "".join(out)
+
+
 # ---------------------------------------------------------------------------
 # Registry + chain
 # ---------------------------------------------------------------------------
@@ -391,6 +538,7 @@ KNOWN_FILTERS: dict[str, type[OutputFilter]] = {
     StripThinkingFilter.name: StripThinkingFilter,
     StripStopMarkersFilter.name: StripStopMarkersFilter,
     StripToolCallXmlFilter.name: StripToolCallXmlFilter,
+    RepairByteFallbackFilter.name: RepairByteFallbackFilter,
 }
 """Registry of string-name → filter class.
 

{coderouter_cli-2.5.3 → coderouter_cli-2.5.4}/pyproject.toml

@@ -11,7 +11,7 @@
 # in plan.md §11.B; once granted, this name will become an alias and
 # `coderouter` will become the canonical distribution name.
 name = "coderouter-cli"
-version = "2.5.3"
+version = "2.5.4"
 description = "Local-first, free-first, fallback-built-in LLM router. Claude Code / OpenAI compatible."
 readme = "README.md"
 requires-python = ">=3.12"

{coderouter_cli-2.5.3 → coderouter_cli-2.5.4}/tests/test_output_filters.py

@@ -61,7 +61,12 @@ def test_validate_rejects_when_mixing_known_and_unknown() -> None:
 
 def test_registry_matches_expected_set() -> None:
     """Regression: if a filter is added, this test reminds us to doc it."""
-    assert set(KNOWN_FILTERS) == {"strip_thinking", "strip_stop_markers", "strip_tool_call_xml"}
+    assert set(KNOWN_FILTERS) == {
+        "strip_thinking",
+        "strip_stop_markers",
+        "strip_tool_call_xml",
+        "repair_byte_fallback",
+    }
 
 
 # ======================================================================

coderouter_cli-2.5.4/tests/test_repair_byte_fallback.py

@@ -0,0 +1,175 @@
+"""Unit tests for ``repair_byte_fallback`` — the Gemma/llama.cpp
+``<0xNN>`` byte-fallback repair filter.
+
+Mirrors the streaming-boundary style of ``test_output_filters.py``: we
+exercise (1) whole-string repair, (2) chunk boundaries inside a single
+token, (3) chunk boundaries inside a multi-byte run, (4) passthrough of
+ordinary text, and (5) lossless handling of malformed/incomplete tokens.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from coderouter.output_filters import (
+    KNOWN_FILTERS,
+    RepairByteFallbackFilter,
+    apply_output_filters,
+)
+
+
+def _repair(text: str) -> tuple[str, bool]:
+    """One-shot repair, returning ``(out, modified)``."""
+    f = RepairByteFallbackFilter()
+    out = f.feed(text, eof=True)
+    return out, f.modified
+
+
+def _stream(chunks: list[str]) -> str:
+    """Feed chunks one at a time, flush at eof, return concatenated output."""
+    f = RepairByteFallbackFilter()
+    out = []
+    for c in chunks:
+        out.append(f.feed(c))
+    out.append(f.feed("", eof=True))
+    return "".join(out)
+
+
+# ----------------------------------------------------------------------
+# registry wiring
+# ----------------------------------------------------------------------
+
+
+def test_registered_in_known_filters() -> None:
+    assert KNOWN_FILTERS["repair_byte_fallback"] is RepairByteFallbackFilter
+
+
+def test_usable_through_chain() -> None:
+    out, applied = apply_output_filters(
+        ["repair_byte_fallback"], "残酷な<0xE3><0x80><0x80>蹂"
+    )
+    assert out == "残酷な　蹂"
+    assert applied == ["repair_byte_fallback"]
+
+
+# ----------------------------------------------------------------------
+# 1. whole-string repair
+# ----------------------------------------------------------------------
+
+
+def test_full_width_space_single_shot() -> None:
+    out, mod = _repair("残酷な<0xE3><0x80><0x80>蹂")
+    assert out == "残酷な　蹂"
+    assert mod is True
+
+
+def test_rare_kanji_single_shot() -> None:
+    # 躙 = U+8E99 = E8 BA 99
+    out, mod = _repair("残酷な蹂<0xE8><0xBA><0x99>だった")
+    assert out == "残酷な蹂躙だった"
+    assert mod is True
+
+
+def test_mixed_multiple_runs() -> None:
+    out, _ = _repair("A<0xE3><0x80><0x80>B<0xE8><0xBA><0x99>C")
+    assert out == "A　B躙C"
+
+
+# ----------------------------------------------------------------------
+# 2. chunk boundary inside a single token
+# ----------------------------------------------------------------------
+
+
+def test_split_inside_token() -> None:
+    assert _stream(["残酷な<0x", "E3><0x80><0x80>蹂"]) == "残酷な　蹂"
+
+
+def test_split_every_char() -> None:
+    s = "x<0xE3><0x80><0x80>y"
+    assert _stream(list(s)) == "x　y"
+
+
+# ----------------------------------------------------------------------
+# 3. chunk boundary inside a multi-byte run
+# ----------------------------------------------------------------------
+
+
+def test_split_between_run_tokens() -> None:
+    assert _stream(["<0xE3>", "<0x80><0x80>"]) == "　"
+
+
+def test_split_run_three_ways() -> None:
+    assert _stream(["pre<0xE8>", "<0xBA>", "<0x99>post"]) == "pre躙post"
+
+
+# ----------------------------------------------------------------------
+# 4. ordinary text passthrough (must not corrupt existing content)
+# ----------------------------------------------------------------------
+
+
+def test_plain_text_untouched() -> None:
+    out, mod = _repair("def foo(): return 1  # 普通の日本語")
+    assert out == "def foo(): return 1  # 普通の日本語"
+    assert mod is False
+
+
+def test_non_byte_angle_tags_untouched() -> None:
+    s = "<think>hmm</think><tool_call>{}</tool_call>"
+    out, mod = _repair(s)
+    assert out == s
+    assert mod is False
+
+
+def test_literal_less_than_zero_passes() -> None:
+    out, _ = _repair("if x <0 and y >1: pass")
+    assert out == "if x <0 and y >1: pass"
+
+
+# ----------------------------------------------------------------------
+# 5. lossless handling of malformed / incomplete tokens
+# ----------------------------------------------------------------------
+
+
+def test_malformed_non_hex_kept_literal() -> None:
+    out, mod = _repair("a<0xZZ>b")
+    assert out == "a<0xZZ>b"
+    assert mod is False
+
+
+def test_incomplete_token_at_eof_kept_literal() -> None:
+    out, _ = _repair("tail<0xE3")
+    assert out == "tail<0xE3"
+
+
+def test_lone_invalid_byte_round_trips() -> None:
+    out, mod = _repair("x<0xFF>y")
+    assert out == "x<0xFF>y"
+    assert mod is True  # a token WAS consumed, even though the byte was undecodable
+
+
+def test_valid_prefix_then_invalid_byte() -> None:
+    out, _ = _repair("<0xE3><0x80><0xFF>")
+    assert out == "<0xE3><0x80><0xFF>"
+
+
+# ----------------------------------------------------------------------
+# invariants
+# ----------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    "s",
+    ["", "hello world", "マルチバイト日本語テキスト", "<<<>>>", "a < b and c > d"],
+)
+def test_no_byte_tokens_means_byte_identical(s: str) -> None:
+    out, mod = _repair(s)
+    assert out == s
+    assert mod is False
+
+
+def test_modified_flag_semantics() -> None:
+    f = RepairByteFallbackFilter()
+    f.feed("plain", eof=False)
+    assert f.modified is False
+    f.feed("<0xE3><0x80><0x80>", eof=True)
+    assert f.modified is True