kolega-code 0.26.6__tar.gz → 0.26.7__tar.gz

{kolega_code-0.26.6 → kolega_code-0.26.7}/CHANGELOG.md

@@ -6,6 +6,17 @@ This project uses GitHub Releases for detailed generated release notes. This fil
 
 ## Unreleased
 
+## 0.26.7 - 2026-08-03
+
+### Fixed
+
+- Local token counting no longer crashes on content that contains a tokenizer
+  special-token literal (for example `<|endoftext|>` or `<|im_start|>`, common in
+  ML/tokenizer files and chat-format prompts); those strings are now counted as
+  ordinary text instead of aborting the run.
+- The Private Project Memory screen no longer crashes with an internal
+  "NoMatches" error when it is closed while still loading.
+
 ## 0.26.6 - 2026-08-03
 
 ### Changed

{kolega_code-0.26.6 → kolega_code-0.26.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kolega-code
-Version: 0.26.6
+Version: 0.26.7
 Summary: Multi-agent coding in the terminal
 Project-URL: Homepage, https://kolega-ai.github.io/kolega-code/
 Project-URL: Documentation, https://kolega-ai.github.io/kolega-code/

{kolega_code-0.26.6 → kolega_code-0.26.7}/kolega_code/__init__.py

@@ -151,4 +151,4 @@ __all__ = [
     "parse_git_status_output",
 ]
 
-__version__ = "0.26.6"
+__version__ = "0.26.7"

{kolega_code-0.26.6 → kolega_code-0.26.7}/kolega_code/agent/__init__.py

@@ -51,4 +51,4 @@ __all__ = [
     "ToolExtension",
 ]
 
-__version__ = "0.26.6"
+__version__ = "0.26.7"

{kolega_code-0.26.6 → kolega_code-0.26.7}/kolega_code/cli/tui/memory_screen.py

@@ -115,6 +115,22 @@ class MemoryScreen(ModalScreen[None]):
         if self._owner._memory_screen is self:
             self._owner._memory_screen = None
 
+    def _work_current(self, generation: int) -> bool:
+        """Whether a worker may still render — not superseded, and still attached.
+
+        A worker finishes its threaded load asynchronously, by which point the
+        screen may have been replaced by a newer refresh or torn down. Rendering
+        then calls ``query_one`` on detached widgets and raises ``NoMatches``
+        *inside the worker* (surfacing as ``WorkerFailed``). ``_work_generation``
+        covers supersede; ``is_attached`` additionally closes the teardown race
+        where child widgets detach before ``on_unmount`` bumps the generation.
+        (``is_mounted`` is the wrong signal here: it stays True after the screen
+        is popped, while ``is_attached`` flips exactly when ``query_one`` starts
+        raising.) The render after this check is synchronous, so ``is_attached``
+        cannot flip mid-render.
+        """
+        return generation == self._work_generation and self.is_attached
+
     # ---- loading -------------------------------------------------------------
 
     def _start_refresh(self, *, select_reference: str | None = None, from_disk: bool = False) -> None:
@@ -148,17 +164,17 @@ class MemoryScreen(ModalScreen[None]):
                     allow_disabled=True,
                 )
         except Exception as error:
-            if generation == self._work_generation:
+            if self._work_current(generation):
                 self._status = None
                 self._all_entries = []
                 self._entries = []
                 self._render_error(str(error))
             return
         finally:
-            if generation == self._work_generation:
+            if self._work_current(generation):
                 self._set_busy(False)
 
-        if generation != self._work_generation:
+        if not self._work_current(generation):
             return
         self._status = status
         self._all_entries = entries
@@ -189,13 +205,13 @@ class MemoryScreen(ModalScreen[None]):
                 allow_disabled=True,
             )
         except Exception as error:
-            if generation == self._work_generation:
+            if self._work_current(generation):
                 self._render_error(str(error))
             return
         finally:
-            if generation == self._work_generation:
+            if self._work_current(generation):
                 self._set_busy(False)
-        if generation != self._work_generation:
+        if not self._work_current(generation):
             return
         self._loaded_entry = entry
         self._render_entry(entry)
@@ -448,7 +464,7 @@ class MemoryScreen(ModalScreen[None]):
         try:
             context = await asyncio.to_thread(self._manager.prompt_context)
         except MemoryUnavailableError as error:
-            if generation != self._work_generation:
+            if not self._work_current(generation):
                 return
             self._set_busy(False)
             self._enter_agent_view(
@@ -458,11 +474,11 @@ class MemoryScreen(ModalScreen[None]):
             )
             return
         except Exception as error:
-            if generation == self._work_generation:
+            if self._work_current(generation):
                 self._set_busy(False)
                 self.query_one("#memory_notice", Static).update(str(error))
             return
-        if generation != self._work_generation:
+        if not self._work_current(generation):
             return
         self._set_busy(False)
         truncated = " · truncated" if context.truncated else ""

kolega_code-0.26.7/kolega_code/llm/providers/_token_encoding.py

@@ -0,0 +1,44 @@
+"""Shared tiktoken helpers for local token *counting*.
+
+Local token counting must never crash on content that merely contains a tiktoken
+special-token literal — e.g. ``<|endoftext|>`` in a GPT-2 / tokenizer file, or
+``<|im_start|>`` in a chat-format prompt. By default ``Encoding.encode`` uses
+``disallowed_special="all"`` and raises a ``ValueError`` the moment it sees such
+a string in ordinary text. That is the right default for *building a request*
+(you don't want to smuggle real control tokens into a prompt), but here we only
+want an approximate size for context management, so those strings should be
+counted as normal text. Passing ``disallowed_special=()`` disables the check —
+exactly what tiktoken's own error message recommends.
+"""
+
+from typing import List
+
+import tiktoken
+
+
+class CountingEncoding:
+    """A tiktoken ``Encoding`` wrapper whose ``encode`` never raises on special
+    tokens.
+
+    It forwards to the wrapped encoding with ``disallowed_special=()`` so
+    special-token literals in the text are counted as ordinary text instead of
+    raising. Only ``encode`` is exposed because counting is the only thing this
+    is used for.
+    """
+
+    __slots__ = ("_encoding",)
+
+    def __init__(self, encoding: "tiktoken.Encoding") -> None:
+        self._encoding = encoding
+
+    def encode(self, text: str) -> List[int]:
+        return self._encoding.encode(text, disallowed_special=())
+
+
+def get_counting_encoding(name: str) -> CountingEncoding:
+    """Return a :class:`CountingEncoding` for the named tiktoken encoding.
+
+    ``tiktoken.get_encoding`` caches the underlying BPE table in its own registry,
+    so repeated calls are cheap; the wrapper is a trivial object around it.
+    """
+    return CountingEncoding(tiktoken.get_encoding(name))

{kolega_code-0.26.6 → kolega_code-0.26.7}/kolega_code/llm/providers/anthropic.py

@@ -5,13 +5,13 @@ import threading
 from typing import Any, AsyncContextManager, Dict, List, Optional, cast
 from weakref import WeakKeyDictionary
 
-import tiktoken
 from anthropic import AsyncAnthropic
 
 from ..models import ContentBlock, Message, MessageChunk, MessageHistory, ToolDefinition
 from ..specs import build_thinking_request_params, get_model_specs
 from ..timeouts import streaming_timeout
 from ..tool_execution_ids import ToolExecutionIdRegistry
+from ._token_encoding import get_counting_encoding
 from .base import BaseLLMProvider
 from .models import GenerationParams, TokenCount
 
@@ -241,7 +241,9 @@ class AnthropicProvider(BaseLLMProvider):
         Returns:
             TokenCount object with estimated input token count
         """
-        encoding = tiktoken.get_encoding("p50k_base")
+        # CountingEncoding so special-token literals (e.g. <|endoftext|>) in the
+        # content are counted as text instead of raising — see _token_encoding.
+        encoding = get_counting_encoding("p50k_base")
         self._ensure_local_token_memos()
         tools = tools or []
 

{kolega_code-0.26.6 → kolega_code-0.26.7}/kolega_code/llm/providers/openai.py

@@ -8,7 +8,6 @@ import os
 import threading
 import uuid
 
-import tiktoken
 from openai import AsyncOpenAI
 
 from ..models import (
@@ -26,6 +25,7 @@ from ..models import (
 from ..specs import MODEL_SPECS, build_thinking_request_params
 from ..timeouts import streaming_timeout
 from ..tool_execution_ids import ToolExecutionIdRegistry
+from ._token_encoding import get_counting_encoding
 from .base import BaseLLMProvider
 from .models import GenerationParams, TokenCount
 
@@ -35,6 +35,10 @@ from .models import GenerationParams, TokenCount
 # de-facto encoding for every OpenAI-compatible provider (DeepSeek, xAI, Fireworks,
 # ... have no native tiktoken table); the count is an estimate for context
 # management, not billing.
+#
+# CountingEncoding treats special-token literals (e.g. ``<|endoftext|>`` in a
+# tokenizer file) as ordinary text instead of raising, so counting never crashes
+# on such content — see _token_encoding for the rationale.
 _ENCODING_NAME = "cl100k_base"
 _encoding = None
 
@@ -46,7 +50,7 @@ OPENROUTER_PROVIDER = "openrouter"
 def _get_encoding():
     global _encoding
     if _encoding is None:
-        _encoding = tiktoken.get_encoding(_ENCODING_NAME)
+        _encoding = get_counting_encoding(_ENCODING_NAME)
     return _encoding
 
 

{kolega_code-0.26.6 → kolega_code-0.26.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "kolega-code"
-version = "0.26.6"
+version = "0.26.7"
 description = "Multi-agent coding in the terminal"
 readme = "README.md"
 license = {file = "LICENSE"}