kon-coding-agent 0.3.3__tar.gz → 0.3.4__tar.gz

{kon_coding_agent-0.3.3 → kon_coding_agent-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kon-coding-agent
-Version: 0.3.3
+Version: 0.3.4
 Summary: Minimal coding agent
 License-File: LICENSE
 Requires-Python: >=3.12

kon_coding_agent-0.3.4/docs/local-models.md

@@ -0,0 +1,80 @@
+# Local Models
+
+This document provides detailed information about running and configuring local models with Kon.
+
+## Tested Models
+
+> Tested on llama server build b8740
+
+| Model | Quantization | Context Length | TPS | System Specs |
+| ----- | -------------- | -------------- | --- | ------------ |
+| `zai-org/glm-4.7-flash` | Q4_K_M | 65,536 | N/A | i7-14700F × 28, 64GB RAM, 24GB VRAM (RTX 3090) |
+| `unsloth/Qwen3.5-27B-GGUF` | Q4_K_M | 32,768 | ~30 | i7-14700F × 28, 64GB RAM, 24GB VRAM (RTX 3090) |
+| `unsloth/gemma-4-26B-A4B-it-GGUF` | UD-Q4_K_M | 32,768 | ~100 | i7-14700F × 28, 64GB RAM, 24GB VRAM (RTX 3090) |
+
+Run Qwen3.5 27B on an RTX 3090 with a 32k context window using llama-server:
+
+```bash
+/path-to-llama-server/llama-server \
+  --model /path-to-model/Qwen3.5-27B-Q4_K_M.gguf \
+  --port 5000 \
+  --ctx-size 32768 \
+  --gpu-layers all \
+  --threads 8 \
+  --threads-batch 8 \
+  --batch-size 1024 \
+  --ubatch-size 512 \
+  --flash-attn on
+```
+
+On this machine, that setup generates at roughly 30 tokens per second.
+
+Then start Kon for a one-off local session:
+
+```bash
+kon --model unsloth/Qwen3.5-27B-GGUF --provider openai \
+  --base-url http://localhost:5000/v1 \
+  --openai-compat-auth none
+```
+
+Run Gemma 4 26B A4B on the same machine using llama-server:
+
+```bash
+/path-to-llama-server/llama-server \
+  --model /path-to-model/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf \
+  --port 5000 \
+  --ctx-size 32768 \
+  --gpu-layers all \
+  --threads 8 \
+  --threads-batch 8 \
+  --batch-size 1024 \
+  --ubatch-size 512 \
+  --flash-attn on \
+  --temperature 1.5
+```
+
+Then start Kon against that local server:
+
+```bash
+kon --model unsloth/gemma-4-26B-A4B-it-GGUF --provider openai \
+  --base-url http://localhost:5000/v1 \
+  --openai-compat-auth none
+```
+
+To avoid passing provider, model, and auth flags every time you start Kon, you can define your local setup in `~/.kon/config.toml`. This also allows you to tune compaction to trigger at a specific point relative to your model's context window.
+
+If this is your default setup, put it in `~/.kon/config.toml` instead:
+
+```toml
+[llm]
+default_provider = "openai"
+default_model = "unsloth/gemma-4-26B-A4B-it-GGUF"
+default_base_url = "http://localhost:5000/v1"
+
+[llm.auth]
+openai_compat = "none" # or "auto"
+
+[compaction]
+# Set this close to your model's context size (e.g., 30000 for a 32k window)
+buffer_tokens = 27768 # 32768 - 5000 (safety margin)
+```

{kon_coding_agent-0.3.3 → kon_coding_agent-0.3.4}/pyproject.toml

@@ -14,7 +14,7 @@ default = true
 
 [project]
 name = "kon-coding-agent"
-version = "0.3.3"
+version = "0.3.4"
 description = "Minimal coding agent"
 readme = "README.md"
 requires-python = ">=3.12"

{kon_coding_agent-0.3.3 → kon_coding_agent-0.3.4}/src/kon/llm/providers/openai_completions.py

@@ -30,7 +30,7 @@ from ...core.types import (
     Usage,
     UserMessage,
 )
-from ..base import BaseProvider, LLMStream, ProviderConfig, resolve_api_key
+from ..base import BaseProvider, LLMStream, ProviderConfig, is_local_base_url, resolve_api_key
 from .openai_compat import supports_developer_role
 from .sanitize import sanitize_surrogates
 
@@ -41,12 +41,13 @@ class OpenAICompletionsCompat:
     supports_developer_role: bool = True
     supports_reasoning_effort: bool = True
     max_tokens_field: Literal["max_tokens", "max_completion_tokens"] = "max_completion_tokens"
-    thinking_format: Literal["openai", "zai", "qwen"] = "openai"
+    thinking_format: Literal["openai", "zai", "qwen", "llama_gemma"] = "openai"
 
 
-def _detect_compat(provider: str, base_url: str) -> OpenAICompletionsCompat:
+def _detect_compat(provider: str, base_url: str, model: str = "") -> OpenAICompletionsCompat:
     normalized_provider = provider.lower()
     normalized_base_url = base_url.lower()
+    normalized_model = model.lower()
     is_zai = (
         normalized_provider == "zai"
         or normalized_provider == "zhipu"
@@ -61,6 +62,13 @@ def _detect_compat(provider: str, base_url: str) -> OpenAICompletionsCompat:
             thinking_format="zai",
         )
 
+    if is_local_base_url(base_url) and "gemma" in normalized_model:
+        return OpenAICompletionsCompat(
+            supports_developer_role=supports_developer_role(provider, base_url),
+            supports_reasoning_effort=False,
+            thinking_format="llama_gemma",
+        )
+
     return OpenAICompletionsCompat(
         supports_developer_role=supports_developer_role(provider, base_url)
     )
@@ -92,7 +100,9 @@ class OpenAICompletionsProvider(BaseProvider):
                 'or configure llm.auth.openai_compat = "auto"/"none" for local endpoints.'
             )
         self._client = AsyncOpenAI(api_key=api_key, base_url=config.base_url)
-        self._compat = _detect_compat(config.provider or "", config.base_url or "")
+        self._compat = _detect_compat(
+            config.provider or "", config.base_url or "", config.model or ""
+        )
 
     async def _stream_impl(
         self,
@@ -138,7 +148,7 @@ class OpenAICompletionsProvider(BaseProvider):
         if compat.thinking_format == "zai":
             if thinking_level and thinking_level != "none":
                 extra_body["thinking"] = {"type": "enabled"}
-        elif compat.thinking_format == "qwen":
+        elif compat.thinking_format in {"qwen", "llama_gemma"}:
             extra_body["enable_thinking"] = bool(thinking_level and thinking_level != "none")
         elif (
             self.supports_reasoning_effort
@@ -238,11 +248,16 @@ class OpenAICompletionsProvider(BaseProvider):
 
         if system_prompt:
             role = "developer" if (compat and compat.supports_developer_role) else "system"
+            prompt_content = sanitize_surrogates(system_prompt)
+            if (
+                compat
+                and compat.thinking_format == "llama_gemma"
+                and self.config.thinking_level != "none"
+                and not prompt_content.startswith("<|think|>")
+            ):
+                prompt_content = "<|think|>" + prompt_content
             result.append(
-                cast(
-                    ChatCompletionMessageParam,
-                    {"role": role, "content": sanitize_surrogates(system_prompt)},
-                )
+                cast(ChatCompletionMessageParam, {"role": role, "content": prompt_content})
             )
 
         pending_images: list[ImageContent] = []

{kon_coding_agent-0.3.3 → kon_coding_agent-0.3.4}/src/kon/session.py

@@ -226,7 +226,7 @@ class Session:
             self._flushed = True
             self._persisted_entries_count = len(self._entries)
         else:
-            with open(self._session_file, "a") as f:
+            with open(self._session_file, "a", encoding="utf-8") as f:
                 f.write(entry.model_dump_json() + "\n")
             self._persisted_entries_count += 1
 
@@ -236,7 +236,7 @@ class Session:
 
         self._session_file.parent.mkdir(parents=True, exist_ok=True)
 
-        with open(self._session_file, "w") as f:
+        with open(self._session_file, "w", encoding="utf-8") as f:
             if self._header:
                 f.write(self._header.model_dump_json() + "\n")
             for entry in self._entries:
@@ -469,7 +469,7 @@ class Session:
         header: SessionHeader | None = None
         entries: list[SessionEntry] = []
 
-        with open(path) as f:
+        with open(path, encoding="utf-8") as f:
             for line in f:
                 line = line.strip()
                 if not line:
@@ -605,7 +605,7 @@ class Session:
         message_count = 0
         first_message = ""
 
-        with open(path) as f:
+        with open(path, encoding="utf-8") as f:
             for line in f:
                 line = line.strip()
                 if not line:

{kon_coding_agent-0.3.3 → kon_coding_agent-0.3.4}/src/kon/ui/app.py

@@ -94,7 +94,7 @@ _CHANGELOG_URL = "https://github.com/0xku/kon/blob/main/CHANGELOG.md"
 try:
     VERSION = version(_PYPI_PACKAGE_NAME)
 except PackageNotFoundError:
-    VERSION = "0.3.3"
+    VERSION = "0.3.4"
 
 _COPILOT_API_TYPES: frozenset[ApiType] = frozenset(
     {ApiType.GITHUB_COPILOT, ApiType.GITHUB_COPILOT_RESPONSES, ApiType.ANTHROPIC_COPILOT}

{kon_coding_agent-0.3.3 → kon_coding_agent-0.3.4}/tests/test_openai_compat.py

@@ -1,9 +1,11 @@
+from typing import Any, cast
+
 import pytest
 
 from kon.llm.base import ProviderConfig, is_local_base_url, resolve_api_key
 from kon.llm.providers.openai_codex_responses import OpenAICodexResponsesProvider
 from kon.llm.providers.openai_compat import supports_developer_role
-from kon.llm.providers.openai_completions import _detect_compat
+from kon.llm.providers.openai_completions import OpenAICompletionsProvider, _detect_compat
 from kon.llm.providers.openai_responses import OpenAIResponsesProvider
 
 
@@ -26,6 +28,111 @@ def test_detect_compat_disables_developer_role_for_local_api() -> None:
     assert compat.supports_reasoning_effort is True
 
 
+def test_detect_compat_uses_llama_gemma_for_local_gemma_models() -> None:
+    compat = _detect_compat(
+        "openai", "http://127.0.0.1:1234/v1", "unsloth/gemma-4-26B-A4B-it-GGUF"
+    )
+
+    assert compat.supports_developer_role is False
+    assert compat.supports_reasoning_effort is False
+    assert compat.thinking_format == "llama_gemma"
+
+
+def test_openai_completions_prefixes_think_token_for_local_gemma() -> None:
+    provider = OpenAICompletionsProvider(
+        ProviderConfig(
+            api_key="test-key",
+            base_url="http://127.0.0.1:1234/v1",
+            model="unsloth/gemma-4-26B-A4B-it-GGUF",
+            provider="openai",
+            thinking_level="medium",
+        )
+    )
+
+    messages = provider._convert_messages([], "You are helpful", provider._compat)
+
+    assert messages[0]["role"] == "system"
+    assert messages[0]["content"] == "<|think|>You are helpful"
+
+
+def test_openai_completions_does_not_prefix_think_token_when_thinking_disabled() -> None:
+    provider = OpenAICompletionsProvider(
+        ProviderConfig(
+            api_key="test-key",
+            base_url="http://127.0.0.1:1234/v1",
+            model="unsloth/gemma-4-26B-A4B-it-GGUF",
+            provider="openai",
+            thinking_level="none",
+        )
+    )
+
+    messages = provider._convert_messages([], "You are helpful", provider._compat)
+
+    assert messages[0]["role"] == "system"
+    assert messages[0]["content"] == "You are helpful"
+
+
+def test_openai_completions_uses_developer_without_think_prefix_for_openai_api() -> None:
+    provider = OpenAICompletionsProvider(
+        ProviderConfig(
+            api_key="test-key",
+            base_url="https://api.openai.com/v1",
+            model="gpt-5",
+            provider="openai",
+            thinking_level="medium",
+        )
+    )
+
+    messages = provider._convert_messages([], "You are helpful", provider._compat)
+
+    assert messages[0]["role"] == "developer"
+    assert messages[0]["content"] == "You are helpful"
+
+
+class _EmptyAsyncIterator:
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        raise StopAsyncIteration
+
+
+class _DummyChatCompletions:
+    def __init__(self) -> None:
+        self.calls: list[dict[str, Any]] = []
+
+    async def create(self, **kwargs):
+        self.calls.append(kwargs)
+        return _EmptyAsyncIterator()
+
+
+@pytest.mark.asyncio
+async def test_openai_completions_sends_enable_thinking_for_local_gemma() -> None:
+    provider = OpenAICompletionsProvider(
+        ProviderConfig(
+            api_key="test-key",
+            base_url="http://127.0.0.1:1234/v1",
+            model="unsloth/gemma-4-26B-A4B-it-GGUF",
+            provider="openai",
+            thinking_level="medium",
+        )
+    )
+    dummy_chat = _DummyChatCompletions()
+    provider._client = cast(
+        Any,
+        type("DummyClient", (), {"chat": type("DummyChat", (), {"completions": dummy_chat})()})(),
+    )
+
+    stream = await provider._stream_impl(messages=[], system_prompt="You are helpful")
+    async for _ in stream:
+        pass
+
+    kwargs = dummy_chat.calls[0]
+    assert kwargs["extra_body"] == {"enable_thinking": True}
+    assert "reasoning_effort" not in kwargs
+    assert kwargs["messages"][0]["content"] == "<|think|>You are helpful"
+
+
 def test_openai_responses_uses_system_for_local_api() -> None:
     provider = OpenAIResponsesProvider(
         ProviderConfig(

{kon_coding_agent-0.3.3 → kon_coding_agent-0.3.4}/uv.lock

@@ -714,7 +714,7 @@ wheels = [
 
 [[package]]
 name = "kon-coding-agent"
-version = "0.3.3"
+version = "0.3.4"
 source = { editable = "." }
 dependencies = [
     { name = "aiofiles" },

kon_coding_agent-0.3.3/docs/local-models.md

@@ -1,42 +0,0 @@
-# Local Models
-
-This document provides detailed information about running and configuring local models with Kon.
-
-## Tested Models
-
-| Model | Quantization | Context Length | TPS | System Specs |
-| ----- | -------------- | -------------- | --- | ------------ |
-| `qwen/qwen3-coder-next` | Q4_K_M | 65,536 | N/A | i7-14700F × 28, 64GB RAM, 24GB VRAM (RTX 3090) |
-| `zai-org/glm-4.7-flash` | Q4_K_M | 65,536 | N/A | i7-14700F × 28, 64GB RAM, 24GB VRAM (RTX 3090) |
-| `unsloth/Qwen3.5-9B-GGUF` | Q4_K_M | 65,536 | N/A | i7-14700F × 28, 64GB RAM, 24GB VRAM (RTX 3090) |
-
-Run a local model using llama-server with the following command:
-
-```bash
-./llama-server \
-  --hf-repo unsloth/Qwen3.5-9B-GGUF \
-  --hf-file Qwen3.5-9B-Q4_K_M.gguf \
-  --port 5000 \
-  -c 65536
-```
-
-Then start Kon for a one-off local session:
-
-```bash
-kon --model unsloth/Qwen3.5-9B-GGUF --provider openai \
-  --base-url http://localhost:5000/v1 \
-  --openai-compat-auth none
-```
-
-If this is your default setup, put it in `~/.kon/config.toml` instead:
-
-```toml
-[llm]
-default_provider = "openai"
-default_model = "unsloth/Qwen3.5-9B-GGUF"
-default_base_url = "http://localhost:5000/v1"
-
-[llm.auth]
-openai_compat = "auto" # or "none" to always inject a placeholder key
-```
-