cluxion-agentplugin-preprocessing 0.3.18__tar.gz → 0.3.19__tar.gz

{cluxion_agentplugin_preprocessing-0.3.18 → cluxion_agentplugin_preprocessing-0.3.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cluxion-agentplugin-preprocessing
-Version: 0.3.18
+Version: 0.3.19
 Summary: Universal agent plugin for Cluxion preprocessing, honesty contracts, clarification, Rust work queue, and resource-aware harness handoff.
 Project-URL: Homepage, https://github.com/cluxion/cluxion-Agentplugin-preprocessing
 Project-URL: Repository, https://github.com/cluxion/cluxion-Agentplugin-preprocessing

{cluxion_agentplugin_preprocessing-0.3.18 → cluxion_agentplugin_preprocessing-0.3.19}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "cluxion-agentplugin-preprocessing"
-version = "0.3.18"
+version = "0.3.19"
 description = "Universal agent plugin for Cluxion preprocessing, honesty contracts, clarification, Rust work queue, and resource-aware harness handoff."
 readme = "README.md"
 requires-python = ">=3.11"

{cluxion_agentplugin_preprocessing-0.3.18 → cluxion_agentplugin_preprocessing-0.3.19}/src/cluxion_agentplugin_preprocessing/plugin.yaml

@@ -1,5 +1,5 @@
 name: cluxion-agentplugin-preprocessing
-version: 0.3.18
+version: 0.3.19
 description: "Universal agent preprocessing plugin: honesty contracts, clarification, Rust work queue, resource-aware harness handoff. Connected AI calls cluxion tools; plugin does not own models."
 author: cluxion
 kind: standalone

{cluxion_agentplugin_preprocessing-0.3.18 → cluxion_agentplugin_preprocessing-0.3.19}/src/cluxion_runtime/core/preprocess.py

@@ -172,8 +172,8 @@ def preprocess_work(
 
 def estimate_tokens(text: str) -> int:
     """Estimate tokens conservatively without depending on the old Cluxion OS package."""
-    cjk = sum(1 for ch in text if ord(ch) > 127)
-    ascii_chars = max(0, len(text) - cjk)
+    ascii_chars = len(text.encode("ascii", "ignore"))
+    cjk = len(text) - ascii_chars
     return max(1, cjk + ascii_chars // 4)
 
 

cluxion_agentplugin_preprocessing-0.3.19/tests/runtime/test_estimate_tokens.py

@@ -0,0 +1,52 @@
+"""Byte-identical parity tests for estimate_tokens optimization."""
+
+from __future__ import annotations
+
+import random
+
+from cluxion_runtime.core.preprocess import estimate_tokens
+
+
+def _estimate_tokens_legacy(text: str) -> int:
+    cjk = sum(1 for ch in text if ord(ch) > 127)
+    ascii_chars = max(0, len(text) - cjk)
+    return max(1, cjk + ascii_chars // 4)
+
+
+def test_estimate_tokens_matches_legacy_fixed_cases() -> None:
+    cases = [
+        ("", 1),
+        ("abcd", 1),
+        ("한글텍스트", 5),
+        ("ab한", 1),
+    ]
+    for text, expected in cases:
+        assert estimate_tokens(text) == expected
+        assert estimate_tokens(text) == _estimate_tokens_legacy(text)
+
+
+def test_estimate_tokens_matches_legacy_random_unicode() -> None:
+    rng = random.Random(0)
+    chars: list[str] = []
+    for _ in range(10_000):
+        roll = rng.random()
+        if roll < 0.4:
+            chars.append(chr(rng.randint(0, 127)))
+        elif roll < 0.7:
+            chars.append(chr(rng.randint(128, 0xFFFF)))
+        else:
+            chars.append(chr(rng.randint(0x10000, 0x10FFFF)))
+    text = "".join(chars)
+    assert estimate_tokens(text) == _estimate_tokens_legacy(text)
+
+
+def test_estimate_tokens_lone_surrogate_does_not_raise() -> None:
+    text = "prefix\ud800suffix"
+    assert estimate_tokens(text) == _estimate_tokens_legacy(text)
+
+
+def test_estimate_tokens_legacy_reference_cases() -> None:
+    assert _estimate_tokens_legacy("") == 1
+    assert _estimate_tokens_legacy("abcd") == 1
+    assert _estimate_tokens_legacy("한글텍스트") == 5
+    assert _estimate_tokens_legacy("ab한") == 1