bantamkit 0.38.1__tar.gz → 0.38.2__tar.gz

{bantamkit-0.38.1 → bantamkit-0.38.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bantamkit
-Version: 0.38.1
+Version: 0.38.2
 Summary: Persistent memory for coding agents: an MCP server for Claude Code, Claude Desktop, Cursor and VS Code Copilot, plus a Python library that lifts small-model agents. Install once, run offline.
 Project-URL: Homepage, https://github.com/Ink01101011/bantamkit
 Project-URL: Repository, https://github.com/Ink01101011/bantamkit
@@ -144,9 +144,9 @@ CPU architecture and Python minor version** (some wheels, such as `pydantic_core
 one platform only), copy `wheels/` across, and install from it:
 
 ```bash
-python -m pip download "bantamkit[mcp]==0.38.1" -d wheels
+python -m pip download "bantamkit[mcp]==0.38.2" -d wheels
 python -m venv <env>
-<env>/bin/pip install --no-index --find-links wheels "bantamkit[mcp]==0.38.1"
+<env>/bin/pip install --no-index --find-links wheels "bantamkit[mcp]==0.38.2"
 <env>/bin/bantamkit-mcp --install cursor
 ```
 

{bantamkit-0.38.1 → bantamkit-0.38.2}/README.md

@@ -113,9 +113,9 @@ CPU architecture and Python minor version** (some wheels, such as `pydantic_core
 one platform only), copy `wheels/` across, and install from it:
 
 ```bash
-python -m pip download "bantamkit[mcp]==0.38.1" -d wheels
+python -m pip download "bantamkit[mcp]==0.38.2" -d wheels
 python -m venv <env>
-<env>/bin/pip install --no-index --find-links wheels "bantamkit[mcp]==0.38.1"
+<env>/bin/pip install --no-index --find-links wheels "bantamkit[mcp]==0.38.2"
 <env>/bin/bantamkit-mcp --install cursor
 ```
 

{bantamkit-0.38.1 → bantamkit-0.38.2}/src/bantamkit/__init__.py

@@ -29,4 +29,4 @@ from bantamkit.structured import StructuredOutputError, extract_json, structured
 # file as its dynamic version source, so the wheel's metadata and the string the MCP
 # server advertises are the same committed bytes, and neither is a function of when
 # someone last ran `pip`.
-__version__ = "0.38.1"
+__version__ = "0.38.2"

{bantamkit-0.38.1 → bantamkit-0.38.2}/src/bantamkit/skillmine.py

@@ -22,12 +22,19 @@ lacks a string `ts` or a known `status`) is skipped.
 - `repo` (R7): the realpath of the root's COMMON git dir — a linked worktree's `commondir`
   is followed — so the worktrees of one repository share one key; `""` outside a repo.
 
-The **signature** is `tokenize_v2` keywords of the prompt (a term in more than half of the
+The **signature** is `tokenize_label` keywords of the prompt (a term in more than half of the
 source's pieces is dropped, then the `KEYWORDS_TOP` most frequent, ties in code-point
 order), the tool-name n-grams for `NGRAM_SIZES`, and the touched paths. **Similarity** is a
 weighted Jaccard over the three, each component whose union is empty left out of both sums,
 answered as an exact reduced fraction `(num, den)`; `meets` compares on integers only, so the
-two runtimes cannot disagree at the `SIM_MIN` boundary.
+two runtimes cannot disagree at the `SIM_MIN` boundary. `tokenize_label` is the miner's own
+tokenizer (#289), not recall's `tokenize_v2`: recall's code-point bigrams split Thai vowels and
+tone marks (`ทำต่อ` became `ทํ ํา าต ต่ ่อ`) and let filler words such as `แล้ว` and `ทำ` decide
+the clusters. It groups a base character with its combining marks and following vowels
+(`ะ า ำ ๅ` and Lao `ະ າ ຳ`, which are Lo, not marks) and a leading vowel with the character
+after it, drops `LABEL_STOPWORDS`, and forms bigrams of those units; spaced-script
+words are kept whole exactly as `tokenize_v2` keeps them. `tokenize_v2` is unchanged, because
+recall is measured on it.
 
 **Clusters** (Task 5): leader clustering per source, the X1 bars on successful members only,
 twice the bar for the installer, recurring steps, pitfalls from failed members, and the X3
@@ -47,7 +54,9 @@ list, or the interrupt marker; a last turn has none). The `MinerCache` keeps eac
 offset and open turn, so an unchanged file is not read and a grown one is read from where the
 last run stopped. Measured shapes (job73 C6 Step 0) added `bash-*`, `task-notification` and
 `local-command-caveat` to the cut list: a `!` command's output and a background task's result
-arrive as non-meta user records.
+arrive as non-meta user records. #289 added `ide_opened_file`, `ide_selection`,
+`ide_diagnostics` and `browser_instruction`: the IDE and the Chrome extension write their
+notices into the user's record, and a prompt made only of them is empty and starts no piece.
 
 `runtime-ts/src/skillmine.ts` is the port;
 `tools/conformance/suites/skill-miner.mjs` compares the two.
@@ -61,6 +70,7 @@ import json
 import math
 import os
 import re
+import unicodedata
 import zlib
 from collections import Counter
 from collections.abc import Iterable, Mapping
@@ -68,7 +78,7 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
 
-from bantamkit.memory.retrieval import tokenize_v2
+from bantamkit.memory.retrieval import _nospace, _wordchar
 from bantamkit.touch.log import AGENT_ID
 
 SOURCE_SHIFTWORK = "shiftwork"
@@ -408,13 +418,141 @@ def _read_bundle(p: str) -> dict[str, Any] | None:
 # ---------------------------------------------------------------------------- signature
 
 
+#: #289: Thai filler words dropped from a label; each is matched as a unit sequence.
+LABEL_STOPWORDS = (
+    "ครับ",
+    "ค่ะ",
+    "แล้ว",
+    "ต่อ",
+    "ทำ",
+    "ให้",
+    "ที่",
+    "และ",
+    "ได้",
+    "หน่อย",
+    "ด้วย",
+    "เลย",
+    "นี้",
+    "ไหม",
+    "อีก",
+    "ก่อน",
+    "เป็น",
+)
+
+
+def _is_leading_vowel(cp: int) -> bool:
+    return 0x0E40 <= cp <= 0x0E44 or 0x0EC0 <= cp <= 0x0EC4
+
+
+#: Thai/Lao following vowels (SARA A, AA, AM, LAKKHANGYAO; Lao A, AA, AM): general category
+#: Lo, yet they close the syllable before them, so a unit takes them the way it takes a mark
+#: (#289 amendment A1).
+_FOLLOWING_VOWELS = frozenset({0x0E30, 0x0E32, 0x0E33, 0x0E45, 0x0EB0, 0x0EB2, 0x0EB3})
+
+
+def _label_fold(text: str) -> str:
+    """NFKC, lower-cased, SARA AM recomposed (NFKC splits it into NIKHAHIT + SARA AA)."""
+    s = unicodedata.normalize("NFKC", text).lower()
+    return s.replace("\u0e4d\u0e32", "\u0e33").replace("\u0ecd\u0eb2", "\u0eb3")
+
+
+def _label_units(run: str) -> list[str]:
+    """A no-space run as units: a mark or a following vowel joins the unit before it, a
+    leading vowel the one after. Each unit is gathered as a list of parts and joined once, so
+    a long run of joining characters stays linear (`out[-1] += ch` copied the unit each time)."""
+    units: list[list[str]] = []
+    pend: list[str] = []
+    for ch in run:
+        joins = ord(ch) in _FOLLOWING_VOWELS or unicodedata.category(ch) in ("Mn", "Mc")
+        if joins and units and not pend:
+            units[-1].append(ch)
+        elif _is_leading_vowel(ord(ch)):
+            pend.append(ch)
+        else:
+            pend.append(ch)
+            units.append(pend)
+            pend = []
+    if pend:
+        units.append(pend)
+    return ["".join(u) for u in units]
+
+
+_STOP_UNITS: tuple[tuple[str, ...], ...] = tuple(
+    sorted(
+        (tuple(_label_units(_label_fold(w))) for w in LABEL_STOPWORDS),
+        key=len,
+        reverse=True,
+    )
+)
+
+
+def _label_run(run: str, out: list[str]) -> None:
+    units = _label_units(run)
+    seg: list[str] = []
+
+    def emit() -> None:
+        if len(seg) == 1:
+            out.append(seg[0])
+        else:
+            out.extend(seg[i] + seg[i + 1] for i in range(len(seg) - 1))
+        seg.clear()
+
+    i = 0
+    while i < len(units):
+        for stop in _STOP_UNITS:
+            if tuple(units[i : i + len(stop)]) == stop:
+                emit()
+                i += len(stop)
+                break
+        else:
+            seg.append(units[i])
+            i += 1
+    emit()
+
+
+def tokenize_label(text: str) -> list[str]:
+    """The miner's tokenizer (#289): `tokenize_v2`'s scan, but a no-space run is cut into
+    units (a base with its marks and following vowels, a leading vowel with what follows),
+    `LABEL_STOPWORDS` end a segment and emit nothing, and a segment emits its one unit or its
+    unit bigrams. A bigram of units can still span a syllable boundary, and a stopword's unit
+    sequence also matches inside a longer word (`ต่อย` -> `ย`): there is no segmenter."""
+    s = _label_fold(text)
+    out: list[str] = []
+    word: list[str] = []
+    run: list[str] = []
+
+    def flush_word() -> None:
+        if word:
+            out.append("".join(word))
+            word.clear()
+
+    def flush_run() -> None:
+        if run:
+            _label_run("".join(run), out)
+            run.clear()
+
+    for ch in s:
+        if _nospace(ord(ch)):
+            flush_word()
+            run.append(ch)
+        elif _wordchar(ch):
+            flush_run()
+            word.append(ch)
+        else:
+            flush_word()
+            flush_run()
+    flush_word()
+    flush_run()
+    return out
+
+
 def doc_freq(pieces: Iterable[Piece]) -> tuple[Counter[str], int]:
-    """Document frequency of every `tokenize_v2` term over one source's pieces, and their count."""
+    """Document frequency of each `tokenize_label` term over one source's pieces, and the count."""
     df: Counter[str] = Counter()
     n = 0
     for p in pieces:
         n += 1
-        df.update(set(tokenize_v2(p.prompt)))
+        df.update(set(tokenize_label(p.prompt)))
     return df, n
 
 
@@ -425,7 +563,7 @@ def ngrams(tools: Iterable[str]) -> frozenset[str]:
 
 def signature(piece: Piece, df: Mapping[str, int], n: int) -> Signature:
     num, den = KEYWORD_DF_MAX
-    counts = Counter(t for t in tokenize_v2(piece.prompt) if df.get(t, 0) * den <= n * num)
+    counts = Counter(t for t in tokenize_label(piece.prompt) if df.get(t, 0) * den <= n * num)
     ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
     return Signature(
         keywords=frozenset(t for t, _ in ranked[:KEYWORDS_TOP]),
@@ -842,7 +980,10 @@ CORRECTIONS = (
 INTERRUPT = "[Request interrupted by user"
 #: R3: spans the harness writes into a user record; none of them is the user's own words.
 #: `bash-*` and `task-notification` are measured additions (C6 Step 0): a `!` command's
-#: output and a background task's result arrive as non-meta user records.
+#: output and a background task's result arrive as non-meta user records. The IDE and
+#: browser tags are #289's, measured 2026-11-06 over the maintainer's 798 session pieces:
+#: `ide_opened_file` was carried by 67, `browser_instruction` by 58, `ide_selection` by 20;
+#: `ide_diagnostics` occurs 151 times in transcripts (0 in pieces) and is cut defensively.
 CUT_TAGS = (
     "system-reminder",
     "command-name",
@@ -855,6 +996,10 @@ CUT_TAGS = (
     "bash-stdout",
     "bash-stderr",
     "task-notification",
+    "ide_opened_file",
+    "ide_selection",
+    "ide_diagnostics",
+    "browser_instruction",
 )
 #: R4: the tools whose input names a file the agent touched.
 PATH_TOOLS = ("Read", "Edit", "MultiEdit", "Write", "NotebookEdit")

{bantamkit-0.38.1 → bantamkit-0.38.2}/tests/test_skillmine.py

@@ -1012,3 +1012,133 @@ def test_a_successful_commit_carries_its_subject_and_a_failed_one_none():
     del old["commits"]  # a cache entry written before the field: read as no commits
     assert skillmine._piece_from(old).commits == ()
     assert skillmine._piece_from({**piece.as_dict(), "commits": [1]}) is None
+
+
+# ---------------------------------------------------------------------------- #289
+
+
+_IDE_NOTICE = (
+    "<ide_opened_file>The user opened the file /x/a.py in the IDE. This may or may not be "
+    "related to the current task.</ide_opened_file>"
+    "<browser_instruction>Claude in Chrome browser automation is available.</browser_instruction>"
+)
+
+
+@pytest.mark.parametrize(
+    ("text", "want"),
+    [
+        ("ทำต่อ", []),
+        ("merge แล้ว tag ต่อ", ["merge", "tag"]),
+        ("อนุมัติแล้ว", ["อนุ", "นุมั", "มัติ"]),
+        ("กลับไปใช้ แผนเดิม", ["กลั", "ลับ", "บไป", "ไปใช้", "แผน", "นเดิ", "เดิม"]),
+        ("เข้าใจแล้วครับ ทำต่อเลย", ["เข้าใจ"]),
+        ("จำนวน", ["จำน", "นว", "วน"]),
+        ("ทำงาน", ["งาน"]),
+        ("ประชุม", ["ประ", "ระชุ", "ชุม"]),
+        ("สร้างไฟล์ใหม่", ["สร้า", "ร้าง", "งไฟ", "ไฟล์", "ล์ให", "ใหม่"]),
+        # Known limit (#289 A1): a stopword's unit sequence matches inside a longer word
+        # ("ต่อ" in "ต่อย"); there is no dictionary segmenter, by ruling.
+        ("ต่อย", ["ย"]),
+    ],
+)
+def test_tokenize_label_vectors(text: str, want: list[str]) -> None:
+    assert skillmine.tokenize_label(text) == want
+
+
+def test_tokenize_label_matches_tokenize_v2_on_ascii() -> None:
+    from bantamkit.memory.retrieval import tokenize_v2
+
+    for text in ["Fix the parser in src/parse.py, then run 2 tests!", "merge PR #12 & tag v0.38"]:
+        assert skillmine.tokenize_label(text) == tokenize_v2(text)
+
+
+_FOLLOWING_VOWELS = "\u0e30\u0e32\u0e33\u0e45\u0eb0\u0eb2\u0eb3"
+
+
+def test_tokenize_label_no_token_starts_with_a_mark_or_following_vowel() -> None:
+    import unicodedata
+
+    texts = [
+        "ทำต่อ",
+        "merge แล้ว tag ต่อ",
+        "อนุมัติแล้ว",
+        "กลับไปใช้ แผนเดิม",
+        "เข้าใจแล้วครับ ทำต่อเลย",
+        "จำนวน",
+        "ประชุม",
+        "สร้างไฟล์ใหม่",
+        "ສະບາຍດີ ຂອບໃຈຫຼາຍໆ",
+    ]
+    toks = [t for x in texts for t in skillmine.tokenize_label(x)]
+    assert toks
+    assert all(unicodedata.category(t[0]) not in ("Mn", "Mc") for t in toks)
+    assert all(t[0] not in _FOLLOWING_VOWELS for t in toks)
+
+
+def test_label_units_attach_following_vowels() -> None:
+    assert skillmine._label_units("ทำ") == ["ทำ"]
+    assert skillmine._label_units("เข้าใจ") == ["เข้า", "ใจ"]
+    assert skillmine._label_units("ประ") == ["ป", "ระ"]
+    # A following vowel with no unit before it starts its own unit.
+    assert skillmine._label_units("\u0e32ก") == ["\u0e32", "ก"]
+    # One pending behind a leading vowel is not attached across it.
+    assert skillmine._label_units("เาก") == ["เา", "ก"]
+    assert skillmine.tokenize_label("เาก") == ["เาก"]
+
+
+def test_label_units_lakkhangyao_and_lao_sara_a() -> None:
+    # U+0E45 LAKKHANGYAO joins the unit before it, like the mark U+0E35 after ษ.
+    assert skillmine._label_units("ฤๅษี") == ["ฤๅ", "ษี"]
+    assert skillmine.tokenize_label("ฤๅษี") == ["ฤๅษี"]
+    # U+0EB0 Lao SARA A joins its base.
+    assert skillmine._label_units("ຈະ") == ["ຈະ"]
+    assert skillmine.tokenize_label("ຈະ") == ["ຈະ"]
+
+
+@pytest.mark.parametrize(
+    "run",
+    ["ก" + "\u0e48" * 10**6, "ก" + "\u0e32" * 10**6, "เ" + "\u0e48" * 10**6],
+    ids=["mark", "following-vowel", "behind-leading-vowel"],
+)
+def test_label_units_linear_on_long_joining_runs(run: str) -> None:
+    # `out[-1] += ch` copied the unit per joining character: ~13.7 s at 1M chars (26 s behind a
+    # leading vowel). Linear it is ~0.3 s, so a 3 s bound fails the old code and not a slow box.
+    import time
+
+    t0 = time.perf_counter()
+    units = skillmine._label_units(run)
+    assert time.perf_counter() - t0 < 3.0
+    assert "".join(units) == run
+
+
+def test_tokenize_v2_unchanged_for_recall() -> None:
+    from bantamkit.memory.retrieval import tokenize_v2
+
+    assert tokenize_v2("ทำต่อ") == ["ทํ", "ํา", "าต", "ต่", "่อ"]
+
+
+def test_signature_uses_tokenize_label() -> None:
+    p = Piece(id="t", source="session", ts="", repo="", prompt="อนุมัติแล้ว ทำต่อ", ok=True)
+    df, n = doc_freq([p])
+    assert set(df) == {"อนุ", "นุมั", "มัติ"}
+    assert signature(p, {}, 1).keywords == frozenset({"อนุ", "นุมั", "มัติ"})
+
+
+def test_clean_prompt_cuts_ide_and_browser_notices() -> None:
+    assert clean_prompt(_IDE_NOTICE) == ""
+    assert clean_prompt("<ide_selection>sel</ide_selection> fix it") == "fix it"
+    assert clean_prompt("<ide_diagnostics>d</ide_diagnostics>") == ""
+
+
+def test_ide_notice_only_prompt_yields_no_piece_or_keyword(tmp_path: Path) -> None:
+    home, repo = _home(tmp_path)
+    t = _T(SID, str(repo))
+    t.prompt(_ts(1), [{"type": "text", "text": _IDE_NOTICE}])
+    t.call(_ts(2), "Read", {"file_path": str(repo / "src" / "parse.py")}, "t1")
+    t.prompt(_ts(3), _IDE_NOTICE)
+    t.call(_ts(4), "Read", {"file_path": str(repo / "src" / "parse.py")}, "t2")
+    t.write(home / ".claude" / "projects" / "-work-repo" / f"{SID}.jsonl")
+    pieces = _run(home)
+    assert pieces == []
+    for word in ("ide", "opened", "browser", "chrome"):
+        assert word not in json.dumps(pieces)