ctok 1.0.0__tar.gz

ctok-1.0.0/.github/workflows/ci.yml

@@ -0,0 +1,54 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write        # to post the gate report as a PR comment
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Install
+        run: uv sync --dev --python ${{ matrix.python-version }}
+
+      - name: Test
+        run: uv run --python ${{ matrix.python-version }} pytest -q
+
+      - name: Report gates, vocabulary and witness coverage
+        if: matrix.python-version == '3.12'
+        run: |
+          uv run --python ${{ matrix.python-version }} python tests/gates.py \
+            | tee gates.md >> "$GITHUB_STEP_SUMMARY"
+          cat gates.md
+
+      # Same-repo pull requests only: a fork's GITHUB_TOKEN is read-only and cannot comment.
+      # The step summary above is always written, so a fork PR still reports, just not inline.
+      - name: Comment gates and witness coverage on the pull request
+        if: >-
+          matrix.python-version == '3.12'
+          && github.event_name == 'pull_request'
+          && github.event.pull_request.head.repo.full_name == github.repository
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR: ${{ github.event.pull_request.number }}
+        run: |
+          gh pr comment "$PR" --body-file gates.md --edit-last --create-if-none \
+            || gh pr comment "$PR" --body-file gates.md

ctok-1.0.0/.github/workflows/publish.yml

@@ -0,0 +1,51 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Test
+        run: |
+          uv sync --dev
+          uv run pytest -q
+
+      - name: Build
+        run: uv build
+
+      - name: Check metadata
+        run: uvx twine check dist/*
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/ctok
+    permissions:
+      # Required for PyPI trusted publishing (OIDC); no API token is stored.
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v5
+        with:
+          name: dist
+          path: dist/
+
+      - uses: pypa/gh-action-pypi-publish@release/v1

ctok-1.0.0/.gitignore

@@ -0,0 +1,15 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+
+.coverage
+.pytest_cache/
+.ruff_cache/
+.venv/
+
+build/
+dist/
+
+.DS_Store
+.env
+.envrc

ctok-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sander Land
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ctok-1.0.0/PKG-INFO

@@ -0,0 +1,133 @@
+Metadata-Version: 2.5
+Name: ctok
+Version: 1.0.0
+Summary: Offline reconstruction of Claude's tokenizer token counts
+Project-URL: Repository, https://github.com/sanderland/ctok
+Project-URL: Issues, https://github.com/sanderland/ctok/issues
+Project-URL: Research, https://tokencontributions.substack.com/p/on-the-biology-of-claudes-tokenizer
+Author: Sander Land
+License-Expression: MIT
+License-File: LICENSE
+Keywords: anthropic,claude,nlp,token-count,tokenizer
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# ctok
+
+`ctok` reconstructs Claude token counts offline, with no API call, network access, or runtime
+dependencies. It is unofficial and is not affiliated with Anthropic.
+
+The reconstruction targets counts. Claude does not expose token boundaries, so `tokenize()` returns
+one valid minimum-cost tiling, not a claim about Anthropic's exact segmentation. The research behind
+the model is described in [On the biology of Claude's tokenizer](https://tokencontributions.substack.com/p/on-the-biology-of-claudes-tokenizer).
+
+## Install
+
+```bash
+pip install ctok
+```
+
+## Quick start
+
+```python
+from ctok import token_count, tokenize
+
+token_count("hello, world")           # 10, using the v3 family
+token_count("hello, world", "4.7")    # 15
+token_count("hello, world", "5.0")    # 10
+
+tokens = tokenize("NASA likes tokenizers")
+assert len(tokens) == token_count("NASA likes tokenizers")
+```
+
+The command-line interface prints the marked stream and its tiling:
+
+```bash
+ctok "hello, world"
+```
+
+## Supported families
+
+`version` is a string such as `"4.7"`. Components are compared as integers, so `"4.10"` sorts
+after `"4.9"`. Python reads the float literal `4.10` as `4.1`, so passing a non-string version
+raises `TypeError`.
+
+| requested version | family | model generation |
+|---|---|---|
+| `"3.0" <= version < "4.7"` | v3, the default | Claude 3 through Opus 4.6 |
+| `"4.7" <= version < "5.0"` | v4.7 | Opus 4.7 through 4.9 |
+| `version >= "5.0"` | v5 | Opus 5 and Sonnet 5 |
+
+v5 uses the v4.7 vocabulary with a different message frame.
+
+## How it works
+
+For one user message, `ctok`:
+
+1. normalizes the text, including NFC and family-specific quote folding;
+2. rewrites it into a stream with word, case, and byte markers;
+3. finds a minimum-cost tiling over the measured vocabulary and UTF-8 byte fallback;
+4. adds the measured message frame.
+
+`token_count(text)` is `len(tokenize(text))`. The output notation makes internal structure visible:
+
+| notation | meaning |
+|---|---|
+| `⟨bow⟩`, `⟨eow⟩` | word boundaries |
+| `⟨shift⟩`, `⟨caps⟩` | case rewrites |
+| `⟨0xNN⟩` | a byte-fallback token |
+| `⟨pad⟩` | part of the single-message frame |
+
+## Measured accuracy
+
+These results compare `ctok` with recorded `count_tokens` responses:
+
+| corpus | role | v3 exact | v4.7 exact |
+|---|---|---:|---:|
+| Goldfish, 350 languages and 350,000 rows | mining | 350,000 | 350,000 |
+| MultiPL-E, 22 programming languages | held out | 22 | 22 |
+| Rosetta Code, 1,741 documents | mining | 1,741 | 1,741 |
+| Rosetta Code, separate 250 documents | held out | 250 | 250 |
+| UDHR, 501 languages | mining (in-sample since 2026-08-12) | 501 | 501 |
+
+v5 is omitted from the table because its deviation from recorded counts matches v4.7's on every
+gated document. Separate API tests cover its message-frame rules.
+
+The stored measurement sets contain no under-counts: 0 of 1,664,940 v3 texts and 0 of 1,722,961
+v4.7 texts. This does not guarantee the result for arbitrary input. Goldfish, the main Rosetta
+sample, and UDHR informed piece selection. MultiPL-E and the separate Rosetta sample did not.
+
+Run the public gates with:
+
+```bash
+uv run pytest
+uv run python tests/gates.py
+```
+
+## Vocabulary evidence
+
+The two vocabulary files contain 48,645 v3 pieces and 15,240 v4.7 pieces. Every entry has a fixed
+membership witness or is one of the structural marker atoms checked by the test suite.
+
+```python
+from ctok import pieces, witness
+
+len(pieces("4.7"))                       # 15240
+witness("⟨bow⟩the⟨eow⟩", "4.7")
+# {'probe': 'the', 'raw': 12, 'kind': 'raw'}
+```
+
+A witness says that one marked piece costs one token in a calibrated probe. It does not prove the
+encoder rewrite or resolve ties between equal-cost tilings. `tests/test_witness.py` checks every
+published witness and requires complete witnessed-or-special coverage.
+
+## License
+
+MIT

ctok-1.0.0/README.md

@@ -0,0 +1,112 @@
+# ctok
+
+`ctok` reconstructs Claude token counts offline, with no API call, network access, or runtime
+dependencies. It is unofficial and is not affiliated with Anthropic.
+
+The reconstruction targets counts. Claude does not expose token boundaries, so `tokenize()` returns
+one valid minimum-cost tiling, not a claim about Anthropic's exact segmentation. The research behind
+the model is described in [On the biology of Claude's tokenizer](https://tokencontributions.substack.com/p/on-the-biology-of-claudes-tokenizer).
+
+## Install
+
+```bash
+pip install ctok
+```
+
+## Quick start
+
+```python
+from ctok import token_count, tokenize
+
+token_count("hello, world")           # 10, using the v3 family
+token_count("hello, world", "4.7")    # 15
+token_count("hello, world", "5.0")    # 10
+
+tokens = tokenize("NASA likes tokenizers")
+assert len(tokens) == token_count("NASA likes tokenizers")
+```
+
+The command-line interface prints the marked stream and its tiling:
+
+```bash
+ctok "hello, world"
+```
+
+## Supported families
+
+`version` is a string such as `"4.7"`. Components are compared as integers, so `"4.10"` sorts
+after `"4.9"`. Python reads the float literal `4.10` as `4.1`, so passing a non-string version
+raises `TypeError`.
+
+| requested version | family | model generation |
+|---|---|---|
+| `"3.0" <= version < "4.7"` | v3, the default | Claude 3 through Opus 4.6 |
+| `"4.7" <= version < "5.0"` | v4.7 | Opus 4.7 through 4.9 |
+| `version >= "5.0"` | v5 | Opus 5 and Sonnet 5 |
+
+v5 uses the v4.7 vocabulary with a different message frame.
+
+## How it works
+
+For one user message, `ctok`:
+
+1. normalizes the text, including NFC and family-specific quote folding;
+2. rewrites it into a stream with word, case, and byte markers;
+3. finds a minimum-cost tiling over the measured vocabulary and UTF-8 byte fallback;
+4. adds the measured message frame.
+
+`token_count(text)` is `len(tokenize(text))`. The output notation makes internal structure visible:
+
+| notation | meaning |
+|---|---|
+| `⟨bow⟩`, `⟨eow⟩` | word boundaries |
+| `⟨shift⟩`, `⟨caps⟩` | case rewrites |
+| `⟨0xNN⟩` | a byte-fallback token |
+| `⟨pad⟩` | part of the single-message frame |
+
+## Measured accuracy
+
+These results compare `ctok` with recorded `count_tokens` responses:
+
+| corpus | role | v3 exact | v4.7 exact |
+|---|---|---:|---:|
+| Goldfish, 350 languages and 350,000 rows | mining | 350,000 | 350,000 |
+| MultiPL-E, 22 programming languages | held out | 22 | 22 |
+| Rosetta Code, 1,741 documents | mining | 1,741 | 1,741 |
+| Rosetta Code, separate 250 documents | held out | 250 | 250 |
+| UDHR, 501 languages | mining (in-sample since 2026-08-12) | 501 | 501 |
+
+v5 is omitted from the table because its deviation from recorded counts matches v4.7's on every
+gated document. Separate API tests cover its message-frame rules.
+
+The stored measurement sets contain no under-counts: 0 of 1,664,940 v3 texts and 0 of 1,722,961
+v4.7 texts. This does not guarantee the result for arbitrary input. Goldfish, the main Rosetta
+sample, and UDHR informed piece selection. MultiPL-E and the separate Rosetta sample did not.
+
+Run the public gates with:
+
+```bash
+uv run pytest
+uv run python tests/gates.py
+```
+
+## Vocabulary evidence
+
+The two vocabulary files contain 48,645 v3 pieces and 15,240 v4.7 pieces. Every entry has a fixed
+membership witness or is one of the structural marker atoms checked by the test suite.
+
+```python
+from ctok import pieces, witness
+
+len(pieces("4.7"))                       # 15240
+witness("⟨bow⟩the⟨eow⟩", "4.7")
+# {'probe': 'the', 'raw': 12, 'kind': 'raw'}
+```
+
+A witness says that one marked piece costs one token in a calibrated probe. It does not prove the
+encoder rewrite or resolve ties between equal-cost tilings. `tests/test_witness.py` checks every
+published witness and requires complete witnessed-or-special coverage.
+
+## License
+
+MIT

ctok-1.0.0/ctok/__init__.py

@@ -0,0 +1,5 @@
+"""Offline reconstruction of Claude's tokenizer token counts."""
+
+from .main import marked_stream, normalize, pieces, token_count, tokenize, witness
+
+__all__ = ["token_count", "tokenize", "normalize", "marked_stream", "pieces", "witness"]

ctok-1.0.0/ctok/constants.py

@@ -0,0 +1,224 @@
+"""The marker notation and the measured character tables.
+
+The tables were measured against ``count_tokens``. Tables marked as enumerated must stay
+enumerated: no Unicode-category rule reproduces them.
+"""
+
+import re
+
+# ---- marker notation ----------------------------------------------------------------------------
+
+# The engine computes over marked strings in which each structural marker is one codepoint.
+BOW_G, EOW_G, PAD_G = "\ufdd0", "\ufdd1", "\ufdd2"
+SHIFT_G, CAPS_G = "\ufdd3", "\ufdd4"
+
+MARKER_GLYPHS = frozenset({BOW_G, EOW_G, SHIFT_G, CAPS_G})
+
+# The public form: named atoms in mathematical angle brackets, practically absent from real text.
+# Literal brackets in text are byte-escaped at render time, so a rendered token always parses back
+# unambiguously.
+L, R = "⟨", "⟩"
+BOW, EOW, SHIFT, CAPS = f"{L}bow{R}", f"{L}eow{R}", f"{L}shift{R}", f"{L}caps{R}"
+PAD = f"{L}pad{R}"
+
+GLYPH_TO_ATOM = {BOW_G: BOW, EOW_G: EOW, PAD_G: PAD, SHIFT_G: SHIFT, CAPS_G: CAPS}
+ATOM_TO_GLYPH = {BOW: BOW_G, EOW: EOW_G, SHIFT: SHIFT_G, CAPS: CAPS_G}
+
+# Scraped text can contain the noncharacters used above. NFC strips BMP private-use input, so these
+# five private-use codepoints are safe internal escapes inserted after NFC. The byte floor and
+# renderer map them back to the original noncharacters.
+LITERAL_MARKER_ESCAPES = {glyph: chr(0xE000 + i) for i, glyph in enumerate(GLYPH_TO_ATOM)}
+ESCAPED_MARKER_LITERALS = {escape: glyph for glyph, escape in LITERAL_MARKER_ESCAPES.items()}
+LITERAL_MARKER_ESCAPE_TABLE = str.maketrans(LITERAL_MARKER_ESCAPES)
+
+# ---- character classes --------------------------------------------------------------------------
+
+WORDY, HARD, DIGIT, PUNCT, SPACE = "wordy", "hard", "digit", "punct", "space"
+
+# Non-ASCII symbols/punctuation that tile over the punct vocabulary rather than standing alone.
+# Enumerated: the behaviour splits per codepoint with no categorical rule (`（` is punct but `）`
+# is hard). Anything unlisted stays HARD.
+PUNCT_SYMS = frozenset("—»«•°„–−£§€…√→（№†└│།·─═█")
+
+# Symbol-letters measured to take the full word model (⟨bow⟩ and ⟨eow⟩ flanks, seam absorption, case
+# markers) exactly like Latin letters. Enumerated blocks: category Nl/So/Lu splits both ways, and
+# the Hangzhou numerals (also Nl) measured markerless, so it is the block that predicts, not the
+# category.
+SYMBOL_LETTERS = (
+    (0x16EE, 0x16F0),   # Runic golden numbers (Nl, caseless)
+    (0x2160, 0x2188),   # Roman numerals (Nl/Lu/Ll, cased pairs)
+    (0x24B6, 0x24E9),   # circled letters (So, cased)
+    (0xA6E6, 0xA6EF),   # Bamum number-letters (Nl, caseless)
+)
+
+# Variation selectors are gc=Mn but take no word model: each costs one more than its base
+# character, where ⟨bow⟩ and ⟨eow⟩ flanks would read three more. The supplementary selectors
+# (U+E0100 to U+E01EF) are astral and already HARD.
+VARIATION_SELECTORS = (0xFE00, 0xFE0F)
+
+# The one canonical-combining-class-9 character that does not separate word runs: measured over
+# as a separator on its own script's consonants and under wherever no letter precedes it. See
+# :func:`normalize.is_separator` for the grid.
+NON_SEPARATORS = frozenset("ฺ")   # THAI CHARACTER PHINTHU
+
+# Marks that separate word runs the way a virama does, but without combining class 9. Measured
+# one character at a time on byte-floor consonant hosts; enumerated, because the rule is
+# orthographic (the neighbouring codepoint is usually a vowel sign that does not split). The
+# U+0300 block is handled as a range instead. See :data:`SEPARATOR_MARKS`.
+SEPARATOR_SIGNS = frozenset((
+    "\u0740",  # SYRIAC FEMININE DOT
+    "\u0741",  # SYRIAC QUSHSHAYA
+    "\u0742",  # SYRIAC RUKKAKHA
+    "\u0743",  # SYRIAC TWO VERTICAL DOTS ABOVE
+    "\u0744",  # SYRIAC TWO VERTICAL DOTS BELOW
+    "\u0745",  # SYRIAC THREE DOTS ABOVE
+    "\u0746",  # SYRIAC THREE DOTS BELOW
+    "\u0747",  # SYRIAC OBLIQUE LINE ABOVE
+    "\u0748",  # SYRIAC OBLIQUE LINE BELOW
+    "\u0749",  # SYRIAC MUSIC
+    "\u074a",  # SYRIAC BARREKH
+    "\u093c",  # DEVANAGARI SIGN NUKTA
+    "\u0951",  # DEVANAGARI STRESS SIGN UDATTA
+    "\u0952",  # DEVANAGARI STRESS SIGN ANUDATTA
+    "\u0953",  # DEVANAGARI GRAVE ACCENT
+    "\u0954",  # DEVANAGARI ACUTE ACCENT
+    "\u09bc",  # BENGALI SIGN NUKTA
+    "\u09fe",  # BENGALI SANDHI MARK
+    "\u0a3c",  # GURMUKHI SIGN NUKTA
+    "\u0abc",  # GUJARATI SIGN NUKTA
+    "\u0afd",  # GUJARATI SIGN THREE-DOT NUKTA ABOVE
+    "\u0afe",  # GUJARATI SIGN CIRCLE NUKTA ABOVE
+    "\u0aff",  # GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE
+    "\u0b3c",  # ORIYA SIGN NUKTA
+    "\u0b55",  # ORIYA SIGN OVERLINE
+    # U+0C03 TELUGU SIGN VISARGA is deliberately not here: it is a spacing mark (Mc, ccc 0) that
+    # measures as plain word material, closing a word with a `ః⟨eow⟩` piece rather than splitting it.
+    "\u0c3c",  # TELUGU SIGN NUKTA
+    "\u0cbc",  # KANNADA SIGN NUKTA
+    "\u0e47",  # THAI CHARACTER MAITAIKHU
+    "\u0e48",  # THAI CHARACTER MAI EK
+    "\u0e49",  # THAI CHARACTER MAI THO
+    "\u0e4a",  # THAI CHARACTER MAI TRI
+    "\u0e4b",  # THAI CHARACTER MAI CHATTAWA
+    "\u0e4c",  # THAI CHARACTER THANTHAKHAT
+    "\u0e4e",  # THAI CHARACTER YAMAKKAN
+    "\u0ec8",  # LAO TONE MAI EK
+    "\u0ec9",  # LAO TONE MAI THO
+    "\u0eca",  # LAO TONE MAI TI
+    "\u0ecb",  # LAO TONE MAI CATAWA
+    "\u0ecc",  # LAO CANCELLATION MARK
+    "\u0ece",  # LAO YAMAKKAN
+    "\u0f18",  # TIBETAN ASTROLOGICAL SIGN -KHYUD PA
+    "\u0f19",  # TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS
+    "\u0f35",  # TIBETAN MARK NGAS BZUNG NYI ZLA
+    "\u0f37",  # TIBETAN MARK NGAS BZUNG SGOR RTAGS
+    "\u0f39",  # TIBETAN MARK TSA -PHRU
+    "\u0f3e",  # TIBETAN SIGN YAR TSHES
+    "\u0f3f",  # TIBETAN SIGN MAR TSHES
+    "\u0f86",  # TIBETAN SIGN LCI RTAGS
+    "\u0f87",  # TIBETAN SIGN YANG RTAGS
+    "\u0fc6",  # TIBETAN SYMBOL PADMA GDAN
+    "\u1037",  # MYANMAR SIGN DOT BELOW
+    "\u17b4",  # KHMER VOWEL INHERENT AQ
+    "\u17b5",  # KHMER VOWEL INHERENT AA
+    "\u17c9",  # KHMER SIGN MUUSIKATOAN
+    "\u17ca",  # KHMER SIGN TRIISAP
+    "\u17cb",  # KHMER SIGN BANTOC
+    "\u17cc",  # KHMER SIGN ROBAT
+    "\u17cd",  # KHMER SIGN TOANDAKHIAT
+    "\u17ce",  # KHMER SIGN KAKABAT
+    "\u17cf",  # KHMER SIGN AHSDA
+    "\u17d0",  # KHMER SIGN SAMYOK SANNYA
+    "\u17d1",  # KHMER SIGN VIRIAM
+    "\u17d3",  # KHMER SIGN BATHAMASAT
+    "\u17dd",  # KHMER SIGN ATTHACAN
+    "\u1a75",  # TAI THAM SIGN TONE-1
+    "\u1a76",  # TAI THAM SIGN TONE-2
+    "\u1a77",  # TAI THAM SIGN KHUEN TONE-3
+    "\u1a78",  # TAI THAM SIGN KHUEN TONE-4
+    "\u1a79",  # TAI THAM SIGN KHUEN TONE-5
+    "\u1a7a",  # TAI THAM SIGN RA HAAM
+    "\u1a7b",  # TAI THAM SIGN MAI SAM
+    "\u1a7c",  # TAI THAM SIGN KHUEN-LUE KARAN
+    "\u1a7f",  # TAI THAM COMBINING CRYPTOGRAMMIC DOT
+    "\u1b34",  # BALINESE SIGN REREKAN
+    "\u1b6b",  # BALINESE MUSICAL SYMBOL COMBINING TEGEH
+    "\u1b6c",  # BALINESE MUSICAL SYMBOL COMBINING ENDEP
+    "\u1b6d",  # BALINESE MUSICAL SYMBOL COMBINING KEMPUL
+    "\u1b6e",  # BALINESE MUSICAL SYMBOL COMBINING KEMPLI
+    "\u1b6f",  # BALINESE MUSICAL SYMBOL COMBINING JEGOGAN
+    "\u1b70",  # BALINESE MUSICAL SYMBOL COMBINING KEMPUL WITH JEGOGAN
+    "\u1b71",  # BALINESE MUSICAL SYMBOL COMBINING KEMPLI WITH JEGOGAN
+    "\u1b72",  # BALINESE MUSICAL SYMBOL COMBINING BENDE
+    "\u1b73",  # BALINESE MUSICAL SYMBOL COMBINING GONG
+    "\ua9b3",  # JAVANESE SIGN CECAK TELU
+))
+
+# ---- normalization ------------------------------------------------------------------------------
+
+# C0/C1 controls the API strips before tokenizing (cost 0), i.e. every gc=Cc except TAB, LF and NUL.
+STRIP_CONTROL = re.compile("[\x01-\x08\x0B-\x1F\x7F\x80-\x9F]")
+
+# BMP private use is stripped the same way, and its two neighbours join into one word. Unassigned
+# codepoints survive and pay their bytes; the supplementary private-use planes are unprobed and
+# deliberately left out.
+STRIP_PRIVATE = re.compile("[-]")
+
+# Lone surrogates are valid in a str but not UTF-8-encodable, so they would crash the byte floor.
+# They can never be genuine tokenizer input; fold each to U+FFFD, which prices as an ordinary
+# 3-byte HARD character. This is what keeps token_count total on any str.
+SURROGATE = re.compile(r"[\ud800-\udfff]")
+
+# Space separators the tokenizer treats identically to U+0020: all Zs except U+3000 (ideographic
+# space), plus Zl/Zp. NB U+3000, TAB and LF each have their own cost. NUL is not stripped but
+# tokenizes exactly like a space, so it is folded here too.
+FUNNY_SPACE = re.compile("[\u00A0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F]")
+
+# The four standard curly quotes fold to their ASCII forms (v3 only; NFC does not do this). The
+# low-9 mark U+201E is a different token and is deliberately not folded.
+QUOTE_FOLD = str.maketrans({"\u2018": "'", "\u2019": "'", "\u201C": '"', "\u201D": '"'})
+
+# The suffixes an apostrophe binds into the word ahead of it, deleting that word's ⟨bow⟩. This is the
+# standard English contraction set, lowercase and whole-word only. Measured per member; other
+# suffixes pay a full boundary even when their own piece exists, so no piece-based rule reproduces
+# this. Case-sensitive and whole-word (`x'S`, `x'LL`, `x'llo`, `x'lls` all pay).
+CONTRACTION_SUFFIXES = frozenset({"s", "t", "d", "m", "ll", "re", "ve"})
+
+# ---- the seam law -------------------------------------------------------------------------------
+
+# ⟨eow⟩ ' ' [case markers] ⟨bow⟩ -> ⟨eow⟩ [case markers] ⟨bow⟩: a single space between two marked
+# spans is the seam and is not written as a character.
+SEAM_RE = re.compile("(.)" + EOW_G + " " + "([" + SHIFT_G + CAPS_G + "]*)" + BOW_G)
+
+# ---- marks that stand outside the word -----------------------------------------------------------
+
+# Combining marks that close the word before themselves, exactly as a virama does
+# (`normalize.is_separator` folds this range in): `q́z` = `⟨bow⟩q⟨eow⟩` + mark + `⟨bow⟩z⟨eow⟩`, so an
+# accent makes two words out of one. Two oracle frames on one-token hosts distinguish the
+# spellings. A sweep covered U+0300 to U+036F on 21 hosts across 13 scripts and both families, with
+# no dissent. U+0345 and the combining Latin letters U+0363 to U+036F pin both ends of the range
+# from outside by reading "inside the word". U+0340, U+0341, U+0343, and U+0344 are NFC-folded away
+# and inside the range for that reason alone.
+SEPARATOR_MARKS = re.compile("[\u0300-\u0344\u0346-\u0362]")
+
+# The same question asked of every other combining block of the BMP: 418 marks swept on the same
+# two frames, both families, no dissent. The split is orthographic, as for Thai:
+#
+#   OUTSIDE the word   accents, tone marks, cantillation and annotation
+#   INSIDE the word    vowel points and combining letters (niqqud, harakat, the Syriac vowels,
+#                      Thaana, Samaritan, the combining Latin/Cyrillic letter blocks)
+#
+# Most ranges are pinned on both sides by an inside-the-word neighbour in the same block; where a
+# range abuts unassigned codepoints, it is written to the marks actually probed. The N'Ko and
+# Mandaic marks have no in-script one-token host and rest on `q` and `б` alone.
+SEPARATOR_ANNOTATIONS = re.compile(
+    "[\u0483-\u0489\u0591-\u05af\u0658\u06df-\u06e0\u06ea-\u06ec"
+    "\u07eb-\u07f3\u07fd\u0859-\u085b\u0898-\u089f\u08ca-\u08d3\u08e0-\u08e1"
+    "\u0818-\u0819\u082d\u08ea-\u08ef\u135d-\u135f"
+    "\u180b-\u180d\u180f\u1939-\u193b"
+    "\u1ab0-\u1abe\u1ac1-\u1acb\u1cd0-\u1ce8\u1ced\u1cf4\u1cf8-\u1cf9"
+    "\u1be6\u1c37\u1cf7"
+    "\u1dc0-\u1dd2\u1df5-\u1dff\u20d0-\u20f0\ua66f-\ua672\ua67c-\ua67d"
+    "\u2cef-\u2cf1\u302a-\u302f\u3099-\u309a"
+    "\ua6f0-\ua6f1\ua8e0-\ua8f1\ua92b-\ua92d\uaabf\uaac1\uabec"
+    "\ufe20-\ufe2f]")