tokmor 1.2.9__py3-none-any.whl

tokmor/token_quality.py

@@ -0,0 +1,1185 @@
+"""
+Token quality postprocessing (OSS core)
+======================================
+
+Goal:
+- Neutral token quality fixes only (NOT NER policy).
+- Applied uniformly across all tokenizers via TokenizerResult.__post_init__.
+
+Extension model (what you asked for):
+- Add **global rules** (safe, language-agnostic)
+- Add **script/family rules** (e.g., cjk/brahmic/rtl/indic/space)
+- Add **language-specific mini rules** (zh/ja/...)
+
+All rules are automatically applied to every tokenizer output.
+
+Safety guards:
+- adjacency checks (offset continuity) when merging
+- bounded lookahead
+- hard length limits to avoid aggressive merges
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Callable, Dict, List, Optional
+
+
+RuleFn = Callable[[List[object], str, str], List[object]]
+
+_GLOBAL_RULES: List[RuleFn] = []
+_FAMILY_RULES: Dict[str, List[RuleFn]] = {}
+_LANG_RULES: Dict[str, List[RuleFn]] = {}
+
+
+def register_global_rule(fn: RuleFn) -> None:
+    _GLOBAL_RULES.append(fn)
+
+
+def register_family_rule(family: str, fn: RuleFn) -> None:
+    family = (family or "").strip().lower()
+    if not family:
+        return
+    _FAMILY_RULES.setdefault(family, []).append(fn)
+
+
+def register_lang_rule(lang: str, fn: RuleFn) -> None:
+    lang = (lang or "").strip().lower().replace("_", "-")
+    if not lang:
+        return
+    _LANG_RULES.setdefault(lang, []).append(fn)
+
+
+def _family_for_lang(lang: str) -> Optional[str]:
+    """
+    Lightweight script/family routing without importing tokenizer modules
+    (avoids circular imports).
+    """
+    ll = (lang or "").lower().replace("_", "-")
+    if ll in {"zh", "zh-cn", "zh-tw", "ja", "ko"}:
+        return "cjk"
+    if ll in {"th", "lo", "my", "km"}:
+        return "brahmic"
+    if ll in {"ar", "he", "fa", "ur", "yi", "ps"}:
+        return "rtl"
+    if ll in {"hi", "bn", "gu", "pa", "mr", "ne", "si", "ta", "te", "kn", "ml", "or", "as", "sa"}:
+        return "indic"
+    return "space"
+
+
+def apply_token_quality(tokens: List[object], *, lang: str, text: str) -> List[object]:
+    """
+    Apply token-quality fixes for a given language.
+
+    `tokens` are expected to be tokmor.base.Token-like objects with:
+      - text: str
+      - start: int
+      - end: int
+    """
+    if not tokens:
+        return tokens
+
+    ll = (lang or "").lower().replace("_", "-")
+    fam = _family_for_lang(ll) or ""
+
+    out = tokens
+    for fn in _GLOBAL_RULES:
+        out = fn(out, ll, text)
+        if not out:
+            return out
+
+    for fn in _FAMILY_RULES.get(fam, []):
+        out = fn(out, ll, text)
+        if not out:
+            return out
+
+    for fn in _LANG_RULES.get(ll, []):
+        out = fn(out, ll, text)
+        if not out:
+            return out
+
+    return out
+
+
+def _rule_merge_digit_groups(tokens: List[object], _lang: str, text: str) -> List[object]:
+    """
+    Global neutral fix: merge common digit-group / decimal splits that happen in whitespace tokenizers.
+
+    Examples:
+    - "22,000" tokenized as ["22", "000"]  -> ["22,000"]
+    - "1.28" tokenized as ["1", "28"]      -> ["1.28"]
+
+    Safety:
+    - only when separated by a single char in the original text
+    - separator must be one of {',', '.', '٬', '٫'}
+    - both sides must be all digits
+    - bounded total length to avoid over-merging
+    """
+    if not tokens or len(tokens) < 2:
+        return tokens
+
+    TokenType = type(tokens[0])
+
+    def _tok_from_span(s: int, e: int):
+        return TokenType(text=text[s:e], start=s, end=e)
+
+    def _is_digits(s: str) -> bool:
+        return bool(s) and all(ch.isdigit() for ch in s)
+
+    SEPS = {",", ".", "٬", "٫"}
+
+    out: List[object] = []
+    i = 0
+    n = len(tokens)
+    while i < n:
+        a = tokens[i]
+        if i + 1 < n:
+            b = tokens[i + 1]
+            a_s, a_e = int(getattr(a, "start")), int(getattr(a, "end"))
+            b_s, b_e = int(getattr(b, "start")), int(getattr(b, "end"))
+            if 0 <= a_s <= a_e <= len(text) and 0 <= b_s <= b_e <= len(text):
+                if b_s == a_e + 1:
+                    sep = text[a_e:b_s]
+                    if sep in SEPS:
+                        at = getattr(a, "text", "") or text[a_s:a_e]
+                        bt = getattr(b, "text", "") or text[b_s:b_e]
+                        if _is_digits(at) and _is_digits(bt):
+                            merged_txt = text[a_s:b_e]
+                            if 1 <= len(merged_txt) <= 32:
+                                out.append(_tok_from_span(a_s, b_e))
+                                i += 2
+                                continue
+        out.append(a)
+        i += 1
+    return out
+
+
+def _rule_zh(tokens: List[object], _lang: str, text: str) -> List[object]:
+    """
+    Chinese token quality fixes (conservative):
+    - Split over-merged suffix+verbish chunks: ...港聘用 -> ...港 + 聘用
+    - Merge dot-connected names (·), allowing small whitespace gaps: 米拉 · 万 托斯 -> 米拉·万托斯
+    - Merge short geo-name chains ending with 港 (recover common over-splits): 维 + 伦德 + 尔港 -> 维伦德尔港
+    - Split stuck function char in very limited cases: 在维... -> 在 + 维...
+    """
+    if not tokens:
+        return tokens
+
+    TokenType = type(tokens[0])
+
+    def _tok(text_: str, start: int, end: int):
+        return TokenType(text=text_, start=start, end=end)
+
+    def _is_cjk(ch: str) -> bool:
+        return bool(ch) and (("\u4e00" <= ch <= "\u9fff") or ("\u3400" <= ch <= "\u4dbf"))
+
+    def _is_name_piece(s: str) -> bool:
+        if not s:
+            return False
+        for ch in s:
+            if ch.isalnum():
+                continue
+            if _is_cjk(ch):
+                continue
+            return False
+        return True
+
+    def _gap(a_end: int, b_start: int) -> str:
+        if a_end < 0 or b_start < 0 or b_start < a_end or b_start > len(text):
+            return ""
+        return text[a_end:b_start]
+
+    def _gap_is_ws(g: str) -> bool:
+        return bool(g) and len(g) <= 3 and g.isspace()
+
+    VERBISH = {
+        "抵达",
+        "聘用",
+        "发布",
+        "宣布",
+        "表示",
+        "前往",
+        "返回",
+        "访问",
+        "会见",
+        "举行",
+        "发生",
+        "完成",
+        "启动",
+        "加入",
+        "离开",
+        "进入",
+    }
+    VERB_TAIL = {"", "了", "着", "过"}
+
+    def _is_verbish_chunk(s: str) -> bool:
+        if not s:
+            return False
+        for v in VERBISH:
+            for tail in VERB_TAIL:
+                if s == (v + tail):
+                    return True
+                if s.startswith(v + tail) and len(s) <= len(v + tail) + 1:
+                    return True
+        return False
+
+    NAME_ENDINGS = {"斯", "尔", "德", "特", "姆", "克", "夫", "诺", "拉", "娜", "尼", "亚", "里", "罗", "多", "恩"}
+
+    def _looks_like_foreign_name_prefix(s: str) -> bool:
+        if not s or len(s) < 2 or len(s) > 8:
+            return False
+        if not all(_is_cjk(ch) for ch in s):
+            return False
+        return s[-1] in NAME_ENDINGS
+
+    # pass 1: split obvious over-merges inside a single token
+    split_out: List[object] = []
+    for t in tokens:
+        txt = getattr(t, "text", "") or ""
+        if len(txt) < 3:
+            split_out.append(t)
+            continue
+
+        did_split = False
+
+        # 0) Dot-name over-merge guard:
+        # e.g., "米拉·万·托斯抵达维伦德尔港" -> "米拉·万·托斯" + "抵达" + "维伦德尔港"
+        if ("·" in txt) and (not did_split):
+            for v in VERBISH:
+                for tail in VERB_TAIL:
+                    vv = v + tail
+                    idx_v = txt.find(vv)
+                    if idx_v <= 0:
+                        continue
+                    if "·" not in txt[:idx_v]:
+                        continue
+                    # Ensure offsets align with text length (avoid corrupt spans).
+                    s0 = int(getattr(t, "start"))
+                    e0 = int(getattr(t, "end"))
+                    if (e0 - s0) != len(txt):
+                        continue
+                    if len(txt) > 80:
+                        continue
+                    left = txt[:idx_v]
+                    rem = txt[idx_v + len(vv) :]
+                    split_out.append(_tok(left, s0, s0 + len(left)))
+                    split_out.append(_tok(vv, s0 + idx_v, s0 + idx_v + len(vv)))
+                    if rem:
+                        split_out.append(_tok(rem, s0 + idx_v + len(vv), e0))
+                    did_split = True
+                    break
+                if did_split:
+                    break
+        if did_split:
+            continue
+
+        idx = txt.rfind("港")
+        if idx != -1 and idx < len(txt) - 1:
+            rem = txt[idx + 1 :]
+            if _is_verbish_chunk(rem):
+                left = txt[: idx + 1]
+                right = txt[idx + 1 :]
+                mid = getattr(t, "start") + len(left)
+                split_out.append(_tok(left, getattr(t, "start"), mid))
+                split_out.append(_tok(right, mid, getattr(t, "end")))
+                did_split = True
+        if did_split:
+            continue
+
+        for v in VERBISH:
+            for tail in VERB_TAIL:
+                suffix = v + tail
+                if txt.endswith(suffix) and len(txt) > len(suffix):
+                    prefix = txt[: -len(suffix)]
+                    if _looks_like_foreign_name_prefix(prefix):
+                        mid = getattr(t, "end") - len(suffix)
+                        split_out.append(_tok(prefix, getattr(t, "start"), mid))
+                        split_out.append(_tok(suffix, mid, getattr(t, "end")))
+                        did_split = True
+                        break
+            if did_split:
+                break
+        if not did_split:
+            split_out.append(t)
+
+    split_out.sort(key=lambda x: getattr(x, "start"))
+
+    # pass 1.5: split limited stuck function char prefixes (very conservative)
+    FUNC_PREFIX = {"在", "到", "于", "从", "往", "去"}
+    split2: List[object] = []
+    for idx, t in enumerate(split_out):
+        txt = getattr(t, "text", "") or ""
+        if len(txt) >= 2 and txt[0] in FUNC_PREFIX:
+            rem = txt[1:]
+            nxt = split_out[idx + 1] if idx + 1 < len(split_out) else None
+            if rem and all(_is_cjk(ch) for ch in rem) and nxt and _is_name_piece(getattr(nxt, "text", "")):
+                s0 = getattr(t, "start")
+                e0 = getattr(t, "end")
+                if (s0 + 1) <= e0:
+                    split2.append(_tok(txt[0], s0, s0 + 1))
+                    split2.append(_tok(rem, s0 + 1, e0))
+                    continue
+        split2.append(t)
+    split2.sort(key=lambda x: getattr(x, "start"))
+
+    # pass 1.6: merge short geo-name chains ending with 港 (contiguous, bounded)
+    geo_merged: List[object] = []
+    i = 0
+    while i < len(split2):
+        t0 = split2[i]
+        if getattr(t0, "text", None) in FUNC_PREFIX:
+            geo_merged.append(t0)
+            i += 1
+            continue
+
+        parts: List[object] = []
+        j = i
+        merged = False
+        while j < len(split2) and len(parts) < 5:
+            cur = split2[j]
+            if parts and getattr(parts[-1], "end") != getattr(cur, "start"):
+                break
+            cur_txt = getattr(cur, "text", "") or ""
+            if not cur_txt or not all(_is_cjk(ch) for ch in cur_txt):
+                break
+            # Never merge verbish chunks into toponyms (keeps dot-name split + verb separate).
+            if _is_verbish_chunk(cur_txt):
+                break
+            parts.append(cur)
+            joined = "".join(getattr(p, "text", "") or "" for p in parts)
+            if len(joined) > 12:
+                break
+            if joined.endswith("港") and len(parts) >= 2:
+                geo_merged.append(_tok(joined, getattr(parts[0], "start"), getattr(parts[-1], "end")))
+                i = j + 1
+                merged = True
+                break
+            j += 1
+        if not merged:
+            geo_merged.append(t0)
+            i += 1
+
+    # pass 2: merge dot-connected names (·) (gap-aware, bounded)
+    merged_names: List[object] = []
+    i = 0
+    while i < len(geo_merged):
+        t0 = geo_merged[i]
+        if not _is_name_piece(getattr(t0, "text", "")):
+            merged_names.append(t0)
+            i += 1
+            continue
+
+        parts = [t0]
+        connectors: List[str] = []
+        j = i + 1
+        saw_dot = False
+
+        while j < len(geo_merged):
+            prev = parts[-1]
+            nxt = geo_merged[j]
+            g = _gap(getattr(prev, "end"), getattr(nxt, "start"))
+            g_strip = g.strip()
+            nxt_text = getattr(nxt, "text", "") or ""
+
+            # Hard stop: do NOT let a dot-name merge swallow verbs / location chunks.
+            # Example to avoid: "米拉·万·托斯抵达维伦德尔港" becoming one token.
+            if _is_verbish_chunk(nxt_text):
+                break
+            if nxt_text and (nxt_text == "港" or nxt_text.endswith("港") or nxt_text.endswith("市") or nxt_text.endswith("省") or nxt_text.endswith("县") or nxt_text.endswith("区") or nxt_text.endswith("州")):
+                break
+
+            if g_strip == "·" and _is_name_piece(getattr(nxt, "text", "")):
+                connectors.append("dot")
+                parts.append(nxt)
+                saw_dot = True
+                j += 1
+                continue
+
+            if (g == "" or _gap_is_ws(g)) and (getattr(nxt, "text", "") == "·") and (j + 1) < len(geo_merged):
+                nxt2 = geo_merged[j + 1]
+                g2 = _gap(getattr(nxt, "end"), getattr(nxt2, "start"))
+                if (g2 == "" or _gap_is_ws(g2)) and _is_name_piece(getattr(nxt2, "text", "")):
+                    connectors.append("dot")
+                    parts.append(nxt2)
+                    saw_dot = True
+                    j += 2
+                    continue
+
+            # Allow limited no-dot concatenation only *within* a dot-name span:
+            # - must already have seen a dot
+            # - and the previous join must have been a dot (immediately-after-dot window)
+            # - and token must be short (name piece), and must not be verbish (checked above)
+            if saw_dot and (g == "" or _gap_is_ws(g)) and _is_name_piece(nxt_text):
+                prev_join = connectors[-1] if connectors else ""
+                if prev_join == "dot" and 1 <= len(nxt_text) <= 3:
+                    connectors.append("ws")
+                    parts.append(nxt)
+                    j += 1
+                    continue
+
+            break
+
+        if saw_dot and len(parts) >= 2:
+            if len(parts) <= 6 and sum(len(getattr(p, "text", "") or "") for p in parts) <= 40:
+                out_txt = getattr(parts[0], "text", "") or ""
+                for k in range(1, len(parts)):
+                    conn = connectors[k - 1] if (k - 1) < len(connectors) else "ws"
+                    if conn == "dot":
+                        out_txt += "·" + (getattr(parts[k], "text", "") or "")
+                    else:
+                        out_txt += (getattr(parts[k], "text", "") or "")
+                merged_names.append(_tok(out_txt, getattr(parts[0], "start"), getattr(parts[-1], "end")))
+                i = j
+                continue
+
+        merged_names.append(t0)
+        i += 1
+
+    # pass 3: merge X + 港 -> X港 (contiguous)
+    out3: List[object] = []
+    i = 0
+    while i < len(merged_names):
+        a = merged_names[i]
+        if i + 1 < len(merged_names):
+            b = merged_names[i + 1]
+            if getattr(a, "end") == getattr(b, "start") and getattr(b, "text", "") == "港":
+                at = getattr(a, "text", "") or ""
+                if at and len(at) <= 12 and all((_is_cjk(ch) or ch.isalnum()) for ch in at):
+                    out3.append(_tok(at + "港", getattr(a, "start"), getattr(b, "end")))
+                    i += 2
+                    continue
+        out3.append(a)
+        i += 1
+
+    return out3
+
+
+def _rule_ja(tokens: List[object], _lang: str, _text: str) -> List[object]:
+    """
+    Japanese compound re-joining (conservative):
+    - Katakana + Kanji suffix merge: モルディン + 港 -> モルディン港
+      suffix candidates: 港/駅/空港/都/道/府/県/市/区/町/村
+    """
+    if not tokens:
+        return tokens
+
+    TokenType = type(tokens[0])
+
+    def _tok(text_: str, start: int, end: int):
+        return TokenType(text=text_, start=start, end=end)
+
+    JA_SUFFIXES = {"港", "駅", "空港", "都", "道", "府", "県", "市", "区", "町", "村"}
+
+    def _is_katakana(ch: str) -> bool:
+        return "\u30a0" <= ch <= "\u30ff"
+
+    def _is_katakana_run(s: str) -> bool:
+        if not s:
+            return False
+        for ch in s:
+            if _is_katakana(ch) or ch in {"ー", "・"}:
+                continue
+            return False
+        return True
+
+    out: List[object] = []
+    i = 0
+    while i < len(tokens):
+        a = tokens[i]
+        if i + 1 < len(tokens):
+            b = tokens[i + 1]
+            if getattr(a, "end") == getattr(b, "start") and _is_katakana_run(getattr(a, "text", "") or "") and (getattr(b, "text", "") in JA_SUFFIXES):
+                at = getattr(a, "text", "") or ""
+                bt = getattr(b, "text", "") or ""
+                if len(at) <= 24 and len(at + bt) <= 28:
+                    out.append(_tok(at + bt, getattr(a, "start"), getattr(b, "end")))
+                    i += 2
+                    continue
+        out.append(a)
+        i += 1
+    return out
+
+
+def _rule_merge_simple_punct_runs(tokens: List[object], _lang: str, text: str) -> List[object]:
+    """
+    Global neutral fix: merge contiguous single-character punctuation tokens into runs.
+
+    Example:
+    - ["!", "!", "!"] -> ["!!!"]
+    - [".", ".", "."] -> ["..."]
+    - ["?", "?", "!", "!"] -> ["??!!"]
+
+    This helps SNS discourse tagging and reduces downstream fragmentation.
+    """
+    if not tokens or len(tokens) < 2:
+        return tokens
+
+    TokenType = type(tokens[0])
+
+    def _tok_from_span(s: int, e: int):
+        return TokenType(text=text[s:e], start=s, end=e)
+
+    P = {"!", "！", "?", "？", ".", "…", "~", "～"}
+
+    out: List[object] = []
+    i = 0
+    n = len(tokens)
+    while i < n:
+        a = tokens[i]
+        at = getattr(a, "text", "") or ""
+        a_s, a_e = int(getattr(a, "start")), int(getattr(a, "end"))
+        if len(at) == 1 and at in P and 0 <= a_s <= a_e <= len(text):
+            j = i + 1
+            end = a_e
+            while j < n:
+                b = tokens[j]
+                bt = getattr(b, "text", "") or ""
+                b_s, b_e = int(getattr(b, "start")), int(getattr(b, "end"))
+                if not (len(bt) == 1 and bt in P and 0 <= b_s <= b_e <= len(text)):
+                    break
+                if b_s != end:
+                    break
+                end = b_e
+                j += 1
+            if j > i + 1:
+                out.append(_tok_from_span(a_s, end))
+                i = j
+                continue
+        out.append(a)
+        i += 1
+    return out
+
+
+def _rule_demesh_hangul_keysmash_inside_token(tokens: List[object], _lang: str, text: str) -> List[object]:
+    """
+    Global neutral fix (demesh):
+    Split tokens that contain an internal Hangul Jamo "keysmash/garble" run.
+
+    Example (noisy SNS):
+      "아ㅣ마ㅓㅣ넣ㄹ아이고" -> ["아", "ㅣ마ㅓㅣ넣ㄹ", "아이고"]
+
+    Safety:
+    - triggers only when the token contains >=3 Hangul Jamo chars (U+3131..U+3163)
+    - requires at least one Hangul syllable char (U+AC00..U+D7AF) in the same token
+    - splits only around the *longest contiguous jamo run* (length>=3)
+    - bounded total token length
+    """
+    if not tokens:
+        return tokens
+
+    TokenType = type(tokens[0])
+
+    def _tok(s: int, e: int):
+        return TokenType(text=text[s:e], start=s, end=e)
+
+    def _is_jamo(ch: str) -> bool:
+        o = ord(ch)
+        return 0x3131 <= o <= 0x3163
+
+    def _is_syllable(ch: str) -> bool:
+        o = ord(ch)
+        return 0xAC00 <= o <= 0xD7AF
+
+    out: List[object] = []
+    for t in tokens:
+        tt = getattr(t, "text", "") or ""
+        a_s, a_e = int(getattr(t, "start")), int(getattr(t, "end"))
+        if not (0 <= a_s <= a_e <= len(text)) or not tt:
+            out.append(t)
+            continue
+        if len(tt) > 48:
+            out.append(t)
+            continue
+
+        jamo_total = sum(1 for ch in tt if _is_jamo(ch))
+        if jamo_total < 3:
+            out.append(t)
+            continue
+        if not any(_is_syllable(ch) for ch in tt):
+            out.append(t)
+            continue
+
+        # find longest contiguous jamo run
+        best = None  # (len, start_idx, end_idx)
+        i = 0
+        while i < len(tt):
+            if not _is_jamo(tt[i]):
+                i += 1
+                continue
+            j = i
+            while j < len(tt) and _is_jamo(tt[j]):
+                j += 1
+            run_len = j - i
+            if run_len >= 3:
+                if best is None or run_len > best[0]:
+                    best = (run_len, i, j)
+            i = j
+
+        if not best:
+            out.append(t)
+            continue
+
+        _, rs, re_ = best
+        # map to absolute spans in original text
+        b_s = a_s + rs
+        b_e = a_s + re_
+        # Keep only meaningful splits; avoid empty tokens
+        if a_s < b_s:
+            out.append(_tok(a_s, b_s))
+        out.append(_tok(b_s, b_e))
+        if b_e < a_e:
+            out.append(_tok(b_e, a_e))
+
+    return [x for x in out if (getattr(x, "text", "") or "")]
+
+
+def _rule_mesh_hangul_keysmash_runs(tokens: List[object], _lang: str, text: str) -> List[object]:
+    """
+    Global neutral fix (mesh):
+    Merge fragmented Hangul keysmash/garble pieces into a single token.
+
+    Example:
+      ["ㅣ", "마", "ㅓㅣ", "넣", "ㄹ"] -> ["ㅣ마ㅓㅣ넣ㄹ"]
+
+    Safety:
+    - merges only contiguous tokens (b.start == a.end)
+    - merges only short pieces (<=6 chars each) and bounded total length
+    - requires >=3 Hangul Jamo chars across the merged run
+    - requires presence of vowel jamo (ㅏ..ㅣ) somewhere in the run
+    - avoids swallowing clear words: stops when it sees a token with >=3 Hangul syllables
+    """
+    if not tokens or len(tokens) < 2:
+        return tokens
+
+    TokenType = type(tokens[0])
+
+    def _tok(s: int, e: int):
+        return TokenType(text=text[s:e], start=s, end=e)
+
+    def _is_jamo(ch: str) -> bool:
+        o = ord(ch)
+        return 0x3131 <= o <= 0x3163
+
+    def _is_vowel_jamo(ch: str) -> bool:
+        o = ord(ch)
+        return 0x314F <= o <= 0x3163  # ㅏ..ㅣ
+
+    def _syllable_count(s: str) -> int:
+        return sum(1 for ch in s if 0xAC00 <= ord(ch) <= 0xD7AF)
+
+    out: List[object] = []
+    i = 0
+    n = len(tokens)
+    while i < n:
+        a = tokens[i]
+        at = getattr(a, "text", "") or ""
+        a_s, a_e = int(getattr(a, "start")), int(getattr(a, "end"))
+        if not (0 <= a_s <= a_e <= len(text)) or not at:
+            out.append(a)
+            i += 1
+            continue
+
+        # start a candidate run only if this token has any jamo or is a tiny mixed piece
+        if len(at) > 6 or (_syllable_count(at) >= 3 and not any(_is_jamo(ch) for ch in at)):
+            out.append(a)
+            i += 1
+            continue
+
+        jamo_cnt = sum(1 for ch in at if _is_jamo(ch))
+        has_vowel = any(_is_vowel_jamo(ch) for ch in at)
+        start = a_s
+        end = a_e
+        j = i + 1
+        parts = [at]
+
+        while j < n:
+            b = tokens[j]
+            bt = getattr(b, "text", "") or ""
+            b_s, b_e = int(getattr(b, "start")), int(getattr(b, "end"))
+            if not bt or not (0 <= b_s <= b_e <= len(text)) or b_s != end:
+                break
+            if len(bt) > 6:
+                break
+            # stop before swallowing a clear multi-syllable word chunk
+            if _syllable_count(bt) >= 3 and not any(_is_jamo(ch) for ch in bt):
+                break
+            # cap merged length
+            if (b_e - start) > 24:
+                break
+            parts.append(bt)
+            end = b_e
+            jamo_cnt += sum(1 for ch in bt if _is_jamo(ch))
+            has_vowel = has_vowel or any(_is_vowel_jamo(ch) for ch in bt)
+            j += 1
+
+        if j > i + 1 and jamo_cnt >= 3 and has_vowel:
+            out.append(_tok(start, end))
+            i = j
+            continue
+
+        out.append(a)
+        i += 1
+
+    return out
+
+
+def _rule_merge_base64_and_heart(tokens: List[object], _lang: str, text: str) -> List[object]:
+    """
+    Global neutral fix:
+    - Re-merge base64/opaque blobs that got split into many tiny punctuation tokens.
+    - Re-merge SNS heart emoticon "<3" if it was split into "<" + "3".
+
+    This is about robustness/UX for "non-sentences", not linguistic correctness.
+    """
+    if not tokens or len(tokens) < 2:
+        return tokens
+
+    TokenType = type(tokens[0])
+
+    def _tok(s: int, e: int):
+        return TokenType(text=text[s:e], start=s, end=e)
+
+    def _is_base64_char(ch: str) -> bool:
+        o = ord(ch)
+        if 48 <= o <= 57 or 65 <= o <= 90 or 97 <= o <= 122:
+            return True
+        return ch in {"+", "/", "="}
+
+    out: List[object] = []
+    i = 0
+    n = len(tokens)
+    while i < n:
+        a = tokens[i]
+        at = getattr(a, "text", "") or ""
+        a_s, a_e = int(getattr(a, "start")), int(getattr(a, "end"))
+        if not at:
+            i += 1
+            continue
+
+        # Merge "<3"
+        if at == "<" and i + 1 < n:
+            b = tokens[i + 1]
+            bt = getattr(b, "text", "") or ""
+            b_s, b_e = int(getattr(b, "start")), int(getattr(b, "end"))
+            if bt == "3" and b_s == a_e:
+                out.append(_tok(a_s, b_e))
+                i += 2
+                continue
+
+        # Merge base64/opaque blob runs split into many tiny tokens.
+        # Conditions:
+        # - contiguous in text (no spaces)
+        # - chars are base64 set [A-Za-z0-9+/=]
+        # - total length bounded
+        # - must contain at least one of "+/=" to avoid swallowing normal words
+        if len(at) <= 128 and all(_is_base64_char(ch) for ch in at) and not any(ch.isspace() for ch in at):
+            start = a_s
+            end = a_e
+            has_sig = any(ch in at for ch in {"+", "/", "="})
+            sig_count = sum(1 for ch in at if ch in {"+", "/", "="})
+            total_len = len(at)
+            j = i + 1
+            while j < n:
+                b = tokens[j]
+                bt = getattr(b, "text", "") or ""
+                b_s, b_e = int(getattr(b, "start")), int(getattr(b, "end"))
+                if not bt or b_s != end:
+                    break
+                if any(ch.isspace() for ch in bt):
+                    break
+                if not all(_is_base64_char(ch) for ch in bt):
+                    break
+                if len(bt) > 128:
+                    break
+                total_len += len(bt)
+                if total_len > 256:
+                    break
+                has_sig = has_sig or any(ch in bt for ch in {"+", "/", "="})
+                sig_count += sum(1 for ch in bt if ch in {"+", "/", "="})
+                end = b_e
+                j += 1
+
+            # Require enough "signal" chars to avoid swallowing normal words like "internationalization"
+            if j > i + 1 and total_len >= 16 and has_sig and sig_count >= 2:
+                out.append(_tok(start, end))
+                i = j
+                continue
+
+        out.append(a)
+        i += 1
+
+    return out
+
+
+def _rule_split_punct_before_social_marker(tokens: List[object], _lang: str, text: str) -> List[object]:
+    """
+    Global neutral fix: split punctuation runs that accidentally swallow a social marker.
+
+    Example:
+    - "!!!#发布会" might yield a token "!!!#" -> split into "!!!" + "#"
+
+    This makes the downstream `_rule_merge_social_handles` effective.
+    """
+    if not tokens:
+        return tokens
+
+    TokenType = type(tokens[0])
+
+    def _tok(s: int, e: int):
+        return TokenType(text=text[s:e], start=s, end=e)
+
+    out: List[object] = []
+    for t in tokens:
+        tt = getattr(t, "text", "") or ""
+        s = int(getattr(t, "start"))
+        e = int(getattr(t, "end"))
+        if not (0 <= s <= e <= len(text)) or len(tt) < 2:
+            out.append(t)
+            continue
+        last = tt[-1]
+        if last in {"#", "@", "$"}:
+            head = tt[:-1]
+            # Only split when head looks like punctuation/emphasis noise.
+            # Keep it conservative to avoid breaking things like "C#".
+            if head and all((not ch.isalnum()) for ch in head):
+                mid = e - 1
+                out.append(_tok(s, mid))
+                out.append(_tok(mid, e))
+                continue
+        out.append(t)
+    return [x for x in out if (getattr(x, "text", "") or "")]
+
+
+def _rule_split_punct_digit_ellipsis_clumps(tokens: List[object], _lang: str, text: str) -> List[object]:
+    """
+    Global neutral fix: split SNS-ish clumps like "!!!23333……" that sometimes appear in corpora.
+
+    This helps downstream SNS marker tagging and avoids treating such clumps as a single token.
+
+    Safety:
+    - only splits when the token is entirely: (punct-run){2,} + (digits){2,} + optional (ellipsis-run){2,}
+    - does not touch normal words or mixed alnum words.
+    """
+    if not tokens:
+        return tokens
+
+    TokenType = type(tokens[0])
+
+    def _tok(s: int, e: int):
+        return TokenType(text=text[s:e], start=s, end=e)
+
+    rx = re.compile(r"^([!！?？~～]{2,})(\d{2,})([.…\.]{2,}|…{2,})?$")
+
+    out: List[object] = []
+    for t in tokens:
+        tt = getattr(t, "text", "") or ""
+        s = int(getattr(t, "start"))
+        e = int(getattr(t, "end"))
+        if not (0 <= s <= e <= len(text)) or len(tt) < 5:
+            out.append(t)
+            continue
+        m = rx.fullmatch(tt)
+        if not m:
+            out.append(t)
+            continue
+        g1, g2, g3 = m.group(1), m.group(2), m.group(3)
+        i1 = s + len(g1)
+        i2 = i1 + len(g2)
+        if not (s < i1 < i2 <= e):
+            out.append(t)
+            continue
+        out.append(_tok(s, i1))
+        out.append(_tok(i1, i2))
+        if g3:
+            i3 = i2 + len(g3)
+            if i2 < i3 <= e:
+                out.append(_tok(i2, i3))
+            else:
+                out.append(_tok(i2, e))
+        continue
+    return [x for x in out if (getattr(x, "text", "") or "")]
+
+
+def _rule_merge_social_handles(tokens: List[object], _lang: str, text: str) -> List[object]:
+    """
+    Global neutral fix: merge contiguous social handles and tags.
+
+    Examples:
+    - "#AI" split as ["#", "AI"] -> ["#AI"]
+    - "@user" split as ["@", "user"] -> ["@user"]
+    - "$TSLA" split as ["$", "TSLA"] -> ["$TSLA"]
+    - "#发布会" split as ["#", "发布", "会"] -> ["#发布会"]
+
+    Safety:
+    - only when contiguous in the original text (b.start == a.end)
+    - tail must be "handle-like" (letters/numbers/marks plus _.- and CJK)
+    - bounded tail length
+    """
+    if not tokens or len(tokens) < 2:
+        return tokens
+
+    TokenType = type(tokens[0])
+
+    def _tok(text_: str, start: int, end: int):
+        return TokenType(text=text_, start=start, end=end)
+
+    def _is_tail_ok(s: str) -> bool:
+        if not s or len(s) > 64:
+            return False
+        for ch in s:
+            if ch.isalnum():
+                continue
+            # allow underscore/dot/hyphen
+            if ch in {"_", ".", "-"}:
+                continue
+            # allow CJK characters in tags
+            o = ord(ch)
+            if 0x4E00 <= o <= 0x9FFF:
+                continue
+            # allow Japanese kana in tags
+            if 0x3040 <= o <= 0x30FF:
+                continue
+            # allow Hangul in tags
+            if 0xAC00 <= o <= 0xD7AF:
+                continue
+            return False
+        return True
+
+    out: List[object] = []
+    i = 0
+    n = len(tokens)
+    while i < n:
+        a = tokens[i]
+        if i + 1 < n:
+            b = tokens[i + 1]
+            at = getattr(a, "text", "") or ""
+            bt = getattr(b, "text", "") or ""
+            if at in {"#", "@", "$"}:
+                a_s, a_e = int(getattr(a, "start")), int(getattr(a, "end"))
+                b_s, b_e = int(getattr(b, "start")), int(getattr(b, "end"))
+                if 0 <= a_s <= a_e <= len(text) and 0 <= b_s <= b_e <= len(text) and b_s == a_e:
+                    # Merge a multi-token tail when it stays contiguous and handle-like.
+                    parts = [bt]
+                    end = b_e
+                    j = i + 2
+                    while j < n:
+                        c = tokens[j]
+                        ct = getattr(c, "text", "") or ""
+                        c_s, c_e = int(getattr(c, "start")), int(getattr(c, "end"))
+                        if c_s != end:
+                            break
+                        # stop if too long
+                        if sum(len(p) for p in parts) + len(ct) > 64:
+                            break
+                        # accept only handle-like chunks
+                        if not _is_tail_ok(ct):
+                            break
+                        parts.append(ct)
+                        end = c_e
+                        j += 1
+                    merged_tail = "".join(parts)
+                    if _is_tail_ok(merged_tail):
+                        out.append(_tok(at + merged_tail, a_s, end))
+                        i = j
+                        continue
+        out.append(a)
+        i += 1
+    return out
+
+
+def _rule_merge_emoji_sequences(tokens: List[object], _lang: str, _text: str) -> List[object]:
+    """
+    Global neutral fix: merge contiguous emoji sequences into a single token.
+
+    Handles common emoji composition characters:
+    - ZWJ (U+200D)
+    - variation selectors (FE0E/FE0F)
+    - skin tone modifiers (U+1F3FB..U+1F3FF)
+    - regional indicators (flags) (U+1F1E6..U+1F1FF)
+
+    Safety:
+    - only merges when tokens are contiguous (a.end == b.start)
+    - only merges tokens that are "emojiish-only"
+    - bounded total length
+    """
+    if not tokens or len(tokens) < 2:
+        return tokens
+
+    TokenType = type(tokens[0])
+
+    def _tok(text_: str, start: int, end: int):
+        return TokenType(text=text_, start=start, end=end)
+
+    def _is_emojiish_char(ch: str) -> bool:
+        o = ord(ch)
+        if ch == "\u200d":  # ZWJ
+            return True
+        if o in {0xFE0E, 0xFE0F}:  # variation selectors
+            return True
+        if 0x1F3FB <= o <= 0x1F3FF:  # skin tone modifiers
+            return True
+        if 0x1F1E6 <= o <= 0x1F1FF:  # regional indicators
+            return True
+        # common emoji blocks
+        if 0x1F300 <= o <= 0x1FAFF:
+            return True
+        # misc symbols / dingbats often used as emoji
+        if 0x2600 <= o <= 0x26FF:
+            return True
+        if 0x2700 <= o <= 0x27BF:
+            return True
+        return False
+
+    def _is_emojiish_token(s: str) -> bool:
+        return bool(s) and all(_is_emojiish_char(ch) for ch in s)
+
+    out: List[object] = []
+    i = 0
+    n = len(tokens)
+    while i < n:
+        a = tokens[i]
+        at = getattr(a, "text", "") or ""
+        if not _is_emojiish_token(at):
+            out.append(a)
+            i += 1
+            continue
+
+        start = int(getattr(a, "start"))
+        end = int(getattr(a, "end"))
+        parts = [at]
+        j = i + 1
+        while j < n:
+            b = tokens[j]
+            bt = getattr(b, "text", "") or ""
+            if not _is_emojiish_token(bt):
+                break
+            if int(getattr(b, "start")) != end:
+                break
+            # cap length to avoid pathological merges
+            if sum(len(p) for p in parts) + len(bt) > 32:
+                break
+            parts.append(bt)
+            end = int(getattr(b, "end"))
+            j += 1
+
+        if j > i + 1:
+            out.append(_tok("".join(parts), start, end))
+            i = j
+            continue
+        out.append(a)
+        i += 1
+    return out
+
+
+def _rule_ko_sns_markers(tokens: List[object], _lang: str, text: str) -> List[object]:
+    """
+    Korean SNS marker splitting (neutral preprocessing):
+    - Split embedded/emergent discourse/emotion markers like ㅋㅋ/ㅎㅎ/ㅠㅠ/ㅜㅜ/ㄷㄷ/ㅇㅇ/ㄹㅇ when they
+      are glued to neighboring words.
+
+    Examples:
+    - "실화냐ㅋㅋ" -> ["실화냐", "ㅋㅋ"]
+    - "아ㅋㅋ진짜" -> ["아", "ㅋㅋ", "진짜"]
+    - "미쳤네ㅠㅠ" -> ["미쳤네", "ㅠㅠ"]
+    - "ㄱㄱ!!!" -> ["ㄱㄱ", "!!!"]  (only splits when punctuation is present)
+
+    Safety:
+    - only triggers for Hangul/Jamo marker runs (rare in formal text)
+    - preserves offsets by slicing the original `text`
+    """
+    if not tokens:
+        return tokens
+
+    ll = (_lang or "").lower().replace("_", "-")
+    if ll != "ko":
+        return tokens
+
+    TokenType = type(tokens[0])
+
+    def _tok(s: int, e: int):
+        return TokenType(text=text[s:e], start=s, end=e)
+
+    # Core SNS marker runs: laughter/cry/surprise/affirmation/emphasis.
+    # - laughter/emotion: ㅋ/ㅎ/ㅠ/ㅜ repeated
+    # - surprise: ㄷㄷ
+    # - affirmation: ㅇㅇ
+    # - emphasis slang: ㄹㅇ, ㅈㄴ, ㅅㅂ, ㅇㅋ etc. (consonant-only runs)
+    import re
+
+    rx_marker = re.compile(r"(?:[ㅋㅎㅠㅜ]{2,}|ㄷ{2,}|ㅇ{2,}|[ㄱ-ㅎ]{2,5})")
+    rx_punct_tail = re.compile(r"[!！?？~…]+$")
+
+    out: List[object] = []
+    for t in tokens:
+        s0 = getattr(t, "text", "") or ""
+        a_s, a_e = int(getattr(t, "start")), int(getattr(t, "end"))
+        if not (0 <= a_s <= a_e <= len(text)) or not s0:
+            out.append(t)
+            continue
+
+        # Split trailing punctuation runs (SNS intensity) when attached to marker-only token.
+        # e.g., "ㄱㄱ!!!" -> "ㄱㄱ" + "!!!"
+        m_tail = rx_punct_tail.search(s0)
+        if m_tail and m_tail.start() > 0:
+            head = s0[: m_tail.start()]
+            tail = s0[m_tail.start() :]
+            # Only do this for jamo/consonant-like heads (avoid splitting normal words like "go!!!")
+            if rx_marker.fullmatch(head):
+                mid = a_s + m_tail.start()
+                out.append(_tok(a_s, mid))
+                out.append(_tok(mid, a_e))
+                continue
+
+        # Find embedded marker runs; if none, keep as-is.
+        hits = list(rx_marker.finditer(s0))
+        if not hits:
+            out.append(t)
+            continue
+
+        # If the whole token is just a marker run, keep as-is (already good token).
+        if len(hits) == 1 and hits[0].start() == 0 and hits[0].end() == len(s0):
+            out.append(t)
+            continue
+
+        # Split into segments using marker spans.
+        cur = 0
+        any_split = False
+        for h in hits:
+            hs, he = h.start(), h.end()
+            if hs > cur:
+                out.append(_tok(a_s + cur, a_s + hs))
+            out.append(_tok(a_s + hs, a_s + he))
+            if hs != 0 or he != len(s0):
+                any_split = True
+            cur = he
+        if cur < len(s0):
+            out.append(_tok(a_s + cur, a_e))
+        if not any_split:
+            # fallback: no meaningful split detected
+            out.append(t)
+
+    # Drop empty tokens defensively
+    out2: List[object] = []
+    for x in out:
+        if (getattr(x, "text", "") or ""):
+            out2.append(x)
+    return out2
+
+
+# Register built-in rules (core defaults)
+register_global_rule(_rule_merge_digit_groups)
+register_global_rule(_rule_demesh_hangul_keysmash_inside_token)
+register_global_rule(_rule_mesh_hangul_keysmash_runs)
+register_global_rule(_rule_merge_simple_punct_runs)
+register_global_rule(_rule_split_punct_before_social_marker)
+register_global_rule(_rule_split_punct_digit_ellipsis_clumps)
+register_global_rule(_rule_merge_base64_and_heart)
+register_global_rule(_rule_merge_social_handles)
+register_global_rule(_rule_merge_emoji_sequences)
+register_lang_rule("ko", _rule_ko_sns_markers)
+register_lang_rule("zh", _rule_zh)
+register_lang_rule("zh-cn", _rule_zh)
+register_lang_rule("zh-tw", _rule_zh)
+register_lang_rule("ja", _rule_ja)
+
+