tokmor 1.2.9__py3-none-any.whl

tokmor/brahmic.py

@@ -0,0 +1,516 @@
+"""
+Brahmic Tokenizer (No External Dependencies)
+=============================================
+
+태국어, 라오어, 미얀마어, 크메르어용 토크나이저
+외부 라이브러리 없이 순수 Python으로 구현
+"""
+
+from __future__ import annotations
+
+import math
+import re
+import unicodedata
+from typing import List, Set, Dict, Optional
+from .base import BaseTokenizer, Token, TokenizerResult, MorphologicalAnalyzer
+from .resources import resolve_seg_lexicon_path, resolve_sea_wordlist_path
+from .morphology.thai_native import ThaiNativeAnalyzer
+
+
+# ============================================================
+# Built-in seed dictionaries (REMOVED in OSS core)
+# ============================================================
+#
+# OSS policy:
+# - Do not embed language wordlists in the core code distribution.
+# - For SEA no-space tokenization quality, provide optional offline assets via:
+#   TOKMOR_DATA_DIR/seg_lexicon/{lang}_wordfreq.pkl  and/or {lang}_wordlist.(pkl|txt)
+#
+# Keep these as empty sets so the tokenizer still works, but quality depends on assets.
+THAI_DICT: Set[str] = set()
+LAO_DICT: Set[str] = set()
+MYANMAR_DICT: Set[str] = set()
+KHMER_DICT: Set[str] = set()
+
+
+class BrahmicTokenizer(BaseTokenizer):
+    """
+    Brahmic 스크립트 토크나이저 (No External Dependencies)
+    """
+
+    SUPPORTED_LANGUAGES = {'th', 'lo', 'my', 'km'}
+
+    # Unicode ranges
+    THAI = '\u0e00-\u0e7f'
+    LAO = '\u0e80-\u0eff'
+    MYANMAR = '\u1000-\u109f'
+    KHMER = '\u1780-\u17ff'
+
+    # 언어별 사전
+    DICTIONARIES = {
+        'th': THAI_DICT,
+        'lo': LAO_DICT,
+        'my': MYANMAR_DICT,
+        'km': KHMER_DICT,
+    }
+
+    def __init__(self, lang: str, use_morphology: bool = False):
+        super().__init__(lang, use_morphology)
+        self._setup_patterns()
+        # Keep a copy of the small built-in dictionary (high-precision).
+        self._builtin_dictionary: Set[str] = set(self.DICTIONARIES.get(lang, set()))
+        self._dictionary = set(self._builtin_dictionary)
+        self._max_word_len = max(len(w) for w in self._dictionary) if self._dictionary else 20
+        self._wordfreq: Optional[Dict[str, int]] = None
+        self._wordfreq_max_len: int = 2
+        self._wordlist_size: int = 0
+        self._load_seg_lexicon()
+        self._load_wordlist()
+        self._max_word_len = max(len(w) for w in self._dictionary) if self._dictionary else 20
+
+    def _load_wordlist(self) -> None:
+        """
+        Optional SEA tokenizer wordlist (offline), used for longest-match segmentation.
+
+        File:
+          seg_lexicon/{lang}_wordlist.pkl (set[str]) or .txt (one token per line)
+        """
+        p = resolve_sea_wordlist_path(self.lang)
+        if not p:
+            return
+        try:
+            words: set[str] = set()
+            if p.suffix.lower() == ".pkl":
+                import pickle
+                obj = pickle.loads(p.read_bytes())
+                if isinstance(obj, set):
+                    words = {w for w in obj if isinstance(w, str) and w}
+                elif isinstance(obj, dict):
+                    # allow {word:freq} too
+                    words = {w for w in obj.keys() if isinstance(w, str) and w}
+            else:
+                # txt
+                for line in p.read_text(encoding="utf-8", errors="ignore").splitlines():
+                    w = line.strip()
+                    if w:
+                        words.add(w)
+            if not words:
+                return
+            # Filter very short entries for km/lo/my to avoid syllable-chunking
+            if self.lang in {"km", "lo", "my"}:
+                words = {w for w in words if len(w) >= 2}
+            self._wordlist_size = len(words)
+            self._dictionary = set(self._dictionary) | set(words)
+        except Exception:
+            return
+
+    def _load_seg_lexicon(self) -> None:
+        """
+        Optional segmentation lexicon for no-space scripts:
+          seg_lexicon/{lang}_wordfreq.pkl  (dict[str,int])
+        """
+        p = resolve_seg_lexicon_path(self.lang)
+        if not p:
+            return
+        try:
+            import pickle
+            obj = pickle.loads(p.read_bytes())
+            if not isinstance(obj, dict):
+                return
+            wf: Dict[str, int] = {}
+            mx = 1
+            for k, v in obj.items():
+                if isinstance(k, str) and k and isinstance(v, int) and v > 0:
+                    # For SEA scripts (km/lo/my), 2-gram frequencies tend to be too "syllable-like"
+                    # and can cause degenerate segmentation into short chunks.
+                    # Filter out very short entries to keep Viterbi candidates more word-like.
+                    if self.lang in {"km", "lo", "my"} and len(k) < 3:
+                        continue
+                    wf[k] = v
+                    if len(k) > mx:
+                        mx = len(k)
+            if wf:
+                # keep bounds conservative (Thai/Lao/Myanmar/Khmer words rarely exceed 12 chars)
+                self._wordfreq = wf
+                self._wordfreq_max_len = max(2, min(int(mx), 12))
+        except Exception:
+            return
+
+    def _setup_patterns(self):
+        pattern_map = {
+            'th': self.THAI,
+            'lo': self.LAO,
+            'my': self.MYANMAR,
+            'km': self.KHMER,
+        }
+        script_range = pattern_map.get(self.lang, self.THAI)
+        self._script_pattern = re.compile(f'[{script_range}]+')
+        self._latin_pattern = re.compile(r'[a-zA-Z0-9]+')
+
+    def _init_morphology(self):
+        # 외부 의존 제거 - 내장 사전 사용
+        self._morphology_analyzer = None
+
+    def tokenize(self, text: str) -> TokenizerResult:
+        text = self.clean_text(text)
+        if not text:
+            return TokenizerResult(tokens=[], text=text, lang=self.lang)
+
+        tokens: List[Token] = []
+
+        for match in self._script_pattern.finditer(text):
+            chunk = match.group()
+            # NOTE:
+            # For km/lo/my, naive n-gram wordfreq lexicons tend to over-segment into short chunks.
+            # Until we have a word-level lexicon, keep the robust longest-match+unknown-grouping path.
+            # For Thai:
+            # - If we have a big token-level wordlist, prefer longest-match with that wordlist (more word-like).
+            # - Otherwise, fall back to Viterbi over wordfreq (may be n-gramy, but still better than tiny dict).
+            # Split native digit runs inside the script chunk BEFORE segmentation.
+            # This prevents mixed letter+digit tokens (e.g., Myanmar: 'က၂' + '၀၂၅') and keeps digits intact.
+            start0 = match.start()
+            cur = 0
+            for part in self._split_native_digit_runs(chunk):
+                if not part:
+                    continue
+                if self._is_native_digit_token(part):
+                    tokens.append(Token(text=part, start=start0 + cur, end=start0 + cur + len(part)))
+                    cur += len(part)
+                    continue
+                if self._wordfreq and self.lang == "th" and self._wordlist_size <= 0:
+                    part_tokens = self._segment_viterbi(part)
+                else:
+                    part_tokens = self._segment_longest_match(part)
+                for t in part_tokens:
+                    if not t:
+                        continue
+                    tokens.append(Token(text=t, start=start0 + cur, end=start0 + cur + len(t)))
+                    cur += len(t)
+
+        # 라틴/숫자
+        for match in self._latin_pattern.finditer(text):
+            overlaps = any(
+                t.start <= match.start() < t.end or t.start < match.end() <= t.end
+                for t in tokens
+            )
+            if not overlaps:
+                tokens.append(Token(text=match.group(), start=match.start(), end=match.end()))
+
+        # punctuation / symbols: include as single-char tokens (helps SBD & downstream)
+        # Only add chars not already covered by script/latin tokens.
+        covered = [False] * (len(text) + 1)
+        for t in tokens:
+            s = max(0, min(len(text), int(t.start)))
+            e = max(0, min(len(text), int(t.end)))
+            for i in range(s, e):
+                covered[i] = True
+        for i, ch in enumerate(text):
+            if covered[i] or ch.isspace():
+                continue
+            # skip characters that are part of our main scripts (should have been covered)
+            if self._script_pattern.match(ch) or self._latin_pattern.match(ch):
+                continue
+            tokens.append(Token(text=ch, start=i, end=i + 1))
+
+        tokens.sort(key=lambda t: t.start)
+        tokens = self._postprocess_marks_and_digits(tokens, text)
+        return TokenizerResult(tokens=tokens, text=text, lang=self.lang, morphology_used=False)
+
+    def _postprocess_marks_and_digits(self, tokens: List[Token], text: str) -> List[Token]:
+        """
+        Postprocess for SEA scripts:
+        - Never allow a token to start with a combining mark (Mn/Mc/Me). If it does, merge into previous token
+          when contiguous. This fixes cases like Khmer coeng/virama or Myanmar vowel signs splitting.
+        - Merge contiguous native-digit runs (Thai/Lao/Myanmar/Khmer digits) into a single token to avoid
+          'per-digit' fragmentation.
+        """
+        if not tokens:
+            return tokens
+
+        def _is_mark(ch: str) -> bool:
+            try:
+                return unicodedata.category(ch) in {"Mn", "Mc", "Me"}
+            except Exception:
+                return False
+
+        # (native digit helpers are shared with pre-seg split)
+
+        out: List[Token] = []
+        for t in tokens:
+            if out:
+                prev = out[-1]
+                # Merge leading combining marks into previous token if contiguous
+                if t.text and _is_mark(t.text[0]) and prev.end == t.start:
+                    prev.text += t.text
+                    prev.end = t.end
+                    continue
+                # Merge native-digit runs if contiguous (e.g., Myanmar digits)
+                if self._is_native_digit_token(prev.text) and self._is_native_digit_token(t.text) and prev.end == t.start:
+                    prev.text += t.text
+                    prev.end = t.end
+                    continue
+            out.append(t)
+
+        return out
+
+    def _segment_longest_match(self, text: str) -> List[str]:
+        """
+        Longest Matching 알고리즘 (improved fallback).
+
+        Key change vs previous version:
+        - If no dictionary match, do NOT emit single-character tokens (degenerate).
+          Instead, group an "unknown span" until the next plausible dictionary boundary,
+          capped to a reasonable max length.
+        """
+        tokens = []
+        pos = 0
+
+        while pos < len(text):
+            # Length-first longest match (product-safe).
+            # Use wordfreq only as a tie-breaker among same-length candidates.
+            best_match = None
+            maxL = min(self._max_word_len, len(text) - pos)
+            wf = self._wordfreq or {}
+            for length in range(maxL, 0, -1):
+                best_freq = -1
+                cand = None
+                candidate = text[pos:pos + length]
+                if candidate in self._dictionary:
+                    cand = candidate
+                    best_freq = wf.get(candidate, 0)
+                # Optionally allow a small set of high-precision builtins to win even if freq is missing.
+                if cand:
+                    best_match = cand
+                    # Prefer built-in dictionary compounds for Thai (e.g., ประเทศไทย, มาตรการ).
+                    if self.lang == "th" and cand in self._builtin_dictionary:
+                        break
+                    # Otherwise, accept the first (longest) match.
+                    break
+
+            if best_match:
+                tokens.append(best_match)
+                pos += len(best_match)
+            else:
+                # Unknown: group forward until we hit a position that can start a known word,
+                # or until we reach a max span length.
+                max_unknown = 8 if self.lang in {"th", "lo"} else 10
+                end_pos = min(len(text), pos + 1)
+                while end_pos < len(text) and (end_pos - pos) < max_unknown:
+                    # stop if the remainder begins with a dictionary word
+                    found = False
+                    for length in range(min(self._max_word_len, len(text) - end_pos), 1, -1):
+                        if text[end_pos:end_pos + length] in self._dictionary:
+                            found = True
+                            break
+                    if found:
+                        break
+                    end_pos += 1
+                if end_pos <= pos:
+                    end_pos = pos + 1
+                tokens.append(text[pos:end_pos])
+                pos = end_pos
+
+        return tokens
+
+    def _is_native_digit_char(self, ch: str) -> bool:
+        o = ord(ch)
+        if self.lang == "th":
+            return 0x0E50 <= o <= 0x0E59
+        if self.lang == "lo":
+            return 0x0ED0 <= o <= 0x0ED9
+        if self.lang == "my":
+            return 0x1040 <= o <= 0x1049
+        if self.lang == "km":
+            return 0x17E0 <= o <= 0x17E9
+        return False
+
+    def _is_native_digit_token(self, s: str) -> bool:
+        return bool(s) and all(self._is_native_digit_char(c) for c in s)
+
+    def _split_native_digit_runs(self, s: str) -> List[str]:
+        """
+        Split a SEA-script run into alternating [letters] / [native-digit-runs].
+        Digits are kept as-is to preserve offsets.
+        """
+        if not s:
+            return []
+        out: List[str] = []
+        i = 0
+        n = len(s)
+        while i < n:
+            ch = s[i]
+            if self._is_native_digit_char(ch):
+                j = i + 1
+                while j < n and self._is_native_digit_char(s[j]):
+                    j += 1
+                out.append(s[i:j])
+                i = j
+            else:
+                j = i + 1
+                while j < n and (not self._is_native_digit_char(s[j])):
+                    j += 1
+                out.append(s[i:j])
+                i = j
+        return out
+
+    def _segment_viterbi(self, run: str) -> List[str]:
+        """
+        Viterbi segmentation over a pure-script run (Thai/Lao/Myanmar/Khmer).
+
+        Candidates are any substrings; scoring prefers:
+        - known words from wordfreq lexicon (coverage)
+        - words from small built-in dictionary (precision)
+        - multi-char groupings over per-char tokens (anti-degenerate)
+        """
+        wf = self._wordfreq or {}
+        max_len = max(self._max_word_len, self._wordfreq_max_len)
+        max_len = max(2, min(int(max_len), 12))
+        n = len(run)
+        if n <= 0:
+            return []
+
+        # Tuned to avoid per-character segmentation even when lexicon is sparse.
+        len_bonus = 0.55
+        single_penalty = 1.0
+        dict_bonus = 1.8
+        unk_base = -1.2
+        unk_len_penalty = 0.25
+
+        best = [-1e100] * (n + 1)
+        back = [-1] * (n + 1)
+        back_len = [1] * (n + 1)
+        best[0] = 0.0
+
+        for i in range(n):
+            if best[i] <= -1e90:
+                continue
+            maxL = min(max_len, n - i)
+            for L in range(1, maxL + 1):
+                w = run[i:i + L]
+                freq = wf.get(w, 0)
+                in_dict = w in self._dictionary
+                sc = best[i]
+                if in_dict or freq > 0:
+                    sc += math.log(float(freq) + 1.0)
+                else:
+                    sc += unk_base - unk_len_penalty * L
+                sc += len_bonus * (L - 1)
+                if L == 1:
+                    sc -= single_penalty
+                if in_dict:
+                    sc += dict_bonus
+                j = i + L
+                if sc > best[j]:
+                    best[j] = sc
+                    back[j] = i
+                    back_len[j] = L
+
+        if back[n] < 0:
+            # fallback safety: never return per-char; use unknown grouping
+            return self._segment_longest_match(run)
+
+        spans = []
+        j = n
+        while j > 0:
+            i = back[j]
+            if i < 0:
+                i = j - 1
+            spans.append((i, j))
+            j = i
+        spans.reverse()
+        return [run[i:j] for i, j in spans]
+
+    def extract_ngrams(self, text: str, min_n: int = 2, max_n: int = 8) -> List[str]:
+        ngrams = []
+        for match in self._script_pattern.finditer(text):
+            chunk = match.group()
+            for n in range(min_n, min(max_n + 1, len(chunk) + 1)):
+                for i in range(len(chunk) - n + 1):
+                    ngrams.append(chunk[i:i+n])
+        return ngrams
+
+
+# 언어별 특화 클래스
+class ThaiTokenizer(BrahmicTokenizer):
+    SUPPORTED_LANGUAGES = {'th'}
+    def __init__(self, use_morphology: bool = False):
+        super().__init__('th', use_morphology)
+        # Ensure the native analyzer is available even when use_morphology=False,
+        # so Thai segmentation stays high-quality by default.
+        if getattr(self, "_morphology_analyzer", None) is None:
+            try:
+                self._morphology_analyzer = ThaiNativeAnalyzer()
+            except Exception:
+                self._morphology_analyzer = None
+
+    def _init_morphology(self):
+        # Native Thai analyzer (offline) provides better segmentation than the tiny dictionary.
+        try:
+            self._morphology_analyzer = ThaiNativeAnalyzer()
+        except Exception:
+            self._morphology_analyzer = None
+
+    def tokenize(self, text: str) -> TokenizerResult:
+        """
+        For Thai, always prefer the internal native segmenter (offline) if available.
+        This is a tokenizer-quality feature, not an external dependency.
+
+        - If morphology is enabled: include lemma/pos where possible and mark morphology_used=True.
+        - If morphology is disabled: return tokens only (no lemma/pos) but still use the better segmenter.
+        """
+        text = self.clean_text(text)
+        if not text:
+            return TokenizerResult(tokens=[], text=text, lang=self.lang)
+
+        # For Thai, prefer the corpus-derived wordlist longest-match path when available.
+        # This tends to produce more word-like tokens on real text than the tiny native dictionary.
+        if (not self.use_morphology) and getattr(self, "_wordlist_size", 0) > 0:
+            return super().tokenize(text)
+
+        if self._morphology_analyzer is not None:
+            try:
+                morps = self._morphology_analyzer.analyze(text)
+                toks: List[Token] = []
+                for m in morps:
+                    if self.use_morphology:
+                        toks.append(Token(text=m.surface, start=m.start, end=m.end, lemma=m.lemma, pos=m.pos))
+                    else:
+                        toks.append(Token(text=m.surface, start=m.start, end=m.end))
+                # include non-covered punctuation/symbols
+                covered = [False] * (len(text) + 1)
+                for t in toks:
+                    for i in range(max(0, t.start), min(len(text), t.end)):
+                        covered[i] = True
+                for i, ch in enumerate(text):
+                    if covered[i] or ch.isspace():
+                        continue
+                    toks.append(Token(text=ch, start=i, end=i + 1))
+                toks.sort(key=lambda t: t.start)
+                return TokenizerResult(tokens=toks, text=text, lang=self.lang, morphology_used=bool(self.use_morphology))
+            except Exception:
+                # fallback to base behavior
+                pass
+
+        return super().tokenize(text)
+
+
+class LaoTokenizer(BrahmicTokenizer):
+    SUPPORTED_LANGUAGES = {'lo'}
+    def __init__(self, use_morphology: bool = False):
+        super().__init__('lo', use_morphology)
+
+
+class MyanmarTokenizer(BrahmicTokenizer):
+    SUPPORTED_LANGUAGES = {'my'}
+    def __init__(self, use_morphology: bool = False):
+        super().__init__('my', use_morphology)
+
+
+class KhmerTokenizer(BrahmicTokenizer):
+    SUPPORTED_LANGUAGES = {'km'}
+    def __init__(self, use_morphology: bool = False):
+        super().__init__('km', use_morphology)
+
+
+# ThaiMorphologyAnalyzer 제거 (외부 의존 제거)
+# pythainlp, attacut 필요 없음