tokmor 1.2.9__py3-none-any.whl

tokmor/space_based.py

@@ -0,0 +1,272 @@
+"""
+Space-Based Tokenizer
+=====================
+
+공백 기반 언어용 토크나이저 (영어, 유럽어 등)
+"""
+
+import re
+import unicodedata
+from typing import List
+from .base import BaseTokenizer, Token, TokenizerResult
+
+
+class SpaceBasedTokenizer(BaseTokenizer):
+    """
+    공백 기반 토크나이저
+
+    대부분의 언어 (영어, 독일어, 프랑스어, 스페인어 등)
+    """
+
+    SUPPORTED_LANGUAGES = {
+        # Germanic
+        'en', 'de', 'nl', 'sv', 'da', 'no', 'nb', 'nn', 'is', 'af',
+        # Romance
+        'fr', 'es', 'pt', 'it', 'ro', 'ca', 'gl', 'oc',
+        # Slavic
+        'ru', 'uk', 'pl', 'cs', 'sk', 'hr', 'sr', 'bg', 'sl', 'mk', 'be',
+        # Baltic
+        'lv', 'lt',
+        # Finno-Ugric
+        'fi', 'et', 'hu',
+        # Other European
+        'el', 'sq', 'mt', 'eu', 'cy', 'ga',
+        # Turkic
+        'tr', 'az', 'kk', 'uz', 'ky', 'tk', 'ug',
+        # Other
+        'id', 'ms', 'tl', 'vi', 'sw', 'ha', 'yo', 'ig', 'zu', 'am',
+        'mn', 'ka', 'hy',
+        # Indic (space-based but special handling)
+        'hi', 'bn', 'gu', 'pa', 'mr', 'ne', 'si', 'ta', 'te', 'kn', 'ml',
+    }
+
+    # 언어별 특수 패턴
+    LANGUAGE_PATTERNS = {
+        # 독일어: 복합어
+        'de': {
+            'compound_split': True,
+            'preserve_case': True,
+        },
+        # 터키어: 접미사
+        'tr': {
+            'agglutinative': True,
+        },
+        # 핀란드어: 접미사
+        'fi': {
+            'agglutinative': True,
+        },
+        # 헝가리어: 접미사
+        'hu': {
+            'agglutinative': True,
+        },
+    }
+
+    def __init__(self, lang: str, use_morphology: bool = False):
+        super().__init__(lang, use_morphology)
+        self._word_pattern = re.compile(r'\b[\w\-\']+\b', re.UNICODE)
+        # Pre-scan special tokens that should stay intact for downstream normalization/value extraction.
+        self._special_pattern = re.compile(
+            r"(https?://\S+|www\.\S+|[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}"
+            r"|[$€£¥₩]\s*[0-9][0-9,._]*(?:\.[0-9]+)?|[0-9][0-9,._]*(?:\.[0-9]+)?\s*(?:%|％)"
+            r"|\d{4}[-/\.]\d{1,2}[-/\.]\d{1,2})",
+            re.UNICODE,
+        )
+        self._config = self.LANGUAGE_PATTERNS.get(lang, {})
+
+    @staticmethod
+    def _is_word_char(ch: str) -> bool:
+        """
+        Unicode-aware word character definition:
+        - letters (L*)
+        - marks (M*)  (important for Indic/Brahmic scripts)
+        - numbers (N*)
+        - joiners (ZWJ/ZWNJ) used in some scripts (e.g., Sinhala conjuncts)
+        """
+        if not ch:
+            return False
+        # Keep joiners inside tokens to avoid breaking conjuncts (e.g., Sinhala: ක්‍රී)
+        if ch in {"\u200d", "\u200c"}:  # ZWJ / ZWNJ
+            return True
+        try:
+            cat0 = unicodedata.category(ch)[0]
+            return cat0 in {"L", "M", "N"}
+        except Exception:
+            return False
+
+    def _iter_word_spans(self, text: str, covered: List[bool]) -> List[Token]:
+        """
+        Find word-like spans using a unicode category scan.
+        This is more robust than regex word-boundaries (\b) for Indic/Brahmic scripts.
+        """
+        tokens: List[Token] = []
+        i = 0
+        n = len(text)
+        while i < n:
+            if i < n and covered[i]:
+                i += 1
+                continue
+            ch = text[i]
+            if ch.isspace():
+                i += 1
+                continue
+            if not self._is_word_char(ch):
+                i += 1
+                continue
+
+            s = i
+            i += 1
+            while i < n:
+                if covered[i]:
+                    break
+                c = text[i]
+                if self._is_word_char(c):
+                    i += 1
+                    continue
+                # allow in-word apostrophes/hyphens when surrounded by word chars
+                if c in {"'", "’", "-"} and (i + 1) < n:
+                    if self._is_word_char(text[i - 1]) and self._is_word_char(text[i + 1]):
+                        i += 1
+                        continue
+                break
+            e = i
+            if e > s:
+                tokens.append(Token(text=text[s:e], start=s, end=e))
+            else:
+                i += 1
+        return tokens
+
+    def tokenize(self, text: str) -> TokenizerResult:
+        """공백 기반 토크나이징"""
+        text = self.clean_text(text)
+        if not text:
+            return TokenizerResult(tokens=[], text=text, lang=self.lang)
+
+        tokens = []
+
+        # 0) Special tokens (URLs/emails/money/percent/dates) first
+        covered = [False] * (len(text) + 1)
+        for m in self._special_pattern.finditer(text):
+            s, e = m.start(), m.end()
+            if s < 0 or e <= s:
+                continue
+            tokens.append(Token(text=m.group(), start=s, end=e))
+            for i in range(s, min(e, len(text))):
+                covered[i] = True
+
+        # 1) Unicode-aware word scan (robust for Indic scripts and generally safe)
+        word_tokens = self._iter_word_spans(text, covered)
+        tokens.extend(word_tokens)
+        for t in word_tokens:
+            for i in range(int(t.start), min(int(t.end), len(text))):
+                covered[i] = True
+
+        # 2) Keep remaining visible symbols as single-char tokens.
+        # This improves SNS/user-generated text handling (emoji, hashtags symbols, punctuation),
+        # while remaining deterministic and easy to filter downstream.
+        for i, ch in enumerate(text):
+            if covered[i]:
+                continue
+            if ch.isspace():
+                continue
+            tokens.append(Token(text=ch, start=i, end=i + 1))
+
+        # 형태소 분석 적용
+        if self.use_morphology and self._morphology_analyzer:
+            tokens = self._apply_morphology(text, tokens)
+
+        tokens.sort(key=lambda t: t.start)
+        return TokenizerResult(
+            tokens=tokens,
+            text=text,
+            lang=self.lang,
+            morphology_used=self.use_morphology and self._morphology_analyzer is not None
+        )
+
+    def _apply_morphology(self, text: str, tokens: List[Token]) -> List[Token]:
+        """형태소 분석 적용"""
+        if not self._morphology_analyzer:
+            return tokens
+
+        analyzed = self._morphology_analyzer.analyze(text)
+
+        # Match analyzed results with tokens
+        # (simplified - assumes same tokenization)
+        for i, token in enumerate(tokens):
+            if i < len(analyzed):
+                token.lemma = analyzed[i].lemma
+                token.pos = analyzed[i].pos
+
+        return tokens
+
+
+class EnglishTokenizer(SpaceBasedTokenizer):
+    """영어 특화 토크나이저"""
+
+    SUPPORTED_LANGUAGES = {'en'}
+
+    # 축약형 패턴
+    CONTRACTIONS = {
+        "n't": " not",
+        "'re": " are",
+        "'ve": " have",
+        "'ll": " will",
+        "'d": " would",
+        "'m": " am",
+        "'s": " is",  # or possessive
+    }
+
+    def __init__(self, lang: str = 'en', use_morphology: bool = False):
+        super().__init__(lang, use_morphology)
+        self._contraction_pattern = re.compile(
+            r"(\w+)(n't|'re|'ve|'ll|'d|'m|'s)",
+            re.IGNORECASE
+        )
+
+    def tokenize(self, text: str, expand_contractions: bool = False) -> TokenizerResult:
+        """영어 토크나이징"""
+        text = self.clean_text(text)
+
+        if expand_contractions:
+            text = self._expand_contractions(text)
+
+        return super().tokenize(text)
+
+    def _expand_contractions(self, text: str) -> str:
+        """축약형 확장"""
+        def replace(match):
+            word = match.group(1)
+            contraction = match.group(2).lower()
+            expansion = self.CONTRACTIONS.get(contraction, contraction)
+            return word + expansion
+
+        return self._contraction_pattern.sub(replace, text)
+
+
+class GermanTokenizer(SpaceBasedTokenizer):
+    """독일어 특화 토크나이저"""
+
+    SUPPORTED_LANGUAGES = {'de'}
+
+    def __init__(self, lang: str = 'de', use_morphology: bool = False):
+        super().__init__(lang, use_morphology)
+
+    def split_compound(self, word: str) -> List[str]:
+        """
+        독일어 복합어 분리 (휴리스틱)
+
+        Note: 완전한 분리를 위해서는 형태소 분석기 필요
+        """
+        # 기본적인 복합어 패턴
+        # 실제로는 사전 기반이나 ML 기반 분리가 필요
+        return [word]  # 기본 구현은 분리 안 함
+
+
+class RussianTokenizer(SpaceBasedTokenizer):
+    """러시아어 특화 토크나이저"""
+
+    SUPPORTED_LANGUAGES = {'ru'}
+
+    def __init__(self, lang: str = 'ru', use_morphology: bool = False):
+        super().__init__(lang, use_morphology)
+        # 키릴 문자 포함 패턴
+        self._word_pattern = re.compile(r'[а-яА-ЯёЁa-zA-Z0-9]+', re.UNICODE)