tokmor 1.2.9__py3-none-any.whl

tokmor/inventory.py

@@ -0,0 +1,51 @@
+"""
+Language inventory (POS-free view)
+==================================
+
+We want an honest, practical view for end users:
+- tokenization works broadly (specialized tokenizers vs fallback)
+- morphology output is always available (at least identity fallback)
+- optional lemma dictionaries may be provided via external assets (TOKMOR_DATA_DIR)
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+from .factory import TOKENIZER_MAP, supported_languages
+from .morphology.unified import unified_supported_languages
+from .resources import resolve_lemma_dict_path
+
+
+def build_language_inventory() -> Dict[str, Any]:
+    toks = supported_languages()
+    morph = unified_supported_languages()
+
+    has_lemma: List[str] = []
+    for lang in toks:
+        if resolve_lemma_dict_path(lang) is not None:
+            has_lemma.append(lang)
+
+    specialized = sorted({k.lower() for k in TOKENIZER_MAP.keys()})
+
+    return {
+        "counts": {
+            "tokenize_languages": len(toks),
+            "morph_languages": len(morph),
+            "lemma_dict_languages": len(has_lemma),
+            "specialized_tokenizers": len(specialized),
+        },
+        "capabilities": {
+            # Deterministic SNS discourse marker hints via segment(..., sns=True, include_sns_tags=True)
+            "sns_discourse_markers": True,
+        },
+        "tokenize_supported": toks,
+        "morph_supported": morph,
+        "lemma_dict_supported": sorted(set(has_lemma)),
+        "specialized_tokenizers": specialized,
+    }
+
+
+
+
+

tokmor/legacy_api.py

@@ -0,0 +1,143 @@
+"""
+Legacy preprocessing API (kept for internal tooling)
+===================================================
+
+This module contains the older POS-free preprocessing functions:
+- tokenize()
+- segment()
+- route()
+
+New integrations should prefer:
+- tokmor.api.unified_tokenize()
+- tokmor.api.ner_preprocess()
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict
+from typing import Any, Dict, List, Literal, Optional, Union
+
+from . import __version__ as _TOKMOR_VERSION
+from .factory import detect_language, get_tokenizer
+from .inventory import build_language_inventory
+from .morphology.unified import get_unified_analyzer
+from .preprocess import normalize_text
+from .lookup_keys import suffixing_latin_keys
+from .routing import route as _route
+from .schema import SCHEMA_VERSION
+
+OutputFormat = Literal["tokens", "tokens_with_offsets"]
+SegmentToken = Dict[str, Any]
+
+
+def languages() -> Dict[str, Any]:
+    return build_language_inventory()
+
+
+def normalize(text: str) -> str:
+    return normalize_text(text)
+
+
+def normalize_sns(text: str) -> str:
+    return normalize_text(text, sns=True)
+
+
+def tokenize(
+    text: str,
+    lang: str = "auto",
+    *,
+    sns: bool = False,
+    morphology: Optional[bool] = None,
+    zh_join_dates: Optional[bool] = None,
+    output: OutputFormat = "tokens",
+) -> Union[List[str], List[Dict[str, Any]]]:
+    text_norm = normalize_text(text, sns=bool(sns))
+    if lang == "auto":
+        lang = detect_language(text_norm)
+
+    if morphology is None:
+        if lang in {"zh", "ja"}:
+            morphology = True
+        elif lang == "ko":
+            morphology = True
+
+    tok = get_tokenizer(lang, use_morphology=morphology, zh_join_dates=zh_join_dates if lang.startswith("zh") else None)
+    res = tok.tokenize(text_norm)
+    if output == "tokens":
+        return res.texts()
+    if output == "tokens_with_offsets":
+        return [asdict(t) for t in res.tokens]
+    raise ValueError(f"unknown output={output}")
+
+
+def segment(
+    text: str,
+    lang: str = "auto",
+    *,
+    sns: bool = False,
+    morphology: Optional[bool] = None,
+    zh_join_dates: Optional[bool] = None,
+    include_morphemes: bool = False,
+    include_keys: bool = False,
+    include_sns_tags: bool = False,
+) -> Dict[str, Any]:
+    text_norm = normalize_text(text, sns=bool(sns))
+    if lang == "auto":
+        lang = detect_language(text_norm)
+
+    if morphology is None:
+        if lang in {"zh", "ja"}:
+            morphology = True
+        elif lang == "ko":
+            morphology = True
+
+    tok = get_tokenizer(lang, use_morphology=morphology, zh_join_dates=zh_join_dates if lang.startswith("zh") else None)
+    res = tok.tokenize(text_norm)
+
+    routing = _route(text_norm, lang=lang)
+    stype = str(routing.get("structure", {}).get("type") or "")
+
+    out_tokens: List[SegmentToken] = []
+    for t in res.tokens:
+        d: SegmentToken = {"text": t.text, "start": t.start, "end": t.end}
+        if include_keys:
+            if stype == "suffixing_latin":
+                d["keys"] = suffixing_latin_keys(t.text, lang=lang)
+            else:
+                d["keys"] = [t.text]
+        if include_sns_tags:
+            from .sns_tags import classify_sns_token
+
+            d["sns"] = classify_sns_token(t.text, lang=lang)
+        out_tokens.append(d)
+
+    return {
+        "schema_version": int(SCHEMA_VERSION),
+        "tokmor_version": str(_TOKMOR_VERSION),
+        "lang": lang,
+        "morphology_used": bool(getattr(res, "morphology_used", False)),
+        "token_count": len(out_tokens),
+        "tokens": out_tokens,
+        "morphemes": (
+            [
+                {"form": r.word, "pos": r.pos, "features": r.features}
+                for r in get_unified_analyzer(lang).analyze(text_norm)
+            ]
+            if include_morphemes
+            else None
+        ),
+    }
+
+
+def route(text: str, lang: str = "auto") -> Dict[str, Any]:
+    text_norm = normalize_text(text)
+    if lang == "auto":
+        lang = detect_language(text_norm)
+    payload = _route(text_norm, lang=lang)
+    return {
+        "schema_version": int(SCHEMA_VERSION),
+        "tokmor_version": str(_TOKMOR_VERSION),
+        "lang": lang,
+        **payload,
+    }
+

tokmor/lemma_store.py

@@ -0,0 +1,102 @@
+"""
+Lemma Store (stdlib-only)
+========================
+
+Goal:
+- very fast lemma lookup for huge lexicons (multi-million+ entries)
+- no external dependencies
+
+Supported backends:
+- Pickle dict: small lexicons (fast but memory heavy)
+- SQLite (sqlite3): large lexicons (fast, low memory)
+"""
+
+from __future__ import annotations
+
+import pickle
+import sqlite3
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+from typing import Dict, Optional, Tuple
+
+
+class BaseLemmaStore:
+    def get(self, key: str) -> Optional[str]:
+        raise NotImplementedError
+
+
+@dataclass
+class PickleLemmaStore(BaseLemmaStore):
+    data: Dict[str, str]
+
+    @classmethod
+    def load(cls, path: Path) -> "PickleLemmaStore":
+        with open(path, "rb") as f:
+            d = pickle.load(f)
+        if not isinstance(d, dict):
+            raise ValueError("Pickle lemma store must be a dict[str,str]")
+        return cls(data=d)
+
+    def get(self, key: str) -> Optional[str]:
+        return self.data.get(key)
+
+
+class SqliteLemmaStore(BaseLemmaStore):
+    """
+    SQLite schema:
+      CREATE TABLE lemma (k TEXT PRIMARY KEY, v TEXT NOT NULL);
+    """
+
+    def __init__(self, path: Path):
+        self.path = Path(path)
+        # check_same_thread=False: allow reuse across threads if needed
+        self._conn = sqlite3.connect(str(self.path), check_same_thread=False)
+        self._conn.row_factory = None
+        # fast read-only settings (safe even for rw db)
+        try:
+            self._conn.execute("PRAGMA journal_mode=OFF;")
+            self._conn.execute("PRAGMA synchronous=OFF;")
+            self._conn.execute("PRAGMA temp_store=MEMORY;")
+            self._conn.execute("PRAGMA cache_size=-20000;")  # ~20MB
+        except Exception:
+            pass
+        self._stmt = self._conn.cursor()
+
+    def close(self) -> None:
+        try:
+            self._stmt.close()
+        except Exception:
+            pass
+        try:
+            self._conn.close()
+        except Exception:
+            pass
+
+    @lru_cache(maxsize=200_000)
+    def get(self, key: str) -> Optional[str]:
+        try:
+            self._stmt.execute("SELECT v FROM lemma WHERE k=? LIMIT 1", (key,))
+            row = self._stmt.fetchone()
+            if not row:
+                return None
+            return row[0]
+        except Exception:
+            return None
+
+
+def load_lemma_store(path: Path) -> BaseLemmaStore:
+    p = Path(path)
+    suf = p.suffix.lower()
+    if suf in (".sqlite", ".db", ".sqlite3"):
+        return SqliteLemmaStore(p)
+    # default: pickle
+    return PickleLemmaStore.load(p)
+
+
+
+
+
+
+
+

tokmor/lookup_keys.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from typing import Iterable, List, Set
+
+
+def _uniq(xs: Iterable[str]) -> List[str]:
+    out: List[str] = []
+    seen: Set[str] = set()
+    for x in xs:
+        if not x:
+            continue
+        if x in seen:
+            continue
+        seen.add(x)
+        out.append(x)
+    return out
+
+
+def _strip_apostrophe(token: str) -> str:
+    # Turkish (and some other Latin-script languages) use apostrophe to separate proper noun and suffix:
+    # Ankara'da, Türkiye'nin, Ali'ye ...
+    if "'" not in token:
+        return token
+    head = token.split("'", 1)[0]
+    if len(head) >= 2:
+        return head
+    return token
+
+
+def _strip_suffix_any(token: str, suffixes: List[str]) -> str:
+    for suf in suffixes:
+        if token.endswith(suf) and len(token) - len(suf) >= 3:
+            return token[: -len(suf)]
+    return token
+
+
+def suffixing_latin_keys(token: str, *, lang: str) -> List[str]:
+    """
+    Conservative lookup keys for suffixing Latin-script languages.
+
+    Goal: help NER/Gazetteer/PMI lookups by removing *very common* clitics/possessives/marker suffixes.
+    Non-goals:
+    - full morphological analysis
+    - lemma recovery (e.g., Finnish consonant gradation)
+    """
+    t0 = token or ""
+    if not t0:
+        return []
+
+    # Only attempt on word-like tokens (keep digits/hyphenated words as-is).
+    # If it's noisy (contains spaces) or too short, keep identity only.
+    if (" " in t0) or (len(t0) < 3):
+        return [t0]
+
+    lang = (lang or "").lower()
+
+    # Start with the surface form.
+    keys: List[str] = [t0]
+
+    # Apostrophe split helps a lot for Turkish proper nouns.
+    ta = _strip_apostrophe(t0)
+    if ta != t0:
+        keys.append(ta)
+
+    # Language-specific conservative strips.
+    # Ordering matters: clitics/possessives first, then heavier case markers.
+    if lang == "fi":
+        # Finnish clitics (very common in text; safe to strip)
+        clitics = ["kin", "kaan", "kään", "han", "hän", "pa", "pä", "ko", "kö"]
+        # Possessives
+        possess = ["nsä", "nsa", "mme", "nne", "ni", "si"]
+        # Case-ish endings (conservative: prefer longer, avoid 1-letter endings)
+        cases = [
+            "ssa", "ssä", "sta", "stä", "lla", "llä", "lta", "ltä", "lle",
+            "na", "nä", "ksi",
+            "tta", "ttä",
+        ]
+
+        t = ta
+        for _ in range(2):
+            t1 = _strip_suffix_any(t, clitics)
+            t2 = _strip_suffix_any(t1, possess)
+            t3 = _strip_suffix_any(t2, cases)
+            if t3 == t:
+                break
+            keys.append(t3)
+            t = t3
+
+    elif lang == "tr":
+        # Turkish common suffix stacks are complex; keep it conservative:
+        # - if apostrophe split happened, it's already a big win for proper nouns.
+        # - also strip a few extremely common locative/ablative markers when present without apostrophe.
+        suffixes = [
+            "daki", "deki",
+            "dan", "den", "tan", "ten",
+            "da", "de", "ta", "te",
+            "lar", "ler",
+        ]
+        t = ta
+        for _ in range(2):
+            t2 = _strip_suffix_any(t, suffixes)
+            if t2 == t:
+                break
+            keys.append(t2)
+            t = t2
+
+    elif lang == "hu":
+        # Hungarian: very conservative subset (case endings are many).
+        suffixes = [
+            "ban", "ben",  # in
+            "ból", "ből", "rol", "ről", "tól", "től",  # from
+            "nak", "nek",  # dative
+            "val", "vel",  # with
+        ]
+        t = ta
+        for _ in range(2):
+            t2 = _strip_suffix_any(t, suffixes)
+            if t2 == t:
+                break
+            keys.append(t2)
+            t = t2
+
+    elif lang == "et":
+        # Estonian: conservative subset.
+        suffixes = [
+            "s",  # inessive is actually -s (often with -sse/-st), but single-letter is risky;
+            # so we don't strip it. Keep longer ones:
+            "sse", "st", "lt", "le", "l",
+            "ga",  # comitative
+        ]
+        # Avoid stripping single-letter suffixes (like "s", "l") here; keep only len>=2
+        suffixes = [s for s in suffixes if len(s) >= 2]
+        t = ta
+        for _ in range(2):
+            t2 = _strip_suffix_any(t, suffixes)
+            if t2 == t:
+                break
+            keys.append(t2)
+            t = t2
+
+    return _uniq(keys)
+
+
+
+

tokmor/models/domain/sentiment/en.json

@@ -0,0 +1,54 @@
+{
+  "version": 1,
+  "lang": "en",
+  "pos": [
+    "good",
+    "great",
+    "awesome",
+    "amazing",
+    "excellent",
+    "fantastic",
+    "nice",
+    "love",
+    "loved",
+    "lovely",
+    "like",
+    "happy",
+    "happiness",
+    "best"
+  ],
+  "neg": [
+    "bad",
+    "terrible",
+    "awful",
+    "horrible",
+    "worst",
+    "hate",
+    "hated",
+    "sad",
+    "angry",
+    "disgusting",
+    "sucks"
+  ],
+  "negators": [
+    "not",
+    "no",
+    "never",
+    "n't"
+  ],
+  "intensifiers": [
+    "very",
+    "really",
+    "so",
+    "super",
+    "extremely"
+  ],
+  "diminishers": [
+    "slightly",
+    "somewhat",
+    "kinda",
+    "kind of",
+    "sort of"
+  ]
+}
+

tokmor/models/domain/sentiment/ko.json

@@ -0,0 +1,52 @@
+{
+  "version": 1,
+  "lang": "ko",
+  "pos": [
+    "좋다",
+    "좋아",
+    "좋음",
+    "좋아요",
+    "최고",
+    "훌륭",
+    "멋지다",
+    "멋져",
+    "사랑",
+    "행복",
+    "기쁘다",
+    "만족"
+  ],
+  "neg": [
+    "나쁘다",
+    "나빠",
+    "싫다",
+    "싫어",
+    "별로",
+    "최악",
+    "짜증",
+    "화나다",
+    "슬프다",
+    "혐오",
+    "불만"
+  ],
+  "negators": [
+    "안",
+    "못",
+    "없다",
+    "없어",
+    "아니",
+    "아니다"
+  ],
+  "intensifiers": [
+    "너무",
+    "진짜",
+    "완전",
+    "엄청",
+    "개"
+  ],
+  "diminishers": [
+    "좀",
+    "약간",
+    "조금"
+  ]
+}
+

tokmor/models/seg_lexicon/zh_extra_dict.json

@@ -0,0 +1,35 @@
+{
+  "蒙特州": "ns",
+  "七名区": "ns",
+  "曼德省": "ns",
+  "举办国": "ns",
+  "苏格兰银行": "nrt",
+  "刑事法院": "nrt",
+  "平奖委员会": "nrt",
+  "就读大学": "nrt",
+  "建筑公司": "nrt",
+  "汽车协会": "nrt",
+  "上调大学": "nrt",
+  "联邦大学": "nrt",
+  "尔奖委员会": "nrt",
+  "富国银行": "nrt",
+  "儿童协会": "nrt",
+  "北京法院": "nrt",
+  "电脑公司": "nrt",
+  "格兰大学": "nrt",
+  "反革命集团": "nrt",
+  "警长协会": "nrt",
+  "宪法委员会": "nrt",
+  "事达公司": "nrt",
+  "花旗集团": "nrt",
+  "大型银行": "nrt",
+  "西方公司": "nrt",
+  "斯坦福大学": "nrt",
+  "美联银行": "nrt",
+  "英国银行": "nrt",
+  "精英集团": "nrt",
+  "犯罪集团": "nrt",
+  "工业集团": "nrt",
+  "调查公司": "nrt",
+  "记者委员会": "nrt"
+}