tokmor 1.2.9__py3-none-any.whl

tokmor/offline.py

@@ -0,0 +1,89 @@
+"""
+Offline enforcement
+===================
+
+TokMor is designed to run fully offline at runtime.
+
+This module hard-blocks any attempt to opt into online / remote behaviors via
+TokMor-related environment variables.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Iterable
+
+
+_TRUTHY = {"1", "true", "yes", "y", "on"}
+
+
+def _is_truthy(v: str | None) -> bool:
+    if v is None:
+        return False
+    return v.strip().lower() in _TRUTHY
+
+
+_ALLOWED_TOKMOR_ENV = {
+    # resource routing
+    "TOKMOR_DATA_DIR",
+    "TOKMOR_MODELS_DIR",
+    "TOKMOR_LEMMA_DICT_DIR",
+    "TOKMOR_INFLECT_DIR",
+    "TOKMOR_INFLECT_KAIKKI_DIR",
+    "TOKMOR_INFLECT_UNIMORPH_DIR",
+    "TOKMOR_COARSE_MODELS_DIR",
+    # runtime toggles (only disabling things is allowed)
+    "TOKMOR_DISABLE_ML",
+    "TOKMOR_DISABLE_LEMMA_PACK",
+    "TOKMOR_DISABLE_EXTENDED_DICT",
+    "TOKMOR_DISABLE_DOMAIN_LEXICONS",
+    # quality knobs (offline-safe; local-only computation)
+    "TOKMOR_COARSE_HYBRID",
+    "TOKMOR_COARSE_HYBRID_MIN_PROB",
+    # language-specific behavior knobs
+    "TOKMOR_KO_LEMMA_STYLE",
+    "TOKMOR_ZH_JOIN_DATES",
+}
+
+
+def _iter_blocked_env_names() -> Iterable[str]:
+    """
+    Block any TokMor-scoped env vars that are not explicitly allowed.
+    This prevents any accidental opt-in flags from being introduced via env vars.
+    """
+    for k in os.environ.keys():
+        ku = k.upper()
+        if not ku.startswith("TOKMOR_"):
+            continue
+        if ku not in _ALLOWED_TOKMOR_ENV:
+            yield k
+
+
+def enforce_offline() -> None:
+    """
+    Enforce offline-only runtime.
+
+    Behavior:
+    - If any non-allowed TOKMOR_* env var is set (non-empty and not an explicit false) -> raise RuntimeError.
+    - This is intentionally strict to prevent accidental online/remote fallbacks.
+    """
+    offenders = []
+    for k in _iter_blocked_env_names():
+        v = os.getenv(k)
+        if v is None:
+            continue
+        vs = v.strip().lower()
+        if not vs:
+            continue
+        if vs in ("0", "false", "no", "n", "off"):
+            continue
+        if _is_truthy(v) or vs:
+            offenders.append(k)
+    if offenders:
+        offenders = sorted(set(offenders))
+        raise RuntimeError(
+            "TokMor is offline-only. Remove/disable non-allowed TOKMOR_* env vars: "
+            + ", ".join(offenders)
+        )
+
+

tokmor/preprocess.py

@@ -0,0 +1,80 @@
+"""
+Preprocessing utilities (POS-free)
+==================================
+
+TokMor's core product direction: fast preprocessing utilities that are useful even
+without POS tagging or PPMI.
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from typing import List
+
+from .base import BaseTokenizer
+
+
+def normalize_text(text: str, *, sns: bool = False) -> str:
+    """
+    Conservative normalization:
+    - Unicode NFC
+    - Strip control chars
+    - Normalize newlines
+    - Collapse excessive whitespace
+    """
+    if text is None:
+        return ""
+    text = BaseTokenizer.clean_text(str(text))
+
+    # SNS-friendly normalization (still deterministic, no language assumptions):
+    # - normalize common fullwidth ASCII variants (＃＠＄％ etc.) so downstream patterns work.
+    if sns:
+        # Convert fullwidth ASCII range FF01-FF5E to ASCII 21-7E.
+        # This is a common SNS normalization and improves URL/@/# detection.
+        def _fw_to_ascii(ch: str) -> str:
+            o = ord(ch)
+            if 0xFF01 <= o <= 0xFF5E:
+                return chr(o - 0xFEE0)
+            return ch
+
+        text = "".join(_fw_to_ascii(ch) for ch in text)
+        # Normalize a few compatibility spaces often seen in SNS copy/paste.
+        text = text.replace("\u3000", " ")  # IDEOGRAPHIC SPACE
+        # Keep NFC, but avoid NFKC here to remain conservative.
+        text = unicodedata.normalize("NFC", text)
+    # Normalize newlines
+    text = text.replace("\r\n", "\n").replace("\r", "\n")
+    # Collapse spaces/tabs but keep newlines
+    text = re.sub(r"[ \t\f\v]+", " ", text)
+    # Trim trailing spaces per line
+    text = "\n".join([ln.strip() for ln in text.split("\n")])
+    # Collapse multiple blank lines
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip()
+
+
+def chunk_tokens(tokens: List[str], *, max_tokens: int, overlap: int = 0) -> List[List[str]]:
+    """
+    Chunk a token sequence into overlapping windows.
+    """
+    if max_tokens <= 0:
+        raise ValueError("max_tokens must be > 0")
+    if overlap < 0:
+        raise ValueError("overlap must be >= 0")
+    if overlap >= max_tokens:
+        raise ValueError("overlap must be < max_tokens")
+
+    out: List[List[str]] = []
+    i = 0
+    n = len(tokens)
+    step = max_tokens - overlap
+    while i < n:
+        out.append(tokens[i : i + max_tokens])
+        i += step
+    return out
+
+
+
+
+

tokmor/resources.py

@@ -0,0 +1,288 @@
+"""
+TokMor Resource Manager
+======================
+
+OSS Core 핵심:
+- 코드(wheel)와 데이터(assets)를 분리한다.
+- 운영에서는 외부 디렉토리(TOKMOR_DATA_DIR)로 리소스를 교체/확장한다.
+
+디렉토리 구조(기본):
+  {data_dir}/
+    lemma_dict/     # 제품 lemma lexicon: {lang}_lemma.pkl or {lang}.pkl
+    seg_lexicon/    # segmentation lexicons (zh/SEA 등)
+    domain/         # optional domain lexicons (e.g., sentiment)
+"""
+
+from __future__ import annotations
+
+import os
+from functools import lru_cache
+from pathlib import Path
+from typing import Optional
+
+
+def _package_models_dir() -> Path:
+    # NOTE: OSS core does not bundle large assets in the wheel/sdist.
+    # This path is used only if a downstream package/vendor bundles assets alongside code.
+    return Path(__file__).parent / "models"
+
+
+def normalize_lang_for_models(lang: str) -> str:
+    """
+    Map Wikipedia-style codes (top100 list) to the closest available model code.
+    This is a pragmatic compatibility shim to make '100 languages' run end-to-end.
+    """
+    l = (lang or "").lower().replace("_", "-")
+    # direct aliases
+    alias = {
+        "simple": "en",
+        # ISO / wiki variants
+        "zh-cn": "zh",
+        "zh-tw": "zh",
+        # top100 mismatches / approximations
+        "als": "gsw",   # Alsatian Wikipedia ≈ Swiss German model
+        "li": "nl",     # Limburgish ≈ Dutch
+        "fy": "nl",     # West Frisian ≈ Dutch (best available)
+        "bs": "hr",     # Bosnian ≈ Croatian (best available)
+        "ast": "es",    # Asturian ≈ Spanish
+        "an": "es",     # Aragonese ≈ Spanish
+        "ckb": "kmr",   # Central Kurdish ≈ Kurmanji Kurdish
+        "ku": "kmr",
+        "ps": "ur",     # Pashto ≈ Urdu (Arabic script)
+        "sd": "ur",     # Sindhi ≈ Urdu
+        "yi": "de",     # Yiddish ≈ German
+        "tg": "ru",     # Tajik ≈ Russian (Cyrillic)
+        "uz": "tr",     # Uzbek ≈ Turkish
+        "mn": "bxr",    # Mongolian ≈ Buryat (Cyrillic)
+        # Missing in a smaller model snapshot: choose closest script/family
+        "ms": "id",     # Malay ≈ Indonesian
+        "su": "id",     # Sundanese ≈ Indonesian
+        "war": "tl",    # Waray ≈ Tagalog
+        "min": "id",    # Minangkabau ≈ Indonesian
+        "eo": "en",     # Esperanto ≈ English (fallback)
+        "oc": "fr",     # Occitan ≈ French
+        "ne": "hi",     # Nepali (Devanagari) ≈ Hindi
+        "kn": "hi",     # Kannada ≈ Hindi (fallback)
+        "pa": "hi",     # Punjabi ≈ Hindi (fallback)
+        "sw": "swl",    # Swahili model code in our snapshot
+        # SEA no-space: use Thai model as fallback if missing
+        "my": "th",
+        "km": "th",
+        "lo": "th",
+    }
+    return alias.get(l, l)
+
+
+def normalize_lang_for_lemma(lang: str) -> str:
+    """
+    Lemma store aliasing: prefer a close high-resource lemma dict when missing.
+    (If no dict exists, UnifiedMorphAnalyzer still falls back to identity lemma.)
+    """
+    l = (lang or "").lower().replace("_", "-")
+    alias = {
+        "simple": "en",
+        "als": "de",
+        "li": "nl",
+        "fy": "nl",
+        "bs": "hr",
+        "ast": "es",
+        "an": "es",
+        "ckb": "fa",
+        "ku": "tr",
+        "ps": "ur",
+        "sd": "ur",
+        "yi": "de",
+        "tg": "ru",
+        "uz": "tr",
+        "mn": "ru",
+        "ms": "id",
+        "su": "id",
+        "war": "tl",
+        "ceb": "tl",
+        "min": "id",
+        "eo": "en",
+        "oc": "fr",
+        "bar": "de",
+        "nds": "de",
+        "lb": "de",
+        "fo": "da",
+        "mt": "it",
+        "am": "ar",
+        "ne": "hi",
+        "kn": "hi",
+        "pa": "hi",
+        "my": "th",
+        "km": "th",
+        "lo": "th",
+    }
+    return alias.get(l, l)
+
+
+@lru_cache(maxsize=1)
+def data_dir() -> Path:
+    """
+    리소스 루트 디렉토리.
+    우선순위:
+    - TOKMOR_DATA_DIR
+    - 패키지 번들 tokmor/models
+    """
+    env = os.getenv("TOKMOR_DATA_DIR")
+    if env:
+        p = Path(env).expanduser()
+        return p
+    return _package_models_dir()
+
+
+def _first_existing(*candidates: Path) -> Path:
+    for c in candidates:
+        if c.exists():
+            return c
+    # return first even if missing (callers often check exists())
+    return candidates[0]
+
+def _first_existing_nonempty_dir(candidates: list[Path], *, glob_pat: str) -> Path:
+    """
+    Like _first_existing, but prefers directories that contain at least one matching file.
+    Used to avoid selecting empty packaged dirs (often only contain .gitkeep) during dev.
+    """
+    for c in candidates:
+        try:
+            if c.exists() and c.is_dir() and any(c.glob(glob_pat)):
+                return c
+        except Exception:
+            continue
+    # fallback: first existing dir
+    for c in candidates:
+        if c.exists():
+            return c
+    return candidates[0]
+
+
+def multilingual_dir() -> Path:
+    env = os.getenv("TOKMOR_MODELS_DIR")  # optional override for legacy naming
+    if env:
+        return Path(env).expanduser()
+    base = data_dir()
+    return _first_existing(base / "multilingual", _package_models_dir() / "multilingual")
+
+
+def pmi_crf_dir() -> Path:
+    """
+    (Deprecated) PMI/CRF resources were removed from OSS core.
+
+    This function remains for source compatibility but always returns the data_dir-based path.
+    """
+    return data_dir() / "pmi_crf"
+
+
+def lemma_dict_dir() -> Path:
+    env = os.getenv("TOKMOR_LEMMA_DICT_DIR")
+    if env:
+        return Path(env).expanduser()
+    base = data_dir()
+    # OSS core: lemma dictionaries are optional external assets.
+    return _first_existing(base / "lemma_dict", _package_models_dir() / "lemma_dict")
+
+
+def resolve_lemma_dict_path(lang: str) -> Optional[Path]:
+    """
+    제품 lemma lexicon 경로를 찾는다.
+    우선순위:
+    - lemma_dict/{lang}.sqlite (or .db/.sqlite3)
+    - lemma_dict/{lang}.pkl
+    - lemma_dict/{lang}_lemma.pkl
+    """
+    lang = (lang or "").lower()
+    if not lang:
+        return None
+
+    # Allow forcing pack-less runtime (use only specialized analyzers + fallback rules).
+    # This is useful for "lite" deployments or experiments where lemma assets are unavailable.
+    v = os.getenv("TOKMOR_DISABLE_LEMMA_PACK", "").strip().lower()
+    if v in {"1", "true", "yes", "y", "on"}:
+        return None
+
+    ld = lemma_dict_dir()
+    for name in (f"{lang}.sqlite", f"{lang}.db", f"{lang}.sqlite3", f"{lang}.pkl", f"{lang}_lemma.pkl"):
+        p = ld / name
+        if p.exists():
+            return p
+
+    return None
+
+
+def _removed_feature(*_args, **_kwargs):
+    raise RuntimeError("TokMor OSS core does not ship inflection packs.")
+
+
+def seg_lexicon_dir() -> Path:
+    """
+    Segmentation lexicon directory (e.g., zh wordfreq for internal segmenters).
+    """
+    base = data_dir()
+    # IMPORTANT:
+    # - Users often set TOKMOR_DATA_DIR to a lemma pack that may NOT contain seg_lexicon/.
+    # - We still want bundled starter lexicons (zh/th/lo/km/my) to work by default.
+    # So: prefer a directory that actually contains lexicon files.
+    return _first_existing_nonempty_dir(
+        [base / "seg_lexicon", _package_models_dir() / "seg_lexicon"],
+        glob_pat="*.pkl",
+    )
+
+
+def sea_wordlist_dir() -> Path:
+    """
+    SEA tokenizer wordlist directory (offline).
+    Files:
+      seg_lexicon/{lang}_wordlist.pkl  (pickled set[str])
+      or seg_lexicon/{lang}_wordlist.txt (one token per line)
+    """
+    # Reuse seg_lexicon location (keeps artifacts simple)
+    return seg_lexicon_dir()
+
+
+def resolve_sea_wordlist_path(lang: str) -> Optional[Path]:
+    lang = (lang or "").lower()
+    if not lang:
+        return None
+    d = sea_wordlist_dir()
+    for name in (f"{lang}_wordlist.pkl", f"{lang}_wordlist.txt"):
+        p = d / name
+        if p.exists():
+            return p
+    return None
+
+
+def resolve_seg_lexicon_path(lang: str) -> Optional[Path]:
+    """
+    Resolve path to segmentation lexicon for a given language.
+    Currently used for zh internal segmenter improvements.
+    """
+    lang = (lang or "").lower()
+    if not lang:
+        return None
+    sd = seg_lexicon_dir()
+    for name in (f"{lang}_wordfreq.pkl", f"{lang}_seg_lexicon.pkl"):
+        p = sd / name
+        if p.exists():
+            return p
+    return None
+
+
+def resolve_extra_dict_path(lang: str) -> Optional[Path]:
+    """
+    Optional extra lexicon for improving segmentation/merges.
+    Example:
+      seg_lexicon/zh_extra_dict.json  (token -> pos)
+    """
+    lang = (lang or "").lower()
+    if not lang:
+        return None
+    sd = seg_lexicon_dir()
+    for name in (f"{lang}_extra_dict.json", f"{lang}_extra_lexicon.json"):
+        p = sd / name
+        if p.exists():
+            return p
+    return None
+
+

tokmor/routing.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, List, Tuple
+
+
+# Script buckets we care about for preprocessing/NER routing.
+_SCRIPT_RX: Dict[str, re.Pattern] = {
+    "hangul": re.compile(r"[\uac00-\ud7af]"),
+    "hiragana": re.compile(r"[\u3040-\u309f]"),
+    "katakana": re.compile(r"[\u30a0-\u30ff]"),
+    "han": re.compile(r"[\u4e00-\u9fff]"),
+    "arabic": re.compile(r"[\u0600-\u06ff]"),
+    "hebrew": re.compile(r"[\u0590-\u05ff]"),
+    "thai": re.compile(r"[\u0e00-\u0e7f]"),
+    "devanagari": re.compile(r"[\u0900-\u097f]"),
+    "cyrillic": re.compile(r"[\u0400-\u04ff]"),
+    "latin": re.compile(r"[A-Za-z]"),
+}
+
+
+def script_counts(text: str) -> Dict[str, int]:
+    t = text or ""
+    return {k: len(rx.findall(t)) for k, rx in _SCRIPT_RX.items()}
+
+
+def script_profile(text: str) -> Dict[str, Any]:
+    """
+    Return an explainable script profile for routing.
+
+    This is intentionally *not* a language detector. It's a script detector.
+    """
+    counts = script_counts(text)
+    total = int(sum(counts.values()))
+    if total <= 0:
+        return {
+            "total": 0,
+            "counts": counts,
+            "ratios": {k: 0.0 for k in counts},
+            "dominant": None,
+            "dominant_ratio": 0.0,
+            "is_mixed": False,
+            "top": [],
+        }
+
+    ratios = {k: (v / total) for k, v in counts.items()}
+    dominant = max(ratios, key=ratios.get)
+    dominant_ratio = float(ratios[dominant])
+    top = sorted(((k, float(ratios[k]), int(counts[k])) for k in ratios if counts[k] > 0), key=lambda x: x[1], reverse=True)
+
+    # Mixed script heuristic: dominant script not strong enough.
+    is_mixed = bool(dominant_ratio < 0.30)
+
+    return {
+        "total": total,
+        "counts": counts,
+        "ratios": {k: round(float(v), 4) for k, v in ratios.items()},
+        "dominant": dominant,
+        "dominant_ratio": round(float(dominant_ratio), 4),
+        "is_mixed": is_mixed,
+        "top": [{"script": s, "ratio": round(r, 4), "count": c} for (s, r, c) in top],
+        "has_kana": bool(counts["hiragana"] + counts["katakana"]),
+        "has_cjk": bool(counts["han"]),
+    }
+
+
+def structure_profile(lang: str, sp: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Structure routing hints for downstream (e.g., NER adapter selection).
+
+    IMPORTANT:
+    - This does NOT promise full morphology for a language.
+    - It only recommends which *type* of adapter tends to matter for NER.
+    """
+    l = (lang or "").lower()
+    dom = str(sp.get("dominant") or "")
+
+    # Default: space-delimited text, unknown morphology complexity.
+    stype = "space_delimited"
+    needs_segmentation = False
+    needs_morpheme_split = False
+
+    # High-signal languages (known by tokmor's lightweight language detector)
+    if l == "ko":
+        stype = "agglutinative_hangul"
+        needs_morpheme_split = True
+    elif l == "ja":
+        stype = "agglutinative_japanese"
+        needs_segmentation = True
+        needs_morpheme_split = True
+    elif l == "zh":
+        stype = "unsegmented_han"
+        needs_segmentation = True
+    elif l in {"fi", "hu", "tr", "et"}:
+        # Rich suffixing languages written in Latin script.
+        # For NER, a conservative suffix-strip/suffix-split adapter is often helpful.
+        stype = "suffixing_latin"
+        needs_morpheme_split = True
+    elif l in {"ar", "he"}:
+        stype = "semitic_rtl"
+        needs_morpheme_split = True
+    elif l == "th":
+        stype = "unsegmented_thai"
+        needs_segmentation = True
+    elif l == "hi":
+        stype = "indic_devanagari"
+    elif l == "ru":
+        stype = "slavic_cyrillic"
+
+    # Script-only hints when lang is unknown/mixed
+    if not l or l == "auto":
+        if dom in {"han", "hiragana", "katakana"} or bool(sp.get("has_kana")):
+            stype = "unsegmented_cjk"
+            needs_segmentation = True
+        elif dom == "thai":
+            stype = "unsegmented_thai"
+            needs_segmentation = True
+        elif dom in {"arabic", "hebrew"}:
+            stype = "rtl_script"
+            needs_morpheme_split = True
+
+    # Recommendations for NER candidate generation (core-neutral)
+    rec = {
+        "prefer_token_quality": True,
+        # For candidate generation, morpheme splitting is especially useful for agglutinative & semitic scripts.
+        "ner_prefer_morpheme_split": bool(needs_morpheme_split),
+        # For no-space scripts, segmentation quality is key.
+        "ner_needs_segmentation": bool(needs_segmentation),
+    }
+
+    return {
+        "type": stype,
+        "needs_segmentation": bool(needs_segmentation),
+        "needs_morpheme_split": bool(needs_morpheme_split),
+        "recommendations": rec,
+    }
+
+
+def route(text: str, *, lang: str) -> Dict[str, Any]:
+    """
+    Combined routing payload (script + structure).
+    """
+    sp = script_profile(text)
+    st = structure_profile(lang, sp)
+    return {"script": sp, "structure": st}
+
+