tokmor 1.2.9__py3-none-any.whl

tokmor/__init__.py

@@ -0,0 +1,77 @@
+"""
+TokMor (Core)
+=============
+
+Small, deterministic multilingual preprocessing:
+- tokenization / segmentation with offsets
+- rule-based morphology (best-effort; language-specific analyzers + fallbacks)
+
+This package does not run ML models at inference time.
+"""
+
+# Offline-only runtime enforcement (hard block any online/remote opt-in flags)
+from .offline import enforce_offline as _enforce_offline
+_enforce_offline()
+
+__version__ = "1.2.9"
+
+from .base import BaseTokenizer, TokenizerResult
+from .factory import (
+    detect_language,
+    get_morphological_analyzer,
+    get_tokenizer,
+    morphology_available,
+    morph_supported_languages,
+    supported_languages,
+    tokenize,
+)
+
+# NER-oriented public APIs (recommended)
+from .api import (  # noqa: F401
+    unified_tokenize,
+    ner_preprocess,
+    sentiment_hint,
+    sentiment_lexicon,
+    languages as preprocess_languages,
+    normalize as preprocess_normalize,
+    normalize_sns as preprocess_normalize_sns,
+    function_word_tag,
+    is_function_word,
+)
+
+# Legacy POS-free preprocessing helpers (compat tooling)
+from .legacy_api import (  # noqa: F401
+    tokenize as legacy_tokenize,
+    segment as legacy_segment,
+    route as legacy_route,
+)
+
+__all__ = [
+    # Core
+    'BaseTokenizer',
+    'TokenizerResult',
+    # Factory functions
+    'get_tokenizer',
+    'tokenize',
+    'detect_language',
+    'get_morphological_analyzer',
+    'supported_languages',
+    'morphology_available',
+    'morph_supported_languages',
+    # Public NER-oriented API
+    'unified_tokenize',
+    'ner_preprocess',
+    'function_word_tag',
+    'is_function_word',
+    # Domain hints (optional)
+    'sentiment_hint',
+    'sentiment_lexicon',
+    # Preprocess helpers
+    'preprocess_languages',
+    'preprocess_normalize',
+    'preprocess_normalize_sns',
+    # Legacy POS-free preprocessing (opt-in)
+    'legacy_tokenize',
+    'legacy_segment',
+    'legacy_route',
+]

tokmor/api.py

@@ -0,0 +1,194 @@
+"""
+TokMor public API (NER-oriented preprocessing)
+=============================================
+
+Primary APIs:
+- unified_tokenize(): offsets + best-effort POS/particle hints for downstream NER systems
+- ner_preprocess(): SNS-aware, function-word hard-block, surface tokens for NER input
+
+Legacy POS-free preprocessing helpers (tokenize/segment/route) were moved to `tokmor.legacy_api`.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Literal, Optional, Union
+
+from . import __version__ as _TOKMOR_VERSION
+from .factory import detect_language
+from .inventory import build_language_inventory
+from .preprocess import normalize_text
+from .schema import SCHEMA_VERSION
+
+
+OutputFormat = Literal["tokens", "tokens_with_offsets"]
+SegmentToken = Dict[str, Any]
+
+def ner_preprocess(
+    text: str,
+    lang: str = "auto",
+    *,
+    sns: bool = True,
+    morphology: Optional[bool] = None,
+    include_token_hints: bool = False,
+    include_function_word_hints: bool = False,
+    drop_function_words: bool = True,
+    include_pos4_hints: bool = False,
+    use_surfaces: bool = True,
+) -> Dict[str, Any]:
+    """
+    Convenience helper for NER pipelines.
+
+    Returns:
+    - `ner_tokens`: tokens suitable for NER input (discourse markers/punct removed)
+    - `sns_entities`: SNS discourse markers as separate NER-style entities
+    - `ner_surfaces`: contiguous merged surfaces (helpful for morpheme-split languages)
+    """
+    if lang == "auto":
+        text_norm = normalize_text(text, sns=bool(sns))
+        lang = detect_language(text_norm)
+    from .ner_prep import ner_preprocess as _ner_preprocess
+    return _ner_preprocess(
+        text,
+        lang=lang,
+        sns=sns,
+        morphology=morphology,
+        include_token_hints=include_token_hints,
+        include_function_word_hints=include_function_word_hints,
+        drop_function_words=drop_function_words,
+        include_pos4_hints=include_pos4_hints,
+        use_surfaces=use_surfaces,
+    )
+
+
+def unified_tokenize(
+    text: str,
+    lang: str = "auto",
+    *,
+    sns: bool = True,
+    morphology: Optional[bool] = None,
+    include_sns_tags: bool = False,
+    include_pos4: bool = True,
+) -> Dict[str, Any]:
+    """
+    Tokenize with offsets plus best-effort POS/particle hints (for downstream NER systems).
+
+    Returns tokens with:
+    - text/start/end
+    - pos (language-specific tag if available)
+    - pos_conf (deterministic heuristic, not ML confidence)
+    - is_particle (ko/ja/zh best-effort)
+    - pos4 (optional): N/V/ADJ/ADV/UNK
+    - sns (optional): discourse marker classification
+    """
+    from .unified_tokens import unified_tokenize as _unified_tokenize
+
+    return _unified_tokenize(
+        text,
+        lang=lang,
+        sns=sns,
+        morphology=morphology,
+        include_sns_tags=include_sns_tags,
+        include_pos4=include_pos4,
+    )
+
+
+def languages() -> Dict[str, Any]:
+    """
+    Return an inventory of supported languages and capabilities.
+    """
+    return build_language_inventory()
+
+
+def normalize(text: str) -> str:
+    """
+    Normalize raw text (conservative; keeps meaning).
+    """
+    return normalize_text(text)
+
+
+def normalize_sns(text: str) -> str:
+    """
+    Normalize raw text for SNS / user-generated content.
+    This is still deterministic and conservative, but includes helpful tweaks
+    like fullwidth ASCII normalization for @/#/URLs.
+    """
+    return normalize_text(text, sns=True)
+
+
+def function_word_tag(lang: str, word: str) -> Optional[str]:
+    """
+    Check if a word is a function word and return its POS tag.
+
+    Uses lang_configs (358 languages) with morphology analyzer fallback.
+
+    Args:
+        lang: Language code (e.g., 'en', 'ko', 'jv')
+        word: Word to check
+
+    Returns:
+        POS tag (DET, PRON, AUX, ADP, CCONJ, ADV, PART) or None if not a function word.
+
+    Example:
+        >>> function_word_tag('en', 'the')
+        'DET'
+        >>> function_word_tag('jv', 'iku')
+        'DET'
+        >>> function_word_tag('en', 'running')
+        None
+    """
+    from .ner_prep import function_word_tag as _function_word_tag
+    return _function_word_tag(lang, word)
+
+
+def is_function_word(lang: str, word: str) -> bool:
+    """
+    Check if a word is a function word.
+
+    Args:
+        lang: Language code (e.g., 'en', 'ko', 'jv')
+        word: Word to check
+
+    Returns:
+        True if word is a function word, False otherwise.
+    """
+    return function_word_tag(lang, word) is not None
+
+
+def sentiment_lexicon(lang: str = "auto") -> Optional[Dict[str, Any]]:
+    """
+    Load the built-in small sentiment lexicon (currently ko/en seed).
+
+    Returns None if unavailable for the requested language.
+    """
+    from .domain.sentiment import load_sentiment_lexicon
+
+    if lang == "auto":
+        # Best effort: infer from empty text is meaningless; caller should pass lang in that case.
+        return None
+
+    lex = load_sentiment_lexicon(lang)
+    if lex is None:
+        return None
+    return {
+        "lang": lex.lang,
+        "pos": sorted(lex.pos),
+        "neg": sorted(lex.neg),
+        "negators": sorted(lex.negators),
+        "intensifiers": sorted(lex.intensifiers),
+        "diminishers": sorted(lex.diminishers),
+    }
+
+
+def sentiment_hint(
+    text: str,
+    lang: str = "auto",
+    *,
+    sns: bool = True,
+) -> Dict[str, Any]:
+    """
+    Best-effort sentiment hint for quick experiments (ko/en seed).
+    """
+    from .domain.sentiment import sentiment_hint as _sentiment_hint
+    return _sentiment_hint(text, lang=lang, sns=bool(sns))
+
+

tokmor/assets.py

@@ -0,0 +1,365 @@
+"""
+TokMor core asset management (offline-first)
+===========================================
+
+This module manages ONLY core assets required for tokenization+morphology quality:
+- lemma_dict/      (optional but improves lemmatization)
+- seg_lexicon/     (optional, zh/SEA segmentation improvements)
+
+It supports:
+- pack_status(): discover what is available via tokmor.resources routing
+- build_snapshot(): create an offline data directory (TOKMOR_DATA_DIR layout) with manifest + SHA256SUMS
+- build_bundle(): tar.gz a snapshot
+- verify_bundle(): verify tar + manifest + SHA256SUMS integrity
+
+No downloads. No network access. Pure filesystem operations.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import json
+import os
+import tarfile
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+
+from . import __version__ as _TOKMOR_VERSION
+from .schema import SCHEMA_VERSION
+from . import resources
+
+
+MANIFEST_NAME = "TOKMOR_MANIFEST.json"
+SHA256SUMS_NAME = "SHA256SUMS"
+
+
+def _sha256_file(p: Path) -> str:
+    h = hashlib.sha256()
+    with p.open("rb") as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def _iter_files(root: Path) -> Iterable[Path]:
+    for p in sorted(root.rglob("*")):
+        if p.is_file():
+            yield p
+
+
+def _copy_file(src: Path, dst: Path) -> None:
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    # copy bytes (avoid shutil to keep dependencies minimal and behavior explicit)
+    dst.write_bytes(src.read_bytes())
+
+
+def _atomic_write_text(path: Path, text: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile("w", delete=False, dir=str(path.parent), encoding="utf-8") as tf:
+        tf.write(text)
+        tmp = Path(tf.name)
+    tmp.replace(path)
+
+
+def _atomic_write_bytes(path: Path, data: bytes) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile("wb", delete=False, dir=str(path.parent)) as tf:
+        tf.write(data)
+        tmp = Path(tf.name)
+    tmp.replace(path)
+
+
+def pack_status() -> Dict[str, Any]:
+    """
+    Report discoverable core assets under current routing configuration.
+    """
+    ld = resources.lemma_dict_dir()
+    sd = resources.seg_lexicon_dir()
+    zh_extra = resources.resolve_extra_dict_path("zh")
+
+    def _count(glob_pat: str, d: Path) -> int:
+        try:
+            if not d.exists():
+                return 0
+            return len(list(d.glob(glob_pat)))
+        except Exception:
+            return 0
+
+    return {
+        "schema_version": int(SCHEMA_VERSION),
+        "tokmor_version": str(_TOKMOR_VERSION),
+        "env": {
+            "TOKMOR_DATA_DIR": os.getenv("TOKMOR_DATA_DIR"),
+            "TOKMOR_LEMMA_DICT_DIR": os.getenv("TOKMOR_LEMMA_DICT_DIR"),
+            "TOKMOR_DISABLE_LEMMA_PACK": os.getenv("TOKMOR_DISABLE_LEMMA_PACK"),
+        },
+        "paths": {
+            "data_dir": str(resources.data_dir()),
+            "lemma_dict_dir": str(ld),
+            "seg_lexicon_dir": str(sd),
+            "zh_extra_dict": str(zh_extra) if zh_extra else None,
+        },
+        "counts": {
+            "lemma_sqlite": _count("*.sqlite", ld),
+            "lemma_db": _count("*.db", ld),
+            "lemma_pkl": _count("*.pkl", ld),
+            "seg_wordfreq_pkl": _count("*_wordfreq.pkl", sd),
+            "seg_wordlist_any": _count("*_wordlist.*", sd),
+            "extra_dict_json": _count("*_extra_dict.json", sd),
+        },
+    }
+
+
+@dataclass
+class SnapshotSpec:
+    out_dir: Path
+    include_lemma: bool = True
+    include_seg_lexicon: bool = True
+    langs: Optional[List[str]] = None  # if set, copy only matching language files
+
+
+def _lang_filter_ok(rel_name: str, langs: Optional[List[str]]) -> bool:
+    if not langs:
+        return True
+    # Accept:
+    # - "{lang}.sqlite" in lemma_dict/
+    # - "{lang}_wordfreq.pkl", "{lang}_wordlist.*", "{lang}_extra_dict.json" in seg_lexicon/
+    base = Path(rel_name).name
+    for l in langs:
+        l2 = (l or "").lower().replace("_", "-")
+        if not l2:
+            continue
+        if base.startswith(l2 + "_") or base.startswith(l2 + "."):
+            return True
+    return False
+
+
+def build_snapshot(spec: SnapshotSpec) -> Dict[str, Any]:
+    """
+    Build an offline snapshot directory with manifest + SHA256SUMS.
+
+    Snapshot layout (subset of TOKMOR_DATA_DIR):
+      out_dir/
+        lemma_dict/...
+        seg_lexicon/...
+        TOKMOR_MANIFEST.json
+        SHA256SUMS
+    """
+    out_dir = Path(spec.out_dir).expanduser().resolve()
+    tmp_root = out_dir.parent / (out_dir.name + ".tmp")
+    if tmp_root.exists():
+        # best-effort cleanup
+        for p in reversed(list(tmp_root.rglob("*"))):
+            try:
+                if p.is_file():
+                    p.unlink()
+                elif p.is_dir():
+                    p.rmdir()
+            except Exception:
+                pass
+        try:
+            tmp_root.rmdir()
+        except Exception:
+            pass
+
+    tmp_root.mkdir(parents=True, exist_ok=True)
+
+    copied: List[str] = []
+    if spec.include_lemma:
+        src = resources.lemma_dict_dir()
+        dst = tmp_root / "lemma_dict"
+        if src.exists():
+            for f in src.iterdir():
+                if not f.is_file():
+                    continue
+                if f.suffix.lower() not in {".sqlite", ".db", ".sqlite3", ".pkl"}:
+                    continue
+                rel = f"lemma_dict/{f.name}"
+                if not _lang_filter_ok(rel, spec.langs):
+                    continue
+                _copy_file(f, dst / f.name)
+                copied.append(rel)
+
+    if spec.include_seg_lexicon:
+        src = resources.seg_lexicon_dir()
+        dst = tmp_root / "seg_lexicon"
+        if src.exists():
+            for f in src.iterdir():
+                if not f.is_file():
+                    continue
+                # keep only known safe lexicon assets (no giant corpora)
+                name = f.name
+                if not (
+                    name.endswith("_wordfreq.pkl")
+                    or name.endswith("_seg_lexicon.pkl")
+                    or name.endswith("_wordlist.pkl")
+                    or name.endswith("_wordlist.txt")
+                    or name.endswith("_extra_dict.json")
+                    or name.endswith("_extra_lexicon.json")
+                ):
+                    continue
+                rel = f"seg_lexicon/{name}"
+                if not _lang_filter_ok(rel, spec.langs):
+                    continue
+                _copy_file(f, dst / name)
+                copied.append(rel)
+
+    # manifest + SHA256SUMS
+    checksums: Dict[str, str] = {}
+    for f in _iter_files(tmp_root):
+        rel = str(f.relative_to(tmp_root)).replace("\\", "/")
+        checksums[rel] = _sha256_file(f)
+
+    manifest: Dict[str, Any] = {
+        "schema_version": int(SCHEMA_VERSION),
+        "tokmor_version": str(_TOKMOR_VERSION),
+        "type": "tokmor_core_snapshot",
+        "assets": {
+            "include_lemma": bool(spec.include_lemma),
+            "include_seg_lexicon": bool(spec.include_seg_lexicon),
+            "langs": spec.langs,
+        },
+        "files": [
+            {"path": rel, "sha256": sha}
+            for rel, sha in sorted(checksums.items(), key=lambda x: x[0])
+        ],
+    }
+
+    _atomic_write_text(tmp_root / MANIFEST_NAME, json.dumps(manifest, ensure_ascii=False, indent=2) + "\n")
+    # recompute checksums including manifest
+    checksums[str(MANIFEST_NAME)] = _sha256_file(tmp_root / MANIFEST_NAME)
+
+    sha_lines = [f"{checksums[p]}  {p}" for p in sorted(checksums.keys())]
+    _atomic_write_text(tmp_root / SHA256SUMS_NAME, "\n".join(sha_lines) + "\n")
+
+    # move into place
+    if out_dir.exists():
+        # replace old snapshot atomically-ish (rename to backup then replace)
+        backup = out_dir.parent / (out_dir.name + ".bak")
+        if backup.exists():
+            # delete previous backup
+            for p in reversed(list(backup.rglob("*"))):
+                try:
+                    if p.is_file():
+                        p.unlink()
+                    elif p.is_dir():
+                        p.rmdir()
+                except Exception:
+                    pass
+            try:
+                backup.rmdir()
+            except Exception:
+                pass
+        out_dir.replace(backup)
+
+    tmp_root.replace(out_dir)
+
+    return {
+        "ok": True,
+        "schema_version": int(SCHEMA_VERSION),
+        "tokmor_version": str(_TOKMOR_VERSION),
+        "out_dir": str(out_dir),
+        "file_count": len(list(_iter_files(out_dir))),
+        "copied_count": len(copied),
+    }
+
+
+def build_bundle(*, snapshot_dir: Path, out_tgz: Path) -> Dict[str, Any]:
+    """
+    Create a tar.gz from a snapshot directory.
+    """
+    snapshot_dir = Path(snapshot_dir).expanduser().resolve()
+    out_tgz = Path(out_tgz).expanduser().resolve()
+    if not snapshot_dir.exists():
+        raise FileNotFoundError(f"snapshot_dir does not exist: {snapshot_dir}")
+
+    # write tgz
+    out_tgz.parent.mkdir(parents=True, exist_ok=True)
+    with tarfile.open(out_tgz, "w:gz") as tf:
+        tf.add(snapshot_dir, arcname=snapshot_dir.name)
+
+    sha = _sha256_file(out_tgz)
+    _atomic_write_text(out_tgz.with_suffix(out_tgz.suffix + ".sha256"), sha + "\n")
+    return {
+        "ok": True,
+        "schema_version": int(SCHEMA_VERSION),
+        "tokmor_version": str(_TOKMOR_VERSION),
+        "bundle": str(out_tgz),
+        "sha256": sha,
+    }
+
+
+def verify_bundle(*, bundle_tgz: Path, expected_sha256: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Verify tar.gz bundle integrity:
+    - optional sha256 match
+    - contains manifest and SHA256SUMS
+    - internal files match SHA256SUMS
+    """
+    bundle_tgz = Path(bundle_tgz).expanduser().resolve()
+    if not bundle_tgz.exists():
+        raise FileNotFoundError(f"bundle not found: {bundle_tgz}")
+
+    actual = _sha256_file(bundle_tgz)
+    if expected_sha256 and (expected_sha256.strip().lower() != actual.lower()):
+        return {"ok": False, "error": "sha256_mismatch", "expected": expected_sha256, "actual": actual}
+
+    # Extract to temp dir (disk-safe; deletes on exit)
+    with tempfile.TemporaryDirectory(prefix="tokmor_bundle_verify_") as td:
+        td_path = Path(td)
+        with tarfile.open(bundle_tgz, "r:gz") as tf:
+            tf.extractall(td_path)
+
+        # assume single root directory
+        roots = [p for p in td_path.iterdir() if p.is_dir()]
+        if not roots:
+            return {"ok": False, "error": "no_root_dir"}
+        root = roots[0]
+
+        man = root / MANIFEST_NAME
+        sums = root / SHA256SUMS_NAME
+        if not man.exists():
+            return {"ok": False, "error": "missing_manifest"}
+        if not sums.exists():
+            return {"ok": False, "error": "missing_sha256sums"}
+
+        # parse SHA256SUMS
+        expected: Dict[str, str] = {}
+        for line in sums.read_text(encoding="utf-8", errors="ignore").splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            # "sha  path"
+            parts = line.split()
+            if len(parts) < 2:
+                continue
+            sha = parts[0].strip()
+            path = parts[-1].strip()
+            expected[path] = sha
+
+        # verify
+        bad: List[Dict[str, str]] = []
+        for path, sha in expected.items():
+            p = root / path
+            if not p.exists() or not p.is_file():
+                bad.append({"path": path, "error": "missing"})
+                continue
+            got = _sha256_file(p)
+            if got.lower() != sha.lower():
+                bad.append({"path": path, "error": "sha256_mismatch", "expected": sha, "actual": got})
+
+        return {
+            "ok": len(bad) == 0,
+            "schema_version": int(SCHEMA_VERSION),
+            "tokmor_version": str(_TOKMOR_VERSION),
+            "bundle": str(bundle_tgz),
+            "bundle_sha256": actual,
+            "errors": bad,
+        }
+
+
+
+