khmerthings 0.4.3__py3-none-any.whl

khmerthings/__init__.py

@@ -0,0 +1,29 @@
+"""khmerthings — deterministic Khmer language tools."""
+
+from khmerthings.clusters import segment_clusters
+from khmerthings.counter import WordCount, analyze, count_words
+from khmerthings.lexicon import WORD_SOURCES, Lexicon, default_lexicon, load_lexicon
+from khmerthings.segmenter import break_words, mark_boundaries
+from khmerthings.sorting import khmer_sort_key, sort_lines
+from khmerthings.tokenizer import Token, TokenType, tokenize
+
+__version__ = "0.4.3"
+
+__all__ = [
+    "WORD_SOURCES",
+    "Lexicon",
+    "Token",
+    "TokenType",
+    "WordCount",
+    "__version__",
+    "analyze",
+    "break_words",
+    "count_words",
+    "default_lexicon",
+    "khmer_sort_key",
+    "load_lexicon",
+    "mark_boundaries",
+    "segment_clusters",
+    "sort_lines",
+    "tokenize",
+]

khmerthings/__main__.py

@@ -0,0 +1,7 @@
+"""Allow ``python -m khmerthings``."""
+
+import sys
+
+from khmerthings.cli import main
+
+sys.exit(main())

khmerthings/chars.py

@@ -0,0 +1,129 @@
+"""Khmer Unicode character classification.
+
+Covers the Khmer block (U+1780-U+17FF) and Khmer Symbols block
+(U+19E0-U+19FF). All functions are pure and operate on single characters;
+multi-character strings raise ``ValueError`` to catch misuse early.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+
+COENG = "្"
+ZERO_WIDTH_SPACE = "​"
+
+# Invisible, deprecated "inherent vowel" format characters that still occur
+# in real-world text; they always belong to the preceding cluster.
+INHERENT_VOWELS = ("឴", "឵")
+
+_CONSONANTS = range(0x1780, 0x17A3)  # ក..អ
+_INDEPENDENT_VOWELS = range(0x17A3, 0x17B4)  # ឣ..ឳ (includes deprecated ឣឤ)
+_DEPENDENT_VOWELS = range(0x17B6, 0x17C6)  # ា..ៅ
+_SIGNS = frozenset(range(0x17C6, 0x17D2)) | {0x17DD}  # ំ..៑ + ៝ (not coeng)
+_DIGITS = range(0x17E0, 0x17EA)  # ០..៩
+_PUNCTUATION = frozenset(range(0x17D4, 0x17DB)) | {0x17DC}  # ។៕៖ៗ៘៙៚ + ៜ
+_KHMER_BLOCK = range(0x1780, 0x1800)
+_KHMER_SYMBOLS = range(0x19E0, 0x1A00)
+
+
+class ScriptClass(Enum):
+    """Coarse script classification of a single character."""
+
+    KHMER = "khmer"
+    LATIN = "latin"
+    DIGIT = "digit"
+    OTHER = "other"
+
+
+def _codepoint(ch: str) -> int:
+    if len(ch) != 1:
+        raise ValueError(f"expected a single character, got {ch!r}")
+    return ord(ch)
+
+
+def is_khmer(ch: str) -> bool:
+    """True if *ch* is in the Khmer or Khmer Symbols Unicode blocks."""
+    cp = _codepoint(ch)
+    return cp in _KHMER_BLOCK or cp in _KHMER_SYMBOLS
+
+
+def is_consonant(ch: str) -> bool:
+    """True for the 33 Khmer consonants ក (U+1780) through អ (U+17A2)."""
+    return _codepoint(ch) in _CONSONANTS
+
+
+def is_independent_vowel(ch: str) -> bool:
+    """True for independent vowels ឣ (U+17A3) through ឳ (U+17B3)."""
+    return _codepoint(ch) in _INDEPENDENT_VOWELS
+
+
+def is_dependent_vowel(ch: str) -> bool:
+    """True for dependent (combining) vowels ា (U+17B6) through ៅ (U+17C5)."""
+    return _codepoint(ch) in _DEPENDENT_VOWELS
+
+
+def is_sign(ch: str) -> bool:
+    """True for combining signs/diacritics (nikahit, reahmuk, bantoc, ...)."""
+    return _codepoint(ch) in _SIGNS
+
+
+def is_coeng(ch: str) -> bool:
+    """True for the subscript-forming sign ្ (U+17D2)."""
+    return ch == COENG
+
+
+def is_inherent_vowel(ch: str) -> bool:
+    """True for the invisible deprecated inherent vowels U+17B4 and U+17B5."""
+    return ch in INHERENT_VOWELS
+
+
+def is_khmer_digit(ch: str) -> bool:
+    """True for Khmer digits ០ (U+17E0) through ៩ (U+17E9)."""
+    return _codepoint(ch) in _DIGITS
+
+
+def is_khmer_punctuation(ch: str) -> bool:
+    """True for Khmer punctuation such as ។ (khan) and ៕ (bariyoosan).
+
+    The currency sign ៛ (U+17DB) is deliberately excluded.
+    """
+    return _codepoint(ch) in _PUNCTUATION
+
+
+def is_khmer_letter_or_mark(ch: str) -> bool:
+    """True if *ch* can be part of a Khmer word (letters and combining marks)."""
+    return (
+        is_consonant(ch)
+        or is_independent_vowel(ch)
+        or is_dependent_vowel(ch)
+        or is_sign(ch)
+        or is_coeng(ch)
+        or is_inherent_vowel(ch)
+    )
+
+
+def khmer_digit_to_int(ch: str) -> int:
+    """Convert a single Khmer digit to its integer value.
+
+    Raises ``ValueError`` for anything that is not a Khmer digit.
+    """
+    cp = _codepoint(ch)
+    if cp not in _DIGITS:
+        raise ValueError(f"not a Khmer digit: {ch!r}")
+    return cp - 0x17E0
+
+
+def script_class(ch: str) -> ScriptClass:
+    """Classify a character as KHMER, LATIN, DIGIT (ASCII), or OTHER.
+
+    Khmer digits classify as KHMER; use :func:`is_khmer_digit` to
+    distinguish them.
+    """
+    cp = _codepoint(ch)
+    if cp in _KHMER_BLOCK or cp in _KHMER_SYMBOLS:
+        return ScriptClass.KHMER
+    if ch.isascii() and ch.isalpha():
+        return ScriptClass.LATIN
+    if ch.isascii() and ch.isdigit():
+        return ScriptClass.DIGIT
+    return ScriptClass.OTHER

khmerthings/cli.py

@@ -0,0 +1,147 @@
+"""Command-line interface: ``khmerthings <tool> ...``.
+
+Each library tool is a subcommand; new tools register a subparser here.
+"""
+
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import json
+import sys
+from collections.abc import Sequence
+
+from khmerthings import __version__
+from khmerthings.counter import analyze
+from khmerthings.lexicon import WORD_SOURCES, Lexicon, load_lexicon
+from khmerthings.segmenter import break_words, mark_boundaries
+from khmerthings.sorting import sort_lines
+
+__all__ = ["main"]
+
+
+def _read_source(path: str) -> tuple[str, str]:
+    """Return (source label, text) for a file path or '-' for stdin."""
+    if path == "-":
+        return "<stdin>", sys.stdin.read()
+    with open(path, encoding="utf-8") as f:
+        return path, f.read()
+
+
+def _lexicon_from_args(args: argparse.Namespace) -> Lexicon:
+    includes = tuple(s.strip() for s in (args.include or "").split(",") if s.strip())
+    return load_lexicon("words", *includes)
+
+
+def _add_include_option(parser: argparse.ArgumentParser) -> None:
+    extra = sorted(set(WORD_SOURCES) - {"words"})
+    parser.add_argument(
+        "--include",
+        metavar=",".join(extra),
+        help=f"extra wordlists to match against, comma-separated (available: {', '.join(extra)})",
+    )
+
+
+def _cmd_count(args: argparse.Namespace) -> int:
+    paths: list[str] = args.files or ["-"]
+    lexicon = _lexicon_from_args(args)
+    results = []
+    for path in paths:
+        source, text = _read_source(path)
+        results.append({"source": source, **dataclasses.asdict(analyze(text, lexicon))})
+
+    if args.json:
+        print(json.dumps(results, ensure_ascii=False, indent=2))
+        return 0
+
+    for result in results:
+        if len(results) > 1:
+            print(f"{result['source']}:")
+        for field in dataclasses.fields(analyze("")):
+            print(f"  {field.name}: {result[field.name]}")
+    return 0
+
+
+def _cmd_segment(args: argparse.Namespace) -> int:
+    paths: list[str] = args.files or ["-"]
+    lexicon = _lexicon_from_args(args)
+    if args.mark:
+        separator = args.separator if args.separator is not None else "​"
+    else:
+        separator = args.separator if args.separator is not None else " "
+    for path in paths:
+        _, text = _read_source(path)
+        for line in text.splitlines():
+            if args.mark:
+                print(mark_boundaries(line, separator, lexicon))
+            else:
+                print(separator.join(break_words(line, lexicon)))
+    return 0
+
+
+def _cmd_sort(args: argparse.Namespace) -> int:
+    paths: list[str] = args.files or ["-"]
+    lines: list[str] = []
+    for path in paths:
+        _, text = _read_source(path)
+        lines.extend(text.splitlines())
+    for line in sort_lines(lines, descending=args.desc):
+        print(line)
+    return 0
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="khmerthings",
+        description="Deterministic Khmer language tools.",
+    )
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    count = subparsers.add_parser("count", help="count words in Khmer (or mixed) text")
+    count.add_argument("files", nargs="*", help="input files, or '-' for stdin (default)")
+    count.add_argument("--json", action="store_true", help="emit machine-readable JSON")
+    _add_include_option(count)
+    count.set_defaults(func=_cmd_count)
+
+    segment = subparsers.add_parser(
+        "segment", help="break Khmer text into words (word segmentation)"
+    )
+    segment.add_argument("files", nargs="*", help="input files, or '-' for stdin (default)")
+    segment.add_argument(
+        "--separator",
+        help="word separator (default: space, or ZWSP with --mark)",
+    )
+    segment.add_argument(
+        "--mark",
+        action="store_true",
+        help="preserve the line as-is and only insert separators at Khmer word boundaries",
+    )
+    _add_include_option(segment)
+    segment.set_defaults(func=_cmd_segment)
+
+    sort = subparsers.add_parser(
+        "sort", help="sort lines in Khmer dictionary order (ascending by default)"
+    )
+    sort.add_argument("files", nargs="*", help="input files, or '-' for stdin (default)")
+    sort.add_argument("--desc", action="store_true", help="sort in descending order")
+    sort.set_defaults(func=_cmd_sort)
+
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    try:
+        result: int = args.func(args)
+    except ValueError as exc:  # e.g. unknown --include source
+        parser.error(str(exc))
+    except OSError as exc:  # e.g. missing or unreadable input file
+        print(f"khmerthings: error: {exc}", file=sys.stderr)
+        return 1
+    return result
+
+
+if __name__ == "__main__":
+    sys.exit(main())

khmerthings/clusters.py

@@ -0,0 +1,77 @@
+"""Khmer character-cluster (KCC) segmentation.
+
+A Khmer character cluster is the smallest user-visible unit of text: a base
+consonant or independent vowel, optionally followed by subscript (coeng)
+consonants, dependent vowels, and combining signs. Cluster boundaries are the
+only positions where a word boundary can legally occur, which makes this the
+foundation for word segmentation.
+
+The segmenter is a deterministic scanner over NFC-normalized text. It never
+drops or reorders characters: ``"".join(segment_clusters(t))`` always equals
+``unicodedata.normalize("NFC", t)``. Malformed sequences (orphan coeng,
+orphan vowel) are attached to the preceding cluster when one exists,
+otherwise emitted as standalone clusters.
+"""
+
+from __future__ import annotations
+
+import unicodedata
+
+from khmerthings.chars import (
+    is_coeng as _is_coeng,
+)
+from khmerthings.chars import (
+    is_consonant,
+    is_dependent_vowel,
+    is_independent_vowel,
+    is_inherent_vowel,
+    is_khmer,
+    is_sign,
+)
+
+__all__ = ["segment_clusters"]
+
+
+def _is_base(ch: str) -> bool:
+    return is_consonant(ch) or is_independent_vowel(ch)
+
+
+def _is_trailing(ch: str) -> bool:
+    """Characters that extend the current cluster (excluding coeng pairs)."""
+    return is_dependent_vowel(ch) or is_sign(ch) or is_inherent_vowel(ch)
+
+
+def segment_clusters(text: str) -> list[str]:
+    """Split *text* into Khmer character clusters.
+
+    Non-Khmer characters are emitted as single-character clusters. The input
+    is NFC-normalized first; the concatenation of the result equals the
+    normalized input.
+    """
+    text = unicodedata.normalize("NFC", text)
+    clusters: list[str] = []
+    i = 0
+    n = len(text)
+    while i < n:
+        ch = text[i]
+        if _is_base(ch):
+            j = i + 1
+            while j < n:
+                c = text[j]
+                if _is_coeng(c) and j + 1 < n and _is_base(text[j + 1]):
+                    j += 2
+                elif _is_trailing(c):
+                    j += 1
+                else:
+                    break
+            clusters.append(text[i:j])
+            i = j
+        elif (_is_trailing(ch) or _is_coeng(ch)) and clusters and is_khmer(clusters[-1][-1]):
+            # Orphan combining mark: attach to the preceding Khmer cluster.
+            clusters[-1] += ch
+            i += 1
+        else:
+            # Digits, symbols, punctuation, non-Khmer, or leading orphan marks.
+            clusters.append(ch)
+            i += 1
+    return clusters

khmerthings/counter.py

@@ -0,0 +1,73 @@
+"""Khmer-aware word counting."""
+
+from __future__ import annotations
+
+import unicodedata
+from dataclasses import dataclass
+
+from khmerthings.chars import is_khmer
+from khmerthings.clusters import segment_clusters
+from khmerthings.lexicon import Lexicon
+from khmerthings.tokenizer import TokenType, tokenize
+
+__all__ = ["WordCount", "analyze", "count_words"]
+
+
+@dataclass(frozen=True, slots=True)
+class WordCount:
+    """Word and character statistics for a text.
+
+    ``total_words`` is the sum of known Khmer words, unknown Khmer word
+    groups, Latin words, and number tokens (ASCII or Khmer digits).
+    ``characters`` counts characters of the NFC-normalized text.
+    """
+
+    total_words: int
+    khmer_words: int
+    unknown_khmer_words: int
+    latin_words: int
+    numbers: int
+    clusters: int
+    khmer_characters: int
+    characters: int
+
+
+def analyze(text: str, lexicon: Lexicon | None = None) -> WordCount:
+    """Compute word and character statistics for *text*."""
+    normalized = unicodedata.normalize("NFC", text)
+    tokens = tokenize(normalized, lexicon)
+
+    khmer_words = 0
+    unknown = 0
+    latin = 0
+    numbers = 0
+    clusters = 0
+    for token in tokens:
+        if token.type is TokenType.KHMER_WORD:
+            khmer_words += 1
+            clusters += len(segment_clusters(token.text))
+        elif token.type is TokenType.KHMER_UNKNOWN:
+            unknown += 1
+            clusters += len(segment_clusters(token.text))
+        elif token.type is TokenType.LATIN:
+            latin += 1
+        elif token.type in (TokenType.NUMBER, TokenType.KHMER_DIGIT):
+            numbers += 1
+            if token.type is TokenType.KHMER_DIGIT:
+                clusters += len(token.text)
+
+    return WordCount(
+        total_words=khmer_words + unknown + latin + numbers,
+        khmer_words=khmer_words,
+        unknown_khmer_words=unknown,
+        latin_words=latin,
+        numbers=numbers,
+        clusters=clusters,
+        khmer_characters=sum(1 for ch in normalized if is_khmer(ch)),
+        characters=len(normalized),
+    )
+
+
+def count_words(text: str, lexicon: Lexicon | None = None) -> int:
+    """Count words in *text* (Khmer words, Latin words, and numbers)."""
+    return analyze(text, lexicon).total_words

khmerthings/data/modern.txt

@@ -0,0 +1,44 @@
+# khmerthings modern lexicon: slang, informal register, loanwords, and
+# trending vocabulary. One entry per line, UTF-8, NFC, Khmer letters/marks only.
+# Hand-curated entry by entry; candidates researched from public sources and
+# contemporary Khmer media usage (spellings cross-checked, no wordlist
+# imported wholesale):
+#   - https://km.wiktionary.org/wiki/ស្ទាវ
+#   - https://thmeythmey.com/detail/130843 (ឡូយ)
+#   - https://ling-app.com/blog/khmer-slang-words/
+# This register shifts quickly; entries here are expected to grow and be
+# revised more often than words.txt.
+#
+# --- slang & informal ---
+ឡូយ
+ស្ទាវ
+ឡប់
+ភ្លើ
+ចង្រៃ
+ម៉ង
+អេម
+ប្រូ
+អញ
+ហែង
+# --- internet, tech & media loanwords ---
+ហ្វេសប៊ុក
+យូធូប
+តេឡេក្រាម
+អនឡាញ
+អ៊ីនធឺណិត
+វីដេអូ
+កាមេរ៉ា
+ហ្គេម
+ស្មាតហ្វូន
+ឡាយ
+សែលហ្វី
+# --- everyday modern loanwords ---
+កូវីដ
+វ៉ាក់សាំង
+ដុល្លារ
+ម៉ាស៊ីន
+ម៉ូដ
+សាំង
+កាដូ
+ភីហ្សា
+ប៊ិច