amharic-segmenter 0.1.0__py3-none-any.whl

amharic_segmenter/__init__.py

@@ -0,0 +1,40 @@
+"""amharic-segmenter: sentence, word and morpheme tokenization for Amharic (አማርኛ)."""
+from __future__ import annotations
+
+from ._version import __version__
+from .morphology import Analysis
+from .normalize import normalize_homophones
+from .tokenizer import (
+    AmharicStemmer,
+    AmharicTokenizer,
+    Token,
+    analyze,
+    lemma,
+    sent_tokenize,
+    sentence_tokenize,
+    split_morphemes,
+    stem,
+    stem_words,
+    tokenize,
+    tokenize_spans,
+    word_tokenize,
+)
+
+__all__ = [
+    "AmharicStemmer",
+    "AmharicTokenizer",
+    "Analysis",
+    "Token",
+    "__version__",
+    "analyze",
+    "lemma",
+    "normalize_homophones",
+    "sent_tokenize",
+    "sentence_tokenize",
+    "split_morphemes",
+    "stem",
+    "stem_words",
+    "tokenize",
+    "tokenize_spans",
+    "word_tokenize",
+]

amharic_segmenter/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

amharic_segmenter/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

amharic_segmenter/charset.py

@@ -0,0 +1,69 @@
+"""Low-level helpers for the Ethiopic (Fidel) script.
+
+Each Ethiopic consonant occupies a block of 8 consecutive code points, one per
+vowel *order*.  The first seven are the classic orders used in Amharic
+grammar books (the names below follow the traditional Ge'ez terminology):
+
+    0 ግዕዝ  (ä)   1 ካዕብ (u)   2 ሣልስ (i)   3 ራብዕ (a)
+    4 ሃምስ  (e)   5 ሳድስ (ə / no vowel)   6 ሳብዕ (o)
+
+The eighth slot of a block is a labialised form (e.g. ሏ) and is ignored here.
+A few blocks (e.g. ቈ, ኰ, ጐ) are irregular in Unicode and are deliberately
+left out, because they never occur as word-final letters in ordinary text.
+"""
+from __future__ import annotations
+
+GEEZ, KAEB, SALIS, RABI, HAMIS, SADIS, SABI = range(7)
+
+ORDER_NAMES = ("ግዕዝ", "ካዕብ", "ሣልስ", "ራብዕ", "ሃምስ", "ሳድስ", "ሳብዕ")
+
+_FIRST = 0x1200
+_LAST_REGULAR = 0x1357
+_IRREGULAR = (
+    (0x1248, 0x124F),
+    (0x1258, 0x125F),
+    (0x1288, 0x128F),
+    (0x12B0, 0x12B7),
+    (0x12C0, 0x12C7),
+    (0x1310, 0x1317),
+)
+
+
+def is_ethiopic_letter(ch: str) -> bool:
+    """True for a character in the Ethiopic syllable range."""
+    return _FIRST <= ord(ch) <= 0x135A
+
+
+def _is_regular(cp: int) -> bool:
+    if not (_FIRST <= cp <= _LAST_REGULAR):
+        return False
+    return not any(lo <= cp <= hi for lo, hi in _IRREGULAR)
+
+
+def fidel_order(ch: str) -> int | None:
+    """Return the vowel order (0-6) of a fidel, or ``None`` if not applicable."""
+    cp = ord(ch)
+    if not _is_regular(cp):
+        return None
+    order = (cp - _FIRST) % 8
+    return order if order <= SABI else None
+
+
+def is_wa_form(ch: str) -> bool:
+    """True for the labialised 8th form of a letter (ሟ, ሏ, ጓ …).
+
+    It is how the feminine ``ዋ`` is written when it fuses into a consonant,
+    e.g. ሐኪም + ዋ -> ሐኪሟ.
+    """
+    cp = ord(ch)
+    return _is_regular(cp) and (cp - _FIRST) % 8 == 7
+
+
+def with_order(ch: str, order: int) -> str:
+    """Return the fidel with the same consonant as ``ch`` in another vowel order."""
+    current = fidel_order(ch)
+    if current is None:
+        raise ValueError(f"{ch!r} is not a regular Ethiopic syllable")
+    if not 0 <= order <= SABI:
+        raise ValueError("order must be between 0 and 6")
+    return chr(ord(ch) - current + order)

amharic_segmenter/cli.py

@@ -0,0 +1,71 @@
+"""Command line interface: ``amharic-segmenter`` / ``python -m amharic_segmenter``."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from ._version import __version__
+from .tokenizer import AmharicTokenizer
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="amharic-segmenter",
+        description="Tokenize Amharic text into sentences, words or morphemes.",
+    )
+    p.add_argument("text", nargs="?", help="text to tokenize (reads stdin when omitted)")
+    p.add_argument("-s", "--sentences", action="store_true", help="split into sentences instead of words")
+    p.add_argument(
+        "-m", "--morphemes", action="store_true", help="split words into prefixes, stems and suffixes"
+    )
+    p.add_argument("--stem", action="store_true", help="print the stem of every word")
+    p.add_argument("--mark-affixes", action="store_true", help="write prefixes as 'በ+' and suffixes as '+ን'")
+    p.add_argument("--strategy", choices=["lexicon", "rules", "none"], default="lexicon",
+                   help="morpheme splitting strategy (default: lexicon)")
+    p.add_argument("--lexicon", metavar="FILE", help="UTF-8 file with extra stems, one per line")
+    p.add_argument(
+        "--normalize", action="store_true", help="collapse homophone letters (ሐ->ሀ, ሠ->ሰ, ዐ->አ, ፀ->ጸ)"
+    )
+    p.add_argument("--no-punct", action="store_true", help="drop punctuation tokens")
+    p.add_argument("--json", action="store_true", help="print the result as JSON")
+    p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    for stream in (sys.stdout, sys.stdin):
+        if hasattr(stream, "reconfigure"):
+            stream.reconfigure(encoding="utf-8")  # type: ignore[union-attr]
+
+    text = args.text if args.text is not None else sys.stdin.read()
+    kwargs = dict(strategy=args.strategy, normalize=args.normalize)
+    tok = (AmharicTokenizer.from_file(args.lexicon, **kwargs) if args.lexicon
+           else AmharicTokenizer(**kwargs))
+
+    if args.stem:
+        result = [tok.stem(t.text) for t in tok.tokenize_spans(text, include_punct=not args.no_punct)
+                  if t.kind != "punct" or not args.no_punct]
+        sep = " "
+    elif args.sentences:
+        result = tok.sentence_tokenize(text)
+        sep = "\n"
+    else:
+        result = tok.tokenize(
+            text,
+            morphemes=args.morphemes,
+            include_punct=not args.no_punct,
+            mark_affixes=args.mark_affixes,
+        )
+        sep = " "
+
+    if args.json:
+        print(json.dumps(result, ensure_ascii=False))
+    else:
+        print(sep.join(result))
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

amharic_segmenter/data/irregular_plurals.tsv

@@ -0,0 +1,43 @@
+# Irregular plurals: singular<TAB>plural   ('#' = comment)
+#
+# Transcribed from photographed pages of አማርኛ ሠዋሰው (1963 E.C.): the Geez-origin
+# plurals in section 2.2 (pp. 42-44) and the reduplicated adjective plurals in
+# section 2.3.4 (pp. 47-48). Both forms are added to the lexicon, so they can take
+# suffixes (መጻሕፍትን) and the regular plural can be added on top (ቃላቶች).
+# Only entries that are clearly legible in the photos are listed - please review.
+
+# --- Geez-origin plurals (section 2.2) ---
+ካህን	ካህናት
+ጳጳስ	ጳጳሳት
+ቃል	ቃላት
+ሐዋርያ	ሐዋርያት
+ንጉሥ	ነገሥታት
+ነቢይ	ነቢያት
+መጽሐፍ	መጻሕፍት
+መምህር	መምህራን
+መስፍን	መሳፍንት
+ደብር	አድባራት
+ቄስ	ቀሳውስት
+መልአክ	መላእክት
+ሊቅ	ሊቃውንት
+ቤት	አብያት
+ርእስ	አርእስት
+ሕዝብ	አሕዛብ
+ዐምድ	አዕማድ
+ገዳም	ገዳማት
+አንቀጽ	አናቅጽ
+ድንግል	ደናግል
+አንበሳ	አናብስት
+ተዋናይ	ተዋናያን
+ማኅበር	ማኅበራት
+ኢትዮጵያ	ኢትዮጵያውያን
+
+# --- adjectives with reduplicated plurals (section 2.3.4) ---
+ደግ	ደጋግ
+ጥቁር	ጥቋቁር
+ረጅም	ረጃጅም
+ትልቅ	ትልልቅ
+ትንሽ	ትንንሽ
+አጭር	አጫጭር
+ቀይ	ቀያይ
+ነጭ	ነጫጭ

amharic_segmenter/data/lexicon.txt

@@ -0,0 +1,110 @@
+# Seed lexicon for amharic-segmenter (UTF-8, one word per line, '#' = comment).
+#
+# These are stems that may take prefixes/suffixes. A split such as ተማሪዎች ->
+# ተማሪ + ዎች is only accepted when the stem is listed here (or when the
+# opt-in "rules" strategy is used). The list starts with the nouns, names and
+# function words that appear as examples in አማርኛ ሠዋሰው (1963 E.C.); grow it
+# with a pull request or pass your own words to AmharicTokenizer(lexicon=...).
+
+# names
+ከበደ
+አበበ
+ተፈራ
+
+# people and relations
+ሰው
+ልጅ
+ተማሪ
+አስተማሪ
+ሐኪም
+ጸሐፊ
+ሠራተኛ
+ዘበኛ
+ገበሬ
+ወንድም
+አባት
+እናት
+ወንድ
+ሴት
+ሴትዮ
+ባለቤት
+
+# animals
+ፈረስ
+በሬ
+በቅሎ
+ላም
+ዶሮ
+
+# things and places
+ቤት
+ጓዳ
+ጣራ
+መጽሐፍ
+ወረቀት
+ብዕር
+በር
+ወንበር
+ጠረጴዛ
+እርሳስ
+እንጀራ
+ልብስ
+ድንጋይ
+መሰንቆ
+ከተማ
+አገር
+ትምህርት
+ጌታ
+
+# adjectives and other stems used in examples
+ችግር
+ክፉ
+ደግ
+ብቻ
+
+# pronouns and demonstratives (take ን, ና, ም)
+እኔ
+እኛ
+አንተ
+አንቺ
+እሱ
+እሷ
+እርሱ
+እርሷ
+እነሱ
+እናንተ
+ይህ
+ይህች
+ያ
+ያች
+እነዚህ
+እነዚያ
+
+# common function words: never split, whatever suffix they happen to end with
+ነው
+ናቸው
+ናት
+ነኝ
+ነህ
+ነሽ
+ነበር
+ነበሩ
+ነበረች
+አይደለም
+አይደሉም
+ውስጥ
+ላይ
+ጋር
+ግን
+እና
+ወይም
+እንዲሁም
+ሁሉ
+ሁሉም
+እንደ
+ወደ
+ስለ
+ብዙ
+ትንሽ
+ዛሬ
+ነገር

amharic_segmenter/morphology.py

@@ -0,0 +1,182 @@
+"""Rule-based morphological segmentation for Amharic nouns and noun phrases.
+
+The rules implemented here come from the grammar book *አማርኛ ሠዋሰው* (1963 E.C.):
+prepositions that attach to the following word, the plural, the definite
+article and possessive suffixes, the object marker ``ን`` and the conjunctions
+``ና`` / ``ም``.  See ``docs/GRAMMAR_RULES.md`` for the rule-by-rule mapping.
+
+A candidate split is only accepted when the remaining *stem* is trustworthy:
+either it is in the lexicon (default, high precision) or, in the opt-in
+``"rules"`` strategy, it is at least ``min_stem`` letters long.
+"""
+from __future__ import annotations
+
+import re
+from collections.abc import Iterator, Set
+from dataclasses import dataclass
+
+from .charset import KAEB, RABI, SABI, SADIS, fidel_order, is_wa_form, with_order
+
+# Prefixes written together with the word that follows them.
+PREFIXES: tuple[str, ...] = ("ስለ", "በ", "ከ", "ለ", "የ")
+MAX_PREFIXES = 2
+
+# Suffix slots, from the stem outward:  plural -> possessive/definite -> ን -> ና/ም
+CONJUNCTIONS: tuple[str, ...] = ("ና", "ም")
+OBJECT_MARKER = "ን"
+POSSESSIVE_DEFINITE: tuple[str, ...] = ("ቸው", "ችሁ", "ችን", "ው", "ዋ", "ህ", "ሽ", "ዬ")
+# Suffixes before which the stem's last letter surfaces in the 'a' order
+# (አገር + ችሁ -> አገራችሁ, ተማሪዎች + ችን -> ተማሪዎቻችን).
+_LINKING: tuple[str, ...] = ("ቸው", "ችሁ", "ችን")
+
+# Prepositions fuse with pronouns that start with እ: በ + እኔ -> በኔ, ከ + እኛ -> ከኛ,
+# እንደ + አንተ -> እንዳንተ.
+_CONTRACTING_PRONOUNS: tuple[str, ...] = ("እኔ", "እኛ", "እሱ", "እሷ", "እነሱ", "እርሱ", "እርሷ", "እናንተ")
+CONTRACTION_PREFIXES: tuple[str, ...] = ("በ", "ከ", "ለ", "የ", "እንደ")
+
+
+def _build_contractions() -> dict[str, tuple[str, str]]:
+    table = {}
+    for prefix in CONTRACTION_PREFIXES:
+        for pronoun in _CONTRACTING_PRONOUNS:
+            table[prefix + pronoun[1:]] = (prefix, pronoun)
+    for pronoun in ("አንተ", "አንቺ"):
+        table["እንዳ" + pronoun[1:]] = ("እንደ", pronoun)
+    return table
+
+
+CONTRACTIONS = _build_contractions()
+
+_ETHIOPIC_WORD = re.compile(r"^[\u1200-\u135A]+$")
+
+
+@dataclass(frozen=True)
+class Analysis:
+    """Result of segmenting one word.
+
+    ``morphemes`` are normalised morphs (underlying forms), so concatenating
+    them does not always reproduce the surface word (e.g. ልጆች -> ልጅ + ች).
+    """
+
+    word: str
+    prefixes: tuple[str, ...] = ()
+    stem: str = ""
+    suffixes: tuple[str, ...] = ()
+
+    @property
+    def morphemes(self) -> list[str]:
+        return [*self.prefixes, self.stem, *self.suffixes]
+
+    @property
+    def is_split(self) -> bool:
+        return bool(self.prefixes or self.suffixes)
+
+
+def _opt(word: str, suffixes: tuple[str, ...]) -> Iterator[tuple[str, str | None]]:
+    yield word, None
+    for s in suffixes:
+        if len(word) > len(s) and word.endswith(s):
+            yield word[: -len(s)], s
+
+
+def _opt_possessive(word: str) -> Iterator[tuple[str, str | None, bool]]:
+    """Possessive / definite-article slot, including fused surface forms.
+
+    The third item is ``True`` for *fused* forms (ሐኪሟ, ልጁ), which only nouns have.
+    They are accepted for known stems only, so verbs such as አየሁ are never touched.
+    """
+    yield word, None, False
+    for s in POSSESSIVE_DEFINITE:
+        if len(word) > len(s) and word.endswith(s):
+            rest = word[: -len(s)]
+            yield rest, s, False
+            if s in _LINKING and rest:
+                last = rest[-1]
+                if fidel_order(last) == RABI:  # አገራችሁ -> አገር, ተማሪዎቻችን -> ተማሪዎች
+                    yield rest[:-1] + with_order(last, SADIS), s, False
+    # feminine 'ዋ' fused into the last letter (labialised form): ሐኪሟ -> ሐኪም + ዋ
+    if len(word) >= 2 and is_wa_form(word[-1]):
+        yield word[:-1] + chr(ord(word[-1]) - 2), "ዋ", True
+    # masculine definite 'ኡ' fused into the last letter: ልጁ -> ልጅ, ተማሪዎቹ -> ተማሪዎች
+    if len(word) >= 2 and fidel_order(word[-1]) == KAEB:
+        yield word[:-1] + with_order(word[-1], SADIS), "ው", True
+
+
+def _opt_plural(word: str) -> Iterator[tuple[str, str | None]]:
+    """Plural slot (the 1948 spelling convention described in the book)."""
+    yield word, None
+    if len(word) > 2 and word.endswith("ዎች"):
+        yield word[:-2], "ዎች"
+    if len(word) >= 2 and word.endswith("ች"):
+        stem0, prev = word[:-2], word[-2]
+        if fidel_order(prev) == SABI:
+            for order in (SADIS, RABI):  # ሰዎች -> ሰው, ጌቶች -> ጌታ
+                yield stem0 + with_order(prev, order), "ች"
+            yield stem0 + prev, "ች"  # በቅሎች -> በቅሎ
+
+
+def _suffix_variants(word: str) -> Iterator[tuple[str, tuple[str, ...], bool]]:
+    for w1, conj in _opt(word, CONJUNCTIONS):
+        for w2, obj in _opt(w1, (OBJECT_MARKER,)):
+            for w3, pos, fused in _opt_possessive(w2):
+                for w4, plural in _opt_plural(w3):
+                    yield w4, tuple(x for x in (plural, pos, obj, conj) if x), fused
+
+
+def _prefix_variants(stem: str, depth: int = MAX_PREFIXES) -> Iterator[tuple[tuple[str, ...], str]]:
+    yield (), stem
+    if depth == 0:
+        return
+    for p in PREFIXES:
+        if len(stem) > len(p) and stem.startswith(p):
+            for rest_prefixes, rest in _prefix_variants(stem[len(p):], depth - 1):
+                yield (p, *rest_prefixes), rest
+
+
+def analyze_word(
+    word: str,
+    lexicon: Set[str],
+    strategy: str = "lexicon",
+    min_stem: int = 2,
+) -> Analysis:
+    """Segment ``word`` into prefixes, a stem and suffixes.
+
+    ``strategy``:
+      * ``"lexicon"`` - accept a split only if the stem is a known word.
+      * ``"rules"``   - also accept any stem of at least ``min_stem`` letters
+        (higher recall, more false splits).
+      * ``"none"``    - never split.
+    """
+    unsplit = Analysis(word, (), word, ())
+    if strategy == "none" or word in lexicon or not _ETHIOPIC_WORD.match(word):
+        return unsplit
+
+    # Ranking of candidate analyses:
+    #  * a stem found in the lexicon always beats an unknown one; among those,
+    #    fewer affixes and then the longer stem win (conservative);
+    #  * for unknown stems ("rules" strategy) avoid guessing prefixes, then peel
+    #    as many suffixes as possible (e.g. ቸው rather than only ው).
+    best: Analysis | None = None
+    best_key: tuple[int, ...] | None = None
+    for stem_candidate, suffixes, fused in _suffix_variants(word):
+        contraction = CONTRACTIONS.get(stem_candidate)
+        if contraction is not None and contraction[1] in lexicon:
+            prefix, pronoun = contraction
+            key = (0, 1 + len(suffixes), -len(pronoun))
+            if best_key is None or key < best_key:
+                best_key, best = key, Analysis(word, (prefix,), pronoun, suffixes)
+        for prefixes, stem in _prefix_variants(stem_candidate):
+            if not (prefixes or suffixes):
+                continue
+            known = stem in lexicon
+            if fused and not known:
+                continue  # fused forms are only trusted for words in the lexicon
+            if not (known or (strategy == "rules" and len(stem) >= min_stem)):
+                continue
+            if known:
+                key: tuple[int, ...] = (0, len(prefixes) + len(suffixes), -len(stem))
+            else:
+                key = (1, len(prefixes), -len(suffixes), len(stem))
+            if best_key is None or key < best_key:
+                best_key, best = key, Analysis(word, prefixes, stem, suffixes)
+    return best or unsplit

amharic_segmenter/normalize.py

@@ -0,0 +1,29 @@
+"""Optional orthographic normalisation for Amharic text.
+
+Amharic has several groups of letters that are pronounced identically
+(ሀ/ሐ/ኀ, ሰ/ሠ, አ/ዐ, ጸ/ፀ).  Writers use them inconsistently, which fragments
+vocabulary in NLP pipelines.  :func:`normalize_homophones` collapses each
+group to one representative while preserving the vowel order, and it never
+changes the length of the text (so token offsets stay valid).
+"""
+from __future__ import annotations
+
+# source block start -> representative block start (7 vowel orders each)
+_BLOCKS = {
+    0x1210: 0x1200,  # ሐ -> ሀ
+    0x1280: 0x1200,  # ኀ -> ሀ
+    0x1220: 0x1230,  # ሠ -> ሰ
+    0x12D0: 0x12A0,  # ዐ -> አ
+    0x1340: 0x1338,  # ፀ -> ጸ
+}
+
+_TABLE = {
+    src + i: dst + i
+    for src, dst in _BLOCKS.items()
+    for i in range(7)
+}
+
+
+def normalize_homophones(text: str) -> str:
+    """Collapse homophone letter families to a single representative."""
+    return text.translate(_TABLE)

amharic_segmenter/tokenizer.py

@@ -0,0 +1,309 @@
+"""Word, sentence and morpheme tokenization for Amharic."""
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable, Iterator
+from functools import lru_cache
+from importlib import resources
+from pathlib import Path
+from typing import NamedTuple
+
+from .morphology import Analysis, analyze_word
+from .normalize import normalize_homophones
+
+_STRATEGIES = ("lexicon", "rules", "none")
+
+# One alternation, tried in order.  Every non-space character belongs to
+# exactly one token, so nothing is ever silently dropped.
+_TOKEN_RE = re.compile(
+    r"(?P<word>[\u1200-\u135A\u135D-\u135F\u1380-\u138F]+)"
+    r"|(?P<ethnum>[\u1369-\u137C]+)"
+    r"|(?P<latin>[A-Za-z\u00C0-\u024F]+(?:['\u2019\-][A-Za-z\u00C0-\u024F]+)*)"
+    r"|(?P<number>\d+(?:[.,]\d+)*)"
+    r"|(?P<punct>\u1361\u1361|[\u1360-\u1368]|[^\w\s])"
+    r"|(?P<other>\S)"
+)
+_KINDS = {
+    "word": "word",
+    "ethnum": "number",
+    "latin": "latin",
+    "number": "number",
+    "punct": "punct",
+    "other": "other",
+}
+
+# Sentence terminators: ። ፧ ፨ ! ? and the old-style double word-separator ፡፡
+_TERMINATOR_RE = re.compile(r"(?:\u1361\u1361|[\u1362\u1367\u1368!?]+)[\u00BB\u201D\"')\]]*")
+
+
+class Token(NamedTuple):
+    """A token with its character offsets in the (normalised) input text."""
+
+    text: str
+    start: int
+    end: int
+    kind: str  # "word", "latin", "number", "punct" or "other"
+
+
+def _parse_lexicon(text: str) -> Iterator[str]:
+    for line in text.splitlines():
+        line = line.strip()
+        if line and not line.startswith("#"):
+            yield line
+
+
+@lru_cache(maxsize=1)
+def load_irregular_plurals() -> tuple[tuple[str, str], ...]:
+    """Load (singular, plural) pairs of irregular plurals shipped with the package."""
+    resource = resources.files("amharic_segmenter") / "data" / "irregular_plurals.tsv"
+    pairs = []
+    for line in _parse_lexicon(resource.read_text(encoding="utf-8")):
+        parts = line.split("\t")
+        if len(parts) == 2:
+            pairs.append((parts[0].strip(), parts[1].strip()))
+    return tuple(pairs)
+
+
+@lru_cache(maxsize=1)
+def load_default_lexicon() -> frozenset:
+    """Load the seed lexicon (plus irregular plural forms) shipped with the package."""
+    resource = resources.files("amharic_segmenter") / "data" / "lexicon.txt"
+    words = set(_parse_lexicon(resource.read_text(encoding="utf-8")))
+    for singular, plural in load_irregular_plurals():
+        words.update((singular, plural))
+    return frozenset(words)
+
+
+class AmharicTokenizer:
+    """Configurable Amharic tokenizer.
+
+    Parameters
+    ----------
+    lexicon:
+        Extra stems (an iterable of words) that are allowed to be split off.
+    default_lexicon:
+        Include the seed lexicon that ships with the package.
+    strategy:
+        ``"lexicon"`` (default, high precision), ``"rules"`` (higher recall) or
+        ``"none"`` (never split morphemes).
+    normalize:
+        Collapse homophone letters (ሐ->ሀ, ሠ->ሰ, ዐ->አ, ፀ->ጸ) before tokenizing.
+    min_stem:
+        Minimum stem length (in letters) for the ``"rules"`` strategy.
+    """
+
+    def __init__(
+        self,
+        lexicon: Iterable[str] | None = None,
+        *,
+        default_lexicon: bool = True,
+        strategy: str = "lexicon",
+        normalize: bool = False,
+        min_stem: int = 2,
+    ) -> None:
+        if strategy not in _STRATEGIES:
+            raise ValueError(f"strategy must be one of {_STRATEGIES}, got {strategy!r}")
+        self.strategy = strategy
+        self.normalize = normalize
+        self.min_stem = min_stem
+        words = set(load_default_lexicon()) if default_lexicon else set()
+        if lexicon is not None:
+            words.update(w.strip() for w in lexicon if w.strip())
+        self._lexicon = frozenset(self._prep(w) for w in words)
+        self._irregular: dict[str, str] = (
+            {self._prep(pl): self._prep(sg) for sg, pl in load_irregular_plurals()}
+            if default_lexicon else {}
+        )
+        self._cache: dict[str, Analysis] = {}
+        self._rules_cache: dict[str, Analysis] = {}
+
+    # -- helpers ---------------------------------------------------------
+    def _prep(self, text: str) -> str:
+        return normalize_homophones(text) if self.normalize else text
+
+    @classmethod
+    def from_file(cls, path: str | Path, **kwargs) -> AmharicTokenizer:
+        """Build a tokenizer with extra stems read from a UTF-8 file."""
+        words = list(_parse_lexicon(Path(path).read_text(encoding="utf-8")))
+        return cls(lexicon=words, **kwargs)
+
+    def add_words(self, words: Iterable[str]) -> None:
+        """Add stems to the lexicon of this tokenizer."""
+        self._lexicon = self._lexicon | {self._prep(w.strip()) for w in words if w.strip()}
+        self._cache.clear()
+        self._rules_cache.clear()
+
+    @property
+    def lexicon(self) -> frozenset:
+        return self._lexicon
+
+    # -- morphology ------------------------------------------------------
+    def analyze(self, word: str) -> Analysis:
+        """Segment one word into prefixes, stem and suffixes."""
+        word = self._prep(word)
+        cached = self._cache.get(word)
+        if cached is None:
+            cached = analyze_word(word, self._lexicon, self.strategy, self.min_stem)
+            self._cache[word] = cached
+        return cached
+
+    def split_morphemes(self, word: str) -> list[str]:
+        """Return the morphemes of one word, e.g. ``['ለ', 'ተማሪ', 'ዎች']``."""
+        return self.analyze(word).morphemes
+
+    def stem(self, word: str, *, aggressive: bool = True) -> str:
+        """Strip prefixes and suffixes and return the stem (like ``PorterStemmer.stem``).
+
+        Known words are segmented with the lexicon.  With ``aggressive=True`` (the
+        default) a word that is not in the lexicon still loses its suffixes using
+        the grammar rules alone, so the stemmer works on any noun, as English
+        stemmers do.  Use ``aggressive=False`` to leave unknown words untouched.
+        """
+        word = self._prep(word)
+        analysis = self.analyze(word)
+        if not analysis.is_split and aggressive and self.strategy == "lexicon":
+            analysis = self._rules_cache.get(word)  # type: ignore[assignment]
+            if analysis is None:
+                analysis = analyze_word(word, self._lexicon, "rules", self.min_stem)
+                self._rules_cache[word] = analysis
+        return analysis.stem
+
+    def lemma(self, word: str) -> str:
+        """Dictionary form of the stem: ``መጻሕፍትን -> መጽሐፍ``, ``ተማሪዎች -> ተማሪ``."""
+        stem = self.analyze(word).stem
+        return self._irregular.get(stem, stem)
+
+    # -- tokenization ----------------------------------------------------
+    def tokenize_spans(self, text: str, *, include_punct: bool = True) -> list[Token]:
+        """Word-level tokens with offsets (into the normalised text)."""
+        text = self._prep(text)
+        tokens = []
+        for m in _TOKEN_RE.finditer(text):
+            kind = _KINDS[m.lastgroup]  # type: ignore[index]
+            if kind == "punct" and not include_punct:
+                continue
+            tokens.append(Token(m.group(), m.start(), m.end(), kind))
+        return tokens
+
+    def tokenize(
+        self,
+        text: str,
+        *,
+        morphemes: bool = False,
+        include_punct: bool = True,
+        mark_affixes: bool = False,
+    ) -> list[str]:
+        """Split text into tokens.
+
+        With ``morphemes=True`` every Amharic word is further split into its
+        prefixes, stem and suffixes.  ``mark_affixes=True`` writes prefixes as
+        ``በ+`` and suffixes as ``+ን`` so the two can be told apart.
+        """
+        out: list[str] = []
+        for tok in self.tokenize_spans(text, include_punct=include_punct):
+            if morphemes and tok.kind == "word":
+                a = self.analyze(tok.text)
+                if mark_affixes:
+                    out.extend(p + "+" for p in a.prefixes)
+                    out.append(a.stem)
+                    out.extend("+" + s for s in a.suffixes)
+                else:
+                    out.extend(a.morphemes)
+            else:
+                out.append(tok.text)
+        return out
+
+    def sentence_tokenize(self, text: str) -> list[str]:
+        """Split text into sentences on ። ፧ ፨ ! ? and ፡፡."""
+        sentences: list[str] = []
+        start = 0
+        for m in _TERMINATOR_RE.finditer(text):
+            end = m.end()
+            if end == len(text) or text[end].isspace():
+                chunk = text[start:end].strip()
+                if chunk:
+                    sentences.append(chunk)
+                start = end
+        tail = text[start:].strip()
+        if tail:
+            sentences.append(tail)
+        return sentences
+
+
+class AmharicStemmer:
+    """Stemmer with the familiar NLTK shape: ``AmharicStemmer().stem("ተማሪዎችን")``."""
+
+    def __init__(self, tokenizer: AmharicTokenizer | None = None, *, aggressive: bool = True) -> None:
+        self.tokenizer = tokenizer or _default_tokenizer()
+        self.aggressive = aggressive
+
+    def stem(self, word: str) -> str:
+        """Return the stem of one word."""
+        return self.tokenizer.stem(word, aggressive=self.aggressive)
+
+    def stem_words(self, words: Iterable[str]) -> list[str]:
+        """Return the stems of many words."""
+        return [self.stem(w) for w in words]
+
+
+@lru_cache(maxsize=1)
+def _default_tokenizer() -> AmharicTokenizer:
+    return AmharicTokenizer()
+
+
+def word_tokenize(text: str) -> list[str]:
+    """Split text into words and punctuation (like ``nltk.word_tokenize``)."""
+    return _default_tokenizer().tokenize(text)
+
+
+def sent_tokenize(text: str) -> list[str]:
+    """Split text into sentences (like ``nltk.sent_tokenize``)."""
+    return _default_tokenizer().sentence_tokenize(text)
+
+
+def stem(word: str, *, aggressive: bool = True) -> str:
+    """Stem of one word, using the default tokenizer."""
+    return _default_tokenizer().stem(word, aggressive=aggressive)
+
+
+def stem_words(words: Iterable[str], *, aggressive: bool = True) -> list[str]:
+    """Stems of many words, using the default tokenizer."""
+    return [stem(w, aggressive=aggressive) for w in words]
+
+
+def tokenize(
+    text: str,
+    *,
+    morphemes: bool = False,
+    include_punct: bool = True,
+    mark_affixes: bool = False,
+) -> list[str]:
+    """Tokenize ``text`` with the default tokenizer."""
+    return _default_tokenizer().tokenize(
+        text, morphemes=morphemes, include_punct=include_punct, mark_affixes=mark_affixes
+    )
+
+
+def tokenize_spans(text: str, *, include_punct: bool = True) -> list[Token]:
+    """Tokens with offsets, using the default tokenizer."""
+    return _default_tokenizer().tokenize_spans(text, include_punct=include_punct)
+
+
+def sentence_tokenize(text: str) -> list[str]:
+    """Split ``text`` into sentences with the default tokenizer."""
+    return _default_tokenizer().sentence_tokenize(text)
+
+
+def analyze(word: str) -> Analysis:
+    """Morphological analysis of a single word."""
+    return _default_tokenizer().analyze(word)
+
+
+def split_morphemes(word: str) -> list[str]:
+    """Morphemes of a single word, using the default tokenizer."""
+    return _default_tokenizer().split_morphemes(word)
+
+
+def lemma(word: str) -> str:
+    """Dictionary form of the stem of ``word``, using the default tokenizer."""
+    return _default_tokenizer().lemma(word)

amharic_segmenter-0.1.0.dist-info/METADATA

@@ -0,0 +1,340 @@
+Metadata-Version: 2.5
+Name: amharic-segmenter
+Version: 0.1.0
+Summary: Sentence/word tokenization, morpheme splitting and stemming for Amharic (አማርኛ), NLTK-style and dependency-free.
+Project-URL: Homepage, https://github.com/Alki45/amharic-segmenter
+Project-URL: Documentation, https://github.com/Alki45/amharic-segmenter#readme
+Project-URL: Issues, https://github.com/Alki45/amharic-segmenter/issues
+Project-URL: Changelog, https://github.com/Alki45/amharic-segmenter/blob/main/CHANGELOG.md
+Author-email: Ali Kibret Muhamed <alikibretmuhamed@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: amharic,ethiopic,ge'ez,low-resource,morphology,nlp,stemmer,stemming,tokenizer,አማርኛ
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest>=7; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: twine; extra == 'dev'
+Description-Content-Type: text/markdown
+
+<div align="center">
+
+# amharic-segmenter
+
+**Sentence and word tokenization, morpheme splitting and stemming for Amharic (አማርኛ)**
+as easy as English NLP, built on real grammar rules, with zero dependencies.
+
+[![PyPI](https://img.shields.io/pypi/v/amharic-segmenter.svg)](https://pypi.org/project/amharic-segmenter/)
+[![Python](https://img.shields.io/pypi/pyversions/amharic-segmenter.svg)](https://pypi.org/project/amharic-segmenter/)
+[![CI](https://github.com/Alki45/amharic-segmenter/actions/workflows/ci.yml/badge.svg)](https://github.com/Alki45/amharic-segmenter/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+[![Code style: ruff](https://img.shields.io/badge/code%20style-ruff-261230.svg)](https://github.com/astral-sh/ruff)
+
+[Install](#installation) · [Quickstart](#quickstart) · [Features](#features) · [API](#api-reference) · [CLI](#command-line) · [How it works](#how-it-works) · [Contributing](#contributing)
+
+</div>
+
+> **በአማርኛ:** አማርኛ ጽሑፍን ወደ ዓረፍተ ነገሮችና ቃላት የሚከፋፍል፣ እንዲሁም ቃላትን ወደ ሞርፊሞች
+> (`በ`፣ `ከ`፣ `ለ`፣ `ን`፣ `ና`፣ `ዎች`…) የሚሰብር ቀላል የፓይቶን ጥቅል። ለማንኛውም የአማርኛ ቋንቋ ፕሮጀክት ነጻ ነው።
+
+---
+
+## Why this exists
+
+General-purpose tokenizers split on whitespace or learn subwords from data. For Amharic that
+leaves two problems:
+
+1. **Ethiopic punctuation is not ASCII.** `።` (full stop), `፣` (comma), `፧` (question mark) and
+   the old word separator `፡` are missed or glued to words by generic tools.
+2. **Amharic is morphologically rich.** Prepositions, plural, possessives, the object marker
+   `ን` and conjunctions are written *attached* to the word, so `ለተማሪዎቻችን` ("for our
+   students") is one whitespace token but five units of meaning.
+
+`amharic-segmenter` handles both with transparent, documented rules taken from an Amharic
+grammar book, so results are explainable and easy to improve.
+
+## Features
+
+- **NLTK-style API**: `sent_tokenize`, `word_tokenize`, `stem`, `AmharicStemmer`. Works right
+  after `pip install`: no model downloads, no setup
+- **Stemming** that strips prefixes and suffixes, also for words it has never seen
+- **Sentence splitting** on `።`, `፧`, `፨`, `!`, `?` and the old-style `፡፡`
+- **Word tokenization** that understands Ethiopic letters, punctuation and numerals
+  (`፩፻`), plus Latin text and decimals, and **never drops a character**
+- **Morpheme segmentation** of nouns and noun phrases: prepositions `በ ከ ለ የ ስለ`, plural
+  (`ች`, `ዎች`), definite article, possessive suffixes, object marker `ን`, conjunctions `ና`/`ም`
+- **Precision-first design:** words are only split when the stem is known, so names like
+  `ከበደ` and words like `ወንድም` stay intact. An opt-in `rules` mode trades precision for recall
+- Handles **pronoun contractions** (`በኔ → በ + እኔ`), **fused feminine forms** (`ሐኪሟ → ሐኪም + ዋ`)
+  and **irregular plurals** (`መጻሕፍትን`, `ቃላቶች`), with `lemma()` to get the dictionary form
+- **Token offsets** for aligning tokens with the original text
+- **Optional normalization** of homophone letters (`ሐ→ሀ`, `ሠ→ሰ`, `ዐ→አ`, `ፀ→ጸ`)
+- **Extensible lexicon**: bring your own words, no retraining needed
+- **Command line tool**, fully typed (`py.typed`), pure Python, **no dependencies**,
+  Python 3.9+
+
+## Installation
+
+```bash
+pip install amharic-segmenter
+```
+
+## Quickstart
+
+If you have used NLTK on English text, you already know how to use it. Install, import, call:
+
+```python
+from amharic_segmenter import word_tokenize, sent_tokenize, stem
+
+text = "ተማሪዎቹ መጻሕፍትን አነበቡ። ከበደና አበበ ለተማሪዎቻቸው ሰጡ።"
+
+sent_tokenize(text)
+# ['ተማሪዎቹ መጻሕፍትን አነበቡ።', 'ከበደና አበበ ለተማሪዎቻቸው ሰጡ።']
+
+word_tokenize(text)
+# ['ተማሪዎቹ', 'መጻሕፍትን', 'አነበቡ', '።', 'ከበደና', 'አበበ', 'ለተማሪዎቻቸው', 'ሰጡ', '።']
+
+[stem(w) for w in word_tokenize(text)]
+# ['ተማሪ', 'መጻሕፍት', 'አነበቡ', '።', 'ከበደ', 'አበበ', 'ተማሪ', 'ሰጡ', '።']
+```
+
+Or with a stemmer object, the way you would use `PorterStemmer`:
+
+```python
+from amharic_segmenter import AmharicStemmer
+
+stemmer = AmharicStemmer()
+stemmer.stem("ተማሪዎችን")                     # 'ተማሪ'
+stemmer.stem_words(["ተማሪዎች", "ላሞች"])      # ['ተማሪ', 'ላም']
+```
+
+| English (NLTK) | Amharic (this package) |
+|----------------|------------------------|
+| `nltk.sent_tokenize(text)` | `sent_tokenize(text)` |
+| `nltk.word_tokenize(text)` | `word_tokenize(text)` |
+| `PorterStemmer().stem(word)` | `AmharicStemmer().stem(word)` or `stem(word)` |
+| `nltk.download("punkt")` | **nothing to download**: rules and data ship inside the package |
+
+### More examples
+
+```python
+from amharic_segmenter import tokenize, sentence_tokenize
+
+text = "ተማሪዎቹ በከተማ ውስጥ ይኖራሉ። ከበደና አበበ ለተማሪዎቻችን መጽሐፎችን ሰጡ።"
+
+sentence_tokenize(text)
+# ['ተማሪዎቹ በከተማ ውስጥ ይኖራሉ።', 'ከበደና አበበ ለተማሪዎቻችን መጽሐፎችን ሰጡ።']
+
+tokenize(text)
+# ['ተማሪዎቹ', 'በከተማ', 'ውስጥ', 'ይኖራሉ', '።', 'ከበደና', 'አበበ', 'ለተማሪዎቻችን', 'መጽሐፎችን', 'ሰጡ', '።']
+
+tokenize(text, morphemes=True)
+# ['ተማሪ', 'ዎች', 'ው', 'በ', 'ከተማ', 'ውስጥ', 'ይኖራሉ', '።', 'ከበደ', 'ና', 'አበበ',
+#  'ለ', 'ተማሪ', 'ዎች', 'ችን', 'መጽሐፍ', 'ች', 'ን', 'ሰጡ', '።']
+
+tokenize(text, morphemes=True, mark_affixes=True)
+# ['ተማሪ', '+ዎች', '+ው', 'በ+', 'ከተማ', 'ውስጥ', 'ይኖራሉ', '።', 'ከበደ', '+ና', 'አበበ',
+#  'ለ+', 'ተማሪ', '+ዎች', '+ችን', 'መጽሐፍ', '+ች', '+ን', 'ሰጡ', '።']
+```
+
+Analyse a single word:
+
+```python
+from amharic_segmenter import analyze
+
+analyze("ለተማሪዎቻችን")
+# Analysis(word='ለተማሪዎቻችን', prefixes=('ለ',), stem='ተማሪ', suffixes=('ዎች', 'ችን'))
+```
+
+Token offsets:
+
+```python
+from amharic_segmenter import tokenize_spans
+
+tokenize_spans("ከበደን አየ።")
+# [Token(text='ከበደን', start=0, end=4, kind='word'),
+#  Token(text='አየ', start=5, end=7, kind='word'),
+#  Token(text='።', start=7, end=8, kind='punct')]
+```
+
+## Command line
+
+```console
+$ amharic-segmenter -m "ለተማሪዎቻችን መጽሐፎችን ሰጡ።"
+ለ ተማሪ ዎች ችን መጽሐፍ ች ን ሰጡ ።
+
+$ amharic-segmenter --sentences --json "አበበ መጣ። ከበደ ሄደ።"
+["አበበ መጣ።", "ከበደ ሄደ።"]
+
+$ amharic-segmenter --stem "ተማሪዎቹ መጻሕፍትን አነበቡ።"
+ተማሪ መጻሕፍት አነበቡ ።
+
+$ echo "ተማሪውንም አየሁ።" | amharic-segmenter -m --mark-affixes
+ተማሪ +ው +ን +ም አየሁ ።
+```
+
+Run `amharic-segmenter --help` for all options (`--strategy`, `--lexicon`, `--normalize`,
+`--no-punct`, `--json`). `python -m amharic_segmenter` works too.
+
+## API reference
+
+### Module-level functions
+
+| Function | Returns | Description |
+|----------|---------|-------------|
+| `sent_tokenize(text)` | `list[str]` | Sentences (NLTK-style name for `sentence_tokenize`) |
+| `word_tokenize(text)` | `list[str]` | Words and punctuation (NLTK-style, same as `tokenize(text)`) |
+| `stem(word, *, aggressive=True)` | `str` | Stem of one word. `aggressive=False` leaves words outside the lexicon alone |
+| `stem_words(words)` | `list[str]` | Stems of many words |
+| `AmharicStemmer().stem(word)` | `str` | Stemmer object, `.stem()` and `.stem_words()` |
+| `tokenize(text, *, morphemes=False, include_punct=True, mark_affixes=False)` | `list[str]` | Split text into tokens, optionally down to morphemes |
+| `tokenize_spans(text, *, include_punct=True)` | `list[Token]` | Word-level tokens with `start`/`end` offsets and `kind` |
+| `sentence_tokenize(text)` | `list[str]` | Split text into sentences |
+| `analyze(word)` | `Analysis` | Prefixes, stem and suffixes of one word |
+| `split_morphemes(word)` | `list[str]` | Morphemes of one word |
+| `lemma(word)` | `str` | Dictionary form of the stem (`መጻሕፍትን → መጽሐፍ`) |
+| `normalize_homophones(text)` | `str` | Collapse `ሐ ኀ → ሀ`, `ሠ → ሰ`, `ዐ → አ`, `ፀ → ጸ` (length-preserving) |
+
+### `AmharicTokenizer`
+
+```python
+from amharic_segmenter import AmharicTokenizer
+
+tok = AmharicTokenizer(
+    lexicon=["መምህር", "ጓደኛ"],   # extra stems that may be split off
+    default_lexicon=True,        # include the built-in seed lexicon
+    strategy="lexicon",          # "lexicon" | "rules" | "none"
+    normalize=False,             # collapse homophone letters first
+)
+
+tok.tokenize("መምህራችንን አየሁ።", morphemes=True)
+tok.add_words(["ጋዜጣ"])
+tok = AmharicTokenizer.from_file("my_words.txt")   # one word per line, '#' comments
+```
+
+| `strategy` | Behaviour |
+|------------|-----------|
+| `"lexicon"` *(default)* | Accept a split only if the stem is in the lexicon. High precision. |
+| `"rules"` | Also accept unknown stems of at least `min_stem` letters. Higher recall, more false splits. |
+| `"none"` | Never split words into morphemes. |
+
+### `Analysis` and `Token`
+
+- `Analysis(word, prefixes, stem, suffixes)` with `.morphemes` and `.is_split`
+- `Token(text, start, end, kind)` where `kind` is `word`, `latin`, `number`, `punct` or `other`
+
+> **Note:** morphemes are *underlying forms*. `ልጆች` becomes `ልጅ + ች` because the plural rule
+> changes the stem's last letter, so joining the morphemes does not always give back the
+> surface word. Use `tokenize_spans` when you need exact offsets.
+
+## How it works
+
+```
+ raw text
+    │
+    ▼
+ ┌──────────────────────────┐   Layer 1: rule-based, always on
+ │ sentences and word tokens │   Ethiopic punctuation, numerals, Latin, no loss
+ └─────────────┬────────────┘
+               │  (optional, morphemes=True)
+               ▼
+ ┌──────────────────────────┐   Layer 2: morphology
+ │ generate candidate splits │   prefixes ← stem → plural → possessive → ን → ና/ም
+ │ accept if stem is known   │   lexicon check keeps precision high
+ └──────────────────────────┘
+```
+
+The morphology layer works on the structure of the Ethiopic script itself: every letter has
+seven vowel *orders*, and the plural and possessive rules are defined in terms of them
+(`ሰው → ሰዎች`, `አገር → አገራችሁ`). The rules are listed with book references in
+[`docs/GRAMMAR_RULES.md`](docs/GRAMMAR_RULES.md).
+
+The rules are based on the grammar book *አማርኛ ሠዋሰው* (1963 E.C.).
+
+## How it relates to other Amharic tools
+
+| Project (PyPI) | Approach | Good for |
+|----------------|----------|----------|
+| **`amharic-segmenter`** (this package) | Grammar rules from a printed grammar book, NLTK-style API | Sentence and word splitting, morpheme splitting, stemming, explainable output |
+| `amharic-tokenizer` | BPE-like merges learned from a corpus | Subword vocabularies for training language models |
+| `amseg` | Rule-based sentence and word splitter, with normalizer and romanizer | Sentence/word splitting and transliteration |
+
+They solve different problems and can be used together, for example `amharic-segmenter` for
+sentences and stems and a trained BPE model for model vocabularies.
+
+## Current scope and limitations
+
+This is an early release (`0.1`, alpha). Please read this before using it in production:
+
+- Only **printed pages 7-110** of the source book have been used so far (chapters 1-5 and the
+  start of chapter 6; the punctuation chapter is still missing). See [`docs/BOOK_COVERAGE.md`](docs/BOOK_COVERAGE.md) for exactly what is
+  and is not covered, e.g. the book's chapter on punctuation.
+- Stemming is a **noun-oriented** suffix stripper, like a light English stemmer. Verbs are
+  left alone rather than guessed. Prefixes (`በ ከ ለ የ`) are only stripped when the rest is a
+  known word, so unknown words lose suffixes only. For unknown nouns it can pick the wrong stem when the
+  last letter is ambiguous (`ጋዜጦች` could come from `ጋዜጣ` or `ጋዜጥ`).
+- Morphology covers **nouns and noun phrases**. **Verb morphology is not implemented yet.**
+- The built-in lexicon is a small seed. Unknown stems are left unsplit in the default mode,
+  so coverage grows as the lexicon grows. Add your own words or send a pull request.
+- Some forms are ambiguous without context (a final `ም` can be a conjunction or part of the
+  word, as in `ወንድም`). The lexicon resolves most cases, not all.
+- Rules reflect one grammar book. Modern usage and dialects may differ; native-speaker
+  review is very welcome.
+
+## Roadmap
+
+- [ ] Punctuation rules from chapter 7 of the source book (pages still to be added)
+- [ ] Verb morphology (agreement, tense, negation, relative `የ…`) from an additional source
+- [ ] Larger open lexicon and a test corpus of annotated sentences
+- [ ] Reduplicated plurals and internal plurals as rules
+- [ ] Subword-vocabulary export for use with ML tokenizers
+- [ ] Evaluation scripts against annotated Amharic data
+
+## Contributing
+
+Contributions are very welcome, especially from Amharic speakers and linguists. The easiest
+first contribution is adding words to
+[`src/amharic_segmenter/data/lexicon.txt`](src/amharic_segmenter/data/lexicon.txt).
+See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, rules for adding grammar rules, and the pull
+request checklist.
+
+```bash
+git clone https://github.com/Alki45/amharic-segmenter.git
+cd amharic-segmenter
+pip install -e ".[dev]"
+pytest
+```
+
+## Citation
+
+If you use this package in research, please cite it:
+
+```bibtex
+@software{amharic_segmenter,
+  title  = {amharic-segmenter: Sentence, word and morpheme tokenization for Amharic},
+  author = {Ali Kibret Muhamed},
+  year   = {2026},
+  url    = {https://github.com/Alki45/amharic-segmenter}
+}
+```
+
+## License
+
+[MIT](LICENSE). The grammar rules are re-implemented in original code; the source book is
+credited above and its text is not redistributed.

amharic_segmenter-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+amharic_segmenter/__init__.py,sha256=UqStY6cNNXRM2t0atM8qoEupn_o8BGR-qchBzJmxWr4,850
+amharic_segmenter/__main__.py,sha256=SCuGk-L-bb1zLj15YeA4J9PvoYe60R0p2hiL5YDrvuo,51
+amharic_segmenter/_version.py,sha256=QTYqXqSTHFRkM9TEgpDFcHvwLbvqHDqvqfQ9EiXkcAM,23
+amharic_segmenter/charset.py,sha256=zhYpKJ-yig3jiXFbX6ic1SYFvpVzDCqXqxvSXt2vHxg,2382
+amharic_segmenter/cli.py,sha256=l4n67KIbSr-hpnCgbml8wWIbF0EgsMeVqTDcrmstAAE,2936
+amharic_segmenter/morphology.py,sha256=Fv3xiYXKC4NojZANYMayjV0LhUAvC6C1paN6UIvxTrg,7960
+amharic_segmenter/normalize.py,sha256=TQuBio21vLFFmeZ4Kr82ykGOnxa0E5NDvxeT_8_Lk7c,1003
+amharic_segmenter/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+amharic_segmenter/tokenizer.py,sha256=8SrmHWZRQQkaopCGGAnF_U3YP_zsNd-mNjbAg-hyIvQ,11755
+amharic_segmenter/data/irregular_plurals.tsv,sha256=WEXdYZX2FF5tO41iGOr-n7ozDC1JtMdO6wYe0OxFjHw,1434
+amharic_segmenter/data/lexicon.txt,sha256=-jALh2Ot71drJJi_4zKhu0wtJn-7I2DVRlbfpt6go04,1706
+amharic_segmenter-0.1.0.dist-info/METADATA,sha256=FGxFHPEgqK4nExtTWkZSyqWe3nBDxjOy5qsRG5LIA_k,16280
+amharic_segmenter-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+amharic_segmenter-0.1.0.dist-info/entry_points.txt,sha256=hm-5Ntu_VCNDAgp71dImtTZ6i_vq-oU4TAvx37rvhPw,65
+amharic_segmenter-0.1.0.dist-info/licenses/LICENSE,sha256=FLUUa47oUr2WLIMBokbK5o4eqmPD_P5pJqrBm193irg,1096
+amharic_segmenter-0.1.0.dist-info/RECORD,,

amharic_segmenter-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

amharic_segmenter-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+amharic-segmenter = amharic_segmenter.cli:main

amharic_segmenter-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ali Kibret Muhamed
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.