yotext 0.1.0__tar.gz

yotext-0.1.0/.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

yotext-0.1.0/.gitignore

@@ -0,0 +1,7 @@
+.venv/
+__pycache__/
+*.pyc
+dist/
+build/
+*.egg-info/
+.pytest_cache/

yotext-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Adedeji Makinde
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

yotext-0.1.0/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.5
+Name: yotext
+Version: 0.1.0
+Summary: Orthographic normalization and diacritic handling for Yorùbá text
+Project-URL: Homepage, https://github.com/adedejimakinde/yotext
+Project-URL: Issues, https://github.com/adedejimakinde/yotext/issues
+Author-email: Adedeji Makinde <makindeadedeji5@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: african-languages,diacritics,low-resource,nlp,text-normalization,unicode,yoruba
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.9
+Provides-Extra: dev
+Requires-Dist: pytest; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# yotext
+
+Orthographic normalization and diacritic handling for Yorùbá text. Zero runtime dependencies. Requires Python 3.9 or newer.
+
+## Why this exists
+
+Yorùbá text in circulation is encoded inconsistently. The underdot that marks ẹ, ọ, and ṣ shows up as U+0323, U+0329, U+0331, or U+032D depending on the keyboard and the era it was typed on. Combining marks often arrive out of canonical order too. The same word ends up as several different byte sequences that look identical on screen and compare unequal in code. `standardize()` folds all of that into one canonical form.
+
+## Tone marks and underdots are different things
+
+This is the part that matters most. The underdot in ẹ, ọ, and ṣ is segmental. ẹ and e are separate phonemes, and removing the underdot turns one word into a different word. Tone marks, the acute in á and the grave in à, are suprasegmental. They mark pitch on top of a vowel, and they do not change which phoneme the vowel is.
+
+Most text preprocessing code lumps both of these under the single label "diacritics" and strips them together. That conflates two different linguistic categories, and it produces wrong output for any task that depends on the ẹ/e or ọ/o distinction.
+
+`strip_tones()` removes tone marks and keeps the underdot. `strip_diacritics()` removes both. I kept both functions because they answer different questions, and code that only offers one of them is answering the wrong question at least some of the time.
+
+## Install
+
+```
+pip install yotext
+```
+
+## Usage
+
+```python
+from yotext import standardize, strip_tones, strip_diacritics, tone_pattern
+
+# Messy input: wrong underdot codepoint (U+0329 instead of U+0323)
+# and a tone mark placed before the underdot instead of after.
+messy = "e\u0300\u0329ko\u0301\u0331"
+standardize(messy)
+# 'ẹ̀kọ́'
+
+strip_tones("ẹ̀kọ́")
+# 'ẹkọ'  tone dropped, underdot kept, ẹ and e stay different words
+
+strip_diacritics("ẹ̀kọ́")
+# 'eko'  fully undiacritized, the baseline used in ablation experiments
+
+tone_pattern("bàbá")
+# 'LH'  useful for checking tone distribution across a corpus
+```
+
+Two strings can look identical and still compare unequal, if one uses a precomposed vowel and the other uses a decomposed one with a non-canonical underdot codepoint.
+
+```python
+s1 = "\u1eb9"       # precomposed ẹ
+s2 = "e\u0329"      # decomposed, non-canonical underdot
+s1 == s2                       # False
+standardize(s1) == standardize(s2)  # True
+```
+
+Running `standardize()` on a corpus before deduplicating or indexing it collapses these variants so equal words compare equal.
+
+## What it guarantees
+
+`standardize()` is idempotent. Running it twice gives the same result as running it once.
+
+`strip_diacritics(s)` equals `strip_diacritics(strip_tones(s))` for any input. Removing tone first and then removing what remains lands on the same undiacritized form as removing everything at once.
+
+Combining marks always come out in canonical order, combining class 220 before combining class 230.
+
+## What it does not do
+
+Version 0.1 does normalization only. It does not restore diacritics. Predicting bẹ̀rẹ̀ from bere, meaning guessing which vowels carry an underdot or a tone mark from plain ASCII input, is not implemented. That is planned for a later release.
+
+## License
+
+MIT.

yotext-0.1.0/README.md

@@ -0,0 +1,69 @@
+# yotext
+
+Orthographic normalization and diacritic handling for Yorùbá text. Zero runtime dependencies. Requires Python 3.9 or newer.
+
+## Why this exists
+
+Yorùbá text in circulation is encoded inconsistently. The underdot that marks ẹ, ọ, and ṣ shows up as U+0323, U+0329, U+0331, or U+032D depending on the keyboard and the era it was typed on. Combining marks often arrive out of canonical order too. The same word ends up as several different byte sequences that look identical on screen and compare unequal in code. `standardize()` folds all of that into one canonical form.
+
+## Tone marks and underdots are different things
+
+This is the part that matters most. The underdot in ẹ, ọ, and ṣ is segmental. ẹ and e are separate phonemes, and removing the underdot turns one word into a different word. Tone marks, the acute in á and the grave in à, are suprasegmental. They mark pitch on top of a vowel, and they do not change which phoneme the vowel is.
+
+Most text preprocessing code lumps both of these under the single label "diacritics" and strips them together. That conflates two different linguistic categories, and it produces wrong output for any task that depends on the ẹ/e or ọ/o distinction.
+
+`strip_tones()` removes tone marks and keeps the underdot. `strip_diacritics()` removes both. I kept both functions because they answer different questions, and code that only offers one of them is answering the wrong question at least some of the time.
+
+## Install
+
+```
+pip install yotext
+```
+
+## Usage
+
+```python
+from yotext import standardize, strip_tones, strip_diacritics, tone_pattern
+
+# Messy input: wrong underdot codepoint (U+0329 instead of U+0323)
+# and a tone mark placed before the underdot instead of after.
+messy = "e\u0300\u0329ko\u0301\u0331"
+standardize(messy)
+# 'ẹ̀kọ́'
+
+strip_tones("ẹ̀kọ́")
+# 'ẹkọ'  tone dropped, underdot kept, ẹ and e stay different words
+
+strip_diacritics("ẹ̀kọ́")
+# 'eko'  fully undiacritized, the baseline used in ablation experiments
+
+tone_pattern("bàbá")
+# 'LH'  useful for checking tone distribution across a corpus
+```
+
+Two strings can look identical and still compare unequal, if one uses a precomposed vowel and the other uses a decomposed one with a non-canonical underdot codepoint.
+
+```python
+s1 = "\u1eb9"       # precomposed ẹ
+s2 = "e\u0329"      # decomposed, non-canonical underdot
+s1 == s2                       # False
+standardize(s1) == standardize(s2)  # True
+```
+
+Running `standardize()` on a corpus before deduplicating or indexing it collapses these variants so equal words compare equal.
+
+## What it guarantees
+
+`standardize()` is idempotent. Running it twice gives the same result as running it once.
+
+`strip_diacritics(s)` equals `strip_diacritics(strip_tones(s))` for any input. Removing tone first and then removing what remains lands on the same undiacritized form as removing everything at once.
+
+Combining marks always come out in canonical order, combining class 220 before combining class 230.
+
+## What it does not do
+
+Version 0.1 does normalization only. It does not restore diacritics. Predicting bẹ̀rẹ̀ from bere, meaning guessing which vowels carry an underdot or a tone mark from plain ASCII input, is not implemented. That is planned for a later release.
+
+## License
+
+MIT.

yotext-0.1.0/pyproject.toml

@@ -0,0 +1,34 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "yotext"
+version = "0.1.0"
+description = "Orthographic normalization and diacritic handling for Yorùbá text"
+requires-python = ">=3.9"
+dependencies = []
+authors = [{name = "Adedeji Makinde", email = "makindeadedeji5@gmail.com"}]
+readme = "README.md"
+license = "MIT"
+keywords = ["yoruba", "nlp", "unicode", "diacritics", "text-normalization", "african-languages", "low-resource"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+    "Topic :: Text Processing :: Linguistic",
+]
+
+[project.optional-dependencies]
+dev = ["pytest"]
+
+[project.urls]
+Homepage = "https://github.com/adedejimakinde/yotext"
+Issues = "https://github.com/adedejimakinde/yotext/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/yotext"]
+artifacts = ["src/yotext/data/*"]

yotext-0.1.0/src/yotext/__init__.py

@@ -0,0 +1,11 @@
+"""yotext: orthographic normalization and diacritic handling for Yorùbá text.
+
+Provides text standardization, tone-mark stripping/extraction, and
+diacritic handling for Yorùbá orthography.
+"""
+
+from .standardize import standardize
+from .tones import strip_tones, strip_diacritics, tone_pattern
+
+__version__ = "0.1.0"
+__all__ = ["standardize", "strip_tones", "strip_diacritics", "tone_pattern", "__version__"]

yotext-0.1.0/src/yotext/constants.py

@@ -0,0 +1,44 @@
+"""Constants used across the yotext package."""
+
+# Combining diacritical marks (Unicode combining characters).
+#
+# DOT_BELOW is SEGMENTAL: it distinguishes separate phonemes (e.g. e-dot-below
+# vs e, o-dot-below vs o) and is part of the letter identity, not the tone.
+# ACUTE, GRAVE, and MACRON are SUPRASEGMENTAL tone marks layered on top of a
+# vowel/syllabic nasal to indicate pitch (high, low, mid). These two
+# categories must never be conflated: stripping "diacritics" for
+# tone-insensitive comparison must not strip DOT_BELOW, and
+# normalizing/stripping underdots must not touch tone marks.
+DOT_BELOW = "\u0323"
+ACUTE     = "\u0301"
+GRAVE     = "\u0300"
+MACRON    = "\u0304"
+
+# Suprasegmental tone marks only. Does not include DOT_BELOW, which is
+# segmental (see note above).
+TONE_MARKS = frozenset({ACUTE, GRAVE, MACRON})
+
+# Non-canonical combining-below marks sometimes substituted for DOT_BELOW
+# (e.g. by fonts, keyboards, or OCR), mapped to the canonical DOT_BELOW.
+UNDERDOT_VARIANTS = {
+    "\u0329": DOT_BELOW,
+    "\u0331": DOT_BELOW,
+    "\u032D": DOT_BELOW,
+}
+
+# Base Latin vowel letters (uppercase and lowercase), before any diacritics.
+VOWELS = frozenset("aeiouAEIOU")
+
+# Syllabic nasal consonant letters (uppercase and lowercase).
+NASALS = frozenset("nmNM")
+
+# Invisible/zero-width Unicode characters to strip during cleanup, mapped
+# via str.translate (values are None to delete).
+INVISIBLES = dict.fromkeys(map(ord, "\u200b\u200c\u200d\ufeff\u00ad"), None)
+
+# Typographic punctuation variants normalized to their plain ASCII forms.
+PUNCT_FIXES = {
+    "\u2018": "'", "\u2019": "'",
+    "\u201c": '"', "\u201d": '"',
+    "\u2013": "-", "\u2014": "-",
+}

yotext-0.1.0/src/yotext/standardize.py

@@ -0,0 +1,36 @@
+"""Orthographic standardization utilities for Yorùbá text."""
+
+import unicodedata
+
+from .constants import INVISIBLES, PUNCT_FIXES, UNDERDOT_VARIANTS
+
+
+def standardize(text: str, *, compose: bool = True) -> str:
+    """Normalize Yorùbá text to a canonical orthographic form.
+
+    Invisible characters are dropped, smart quotes/dashes are replaced with
+    their plain ASCII equivalents, and non-canonical combining-below marks
+    are folded onto the canonical DOT_BELOW. The text is renormalized to NFD
+    a second time after that substitution because swapping one combining
+    mark for another can leave the combining-mark sequence out of Unicode
+    canonical order: combining class 220 (below) must sort before combining
+    class 230 (above), and a substituted mark does not automatically end up
+    in the right position relative to its neighbors. Re-running NFD
+    canonically reorders the sequence so the result composes correctly.
+
+    Args:
+        text: The input text.
+        compose: If True (default), return the NFC form. If False, return
+            the NFD form.
+
+    Returns:
+        The standardized text, in NFC or NFD form depending on `compose`.
+    """
+    text = text.translate(INVISIBLES)
+    for old, new in PUNCT_FIXES.items():
+        text = text.replace(old, new)
+    text = unicodedata.normalize("NFD", text)
+    for old, new in UNDERDOT_VARIANTS.items():
+        text = text.replace(old, new)
+    text = unicodedata.normalize("NFD", text)
+    return unicodedata.normalize("NFC", text) if compose else text

yotext-0.1.0/src/yotext/tones.py

@@ -0,0 +1,86 @@
+"""Tone mark and diacritic handling for Yorùbá text.
+
+Underdots are SEGMENTAL: they distinguish separate phonemes (ẹ vs e, ọ vs o)
+and are part of a letter's identity. Tone marks (acute, grave, and the
+implicit mid tone) are SUPRASEGMENTAL: they indicate pitch on top of a
+vowel/syllabic nasal, independent of which phoneme it is. Because of this,
+stripping tone marks and stripping diacritics are genuinely different
+operations: strip_tones removes only pitch information and must leave
+underdots untouched, while strip_diacritics removes every combining mark,
+underdots included, collapsing segmental distinctions along with tone.
+"""
+
+import unicodedata
+
+from .constants import ACUTE, GRAVE, TONE_MARKS, VOWELS
+from .standardize import standardize
+
+
+def strip_tones(text: str) -> str:
+    """Remove suprasegmental tone marks, preserving segmental underdots.
+
+    Only characters in TONE_MARKS are removed; DOT_BELOW is not a tone mark
+    and survives, since it marks a distinct phoneme (e.g. ẹ vs e) rather than
+    pitch.
+
+    Args:
+        text: The input text.
+
+    Returns:
+        The text with tone marks removed, in NFC form.
+    """
+    decomposed = standardize(text, compose=False)
+    filtered = "".join(c for c in decomposed if c not in TONE_MARKS)
+    return unicodedata.normalize("NFC", filtered)
+
+
+def strip_diacritics(text: str) -> str:
+    """Remove every combining mark, including segmental underdots.
+
+    Unlike strip_tones, this removes all combining characters (any character
+    for which unicodedata.combining() is non-zero), which collapses
+    segmental distinctions such as ẹ/e and ọ/o along with tone information.
+
+    Args:
+        text: The input text.
+
+    Returns:
+        The text with all combining marks removed.
+    """
+    decomposed = standardize(text, compose=False)
+    return "".join(c for c in decomposed if not unicodedata.combining(c))
+
+
+def tone_pattern(text: str) -> str:
+    """Derive the tone pattern of a text as a string of H/M/L markers.
+
+    Walks the decomposed text; for each base character, the combining marks
+    that immediately follow it are inspected. Non-vowel bases are skipped.
+    For vowel bases, "H" is emitted if ACUTE is present among its marks, "L"
+    if GRAVE is present, and "M" otherwise (mid tone, unmarked).
+
+    Args:
+        text: The input text.
+
+    Returns:
+        A string of "H", "M", and "L" characters, one per vowel.
+    """
+    decomposed = standardize(text, compose=False)
+    pattern = []
+    i = 0
+    n = len(decomposed)
+    while i < n:
+        base = decomposed[i]
+        i += 1
+        marks = []
+        while i < n and unicodedata.combining(decomposed[i]) != 0:
+            marks.append(decomposed[i])
+            i += 1
+        if base in VOWELS:
+            if ACUTE in marks:
+                pattern.append("H")
+            elif GRAVE in marks:
+                pattern.append("L")
+            else:
+                pattern.append("M")
+    return "".join(pattern)

yotext-0.1.0/tests/test_standardize.py

@@ -0,0 +1,43 @@
+"""Tests for yotext.standardize."""
+
+from yotext.standardize import standardize
+
+
+def test_mark_ordering_reorders_combining_classes():
+    result = standardize("e\u0301\u0329", compose=False)
+    assert result == "e\u0323\u0301"
+
+
+def test_idempotence():
+    for s in ["", "e\u0301\u0329", "plain text", "\u2018hi\u2019", "\u200bzero width"]:
+        once = standardize(s)
+        twice = standardize(once)
+        assert once == twice
+
+
+def test_invisibles_removed():
+    assert standardize("a\u200bb") == "ab"
+    assert standardize("a\u200cb") == "ab"
+    assert standardize("a\u200db") == "ab"
+    assert standardize("a\ufeffb") == "ab"
+    assert standardize("a\u00adb") == "ab"
+
+
+def test_punctuation_normalized():
+    assert standardize("\u2018hi\u2019") == "'hi'"
+    assert standardize("\u201chi\u201d") == '"hi"'
+    assert standardize("a\u2013b") == "a-b"
+    assert standardize("a\u2014b") == "a-b"
+
+
+def test_underdot_variants_fold_to_canonical():
+    for variant in ("\u0329", "\u0331", "\u032D"):
+        result = standardize("e" + variant, compose=False)
+        assert result == "e\u0323"
+
+
+def test_compose_flag():
+    nfd = standardize("e\u0323\u0301", compose=False)
+    nfc = standardize("e\u0323\u0301", compose=True)
+    assert nfd == "e\u0323\u0301"
+    assert nfc == "\u1eb9\u0301"

yotext-0.1.0/tests/test_tones.py

@@ -0,0 +1,57 @@
+"""Tests for yotext.tones."""
+
+from yotext.tones import strip_tones, strip_diacritics, tone_pattern
+
+
+def test_strip_tones_removes_acute():
+    result = strip_tones("e\u0301")
+    assert result == "e"
+
+
+def test_strip_tones_removes_grave():
+    result = strip_tones("e\u0300")
+    assert result == "e"
+
+
+def test_strip_tones_removes_macron():
+    result = strip_tones("e\u0304")
+    assert result == "e"
+
+
+def test_strip_tones_preserves_underdot():
+    result = strip_tones("\u1eb9\u0301")
+    assert result == "\u1eb9"
+
+
+def test_strip_diacritics_removes_everything_combining():
+    result = strip_diacritics("\u1eb9\u0301")
+    assert result == "e"
+
+
+def test_consistency_invariant():
+    for s in [
+        "\u1eb9\u0300k\u1ecd\u0301",
+        "b\u00e0b\u00e1",
+        "plain",
+        "",
+        "\u1ecd\u0300w\u1ecd\u0300",
+        "e\u0301\u0329",
+        "\u1e63\u00e9",
+    ]:
+        assert strip_diacritics(s) == strip_diacritics(strip_tones(s))
+
+
+def test_tone_pattern_known_words():
+    assert tone_pattern("\u1eb9\u0300k\u1ecd\u0301") == "LH"
+    assert tone_pattern("b\u00e0b\u00e1") == "LH"
+    assert tone_pattern("k\u00e1\u00e0r\u1ecd\u0300") == "HLL"
+
+
+def test_tone_pattern_unmarked_vowels_yield_mid():
+    assert tone_pattern("aeiou") == "MMMMM"
+
+
+def test_empty_string():
+    assert strip_tones("") == ""
+    assert strip_diacritics("") == ""
+    assert tone_pattern("") == ""