euskaphone 0.1.0a2__py3-none-any.whl

euskaphone/__init__.py

@@ -0,0 +1,122 @@
+"""euskaphone — dialect-aware Basque phonemization on the orthography2ipa lattice.
+
+The phonemizer drives the shared orthography2ipa candidate lattice: a dialect is
+an orthography2ipa Basque lect spec, and the lattice — the spec's grapheme
+table, ``allophones`` and cross-word sandhi — produces the dialect's phonology
+directly. euskaphone contributes the stages orthography2ipa leaves to the
+caller: vigesimal (base-20) number verbalization, Spanish/French code-switch
+handling with nativization onto the Basque inventory, and a proper-name/loan
+lexicon hook. Basque orthography is near-phonemic, so — by design — there is no
+homograph subsystem. See :mod:`euskaphone.lattice_core`.
+"""
+from typing import Optional
+
+from euskaphone.version import __version__
+from euskaphone.lattice_core import phonemize as _phonemize, register_lexicon
+from euskaphone.lexicons import (
+    apply_builtins, register_hitz_overlay, register_toponyms,
+)
+from euskaphone.pitch_accent import (
+    AccentClass, PitchAccentLexicon, annotate_sentence as _annotate_pitch,
+    default_lexicon,
+)
+from euskaphone.registry import (
+    DEFAULT_DIALECT, dialect_aliases, is_supported as _is_supported,
+    list_dialects, resolve_lect,
+)
+
+#: The lects whose lexical pitch accent euskaphone can annotate. The shipped
+#: cited lexicon documents the Northern Bizkaian sub-area of Biscayan.
+_PITCH_ACCENT_LECTS = ("eu-x-bizkaiera",)
+
+
+class EuskaPhonemizer:
+    """Dialect-aware Basque phonemization on the orthography2ipa lattice.
+
+    The eight Basque lects orthography2ipa ships are reachable by their BCP-47
+    codes ("eu", "eu-x-bizkaiera", "eu-x-zuberera", …) or by a human-readable
+    alias ("batua", "biscayan", "souletin"); the full list comes from
+    :func:`euskaphone.list_dialects`.
+
+    Built-in lexicons (:mod:`euskaphone.lexicons`) are wired at construction:
+    the Euskaltzaindia toponym seed is registered by default (``toponyms=True``,
+    opt-out) unless a caller already registered a lexicon for ``eu``; the HiTZ
+    proper-noun overlay is opt-in (``lexicon="hitz"``) and carries a benchmark
+    circularity caveat.
+    """
+
+    def __init__(self, toponyms: bool = True,
+                 lexicon: Optional[str] = None) -> None:
+        apply_builtins(toponyms=toponyms, lexicon=lexicon)
+
+    def phonemize_sentence(self, sentence: str, dialect: str = DEFAULT_DIALECT,
+                           contact: str = "auto",
+                           pitch_accent: bool = False) -> str:
+        """Phonemize ``sentence`` for the ``dialect`` lect.
+
+        Parameters:
+            sentence: Input text to phonemize.
+            dialect: A lect code or human-readable alias (see
+                :func:`euskaphone.list_dialects` /
+                :func:`euskaphone.dialect_aliases`). Defaults to Standard Batua.
+            contact: Embedded-language policy — ``"auto"`` (detect and classify
+                each contact word per-word among es/fr/en, unclassified words
+                falling to the dialect's side), ``"es"``, ``"fr"``, ``"en"`` or
+                ``"none"`` (no code-switching).
+            pitch_accent: When ``True``, annotate lexical Northern Bizkaian pitch
+                accent (:mod:`euskaphone.pitch_accent`) — a mark on the accented
+                syllable of each lexically accented word, nothing on unaccented
+                or unknown words. Opt-in and only defined for Biscayan
+                (``eu-x-bizkaiera`` / ``biscayan``); a ``ValueError`` is raised
+                for any other lect. Annotation is word-level, so cross-word
+                sandhi is not applied in this mode.
+
+        Returns:
+            Space-separated IPA for each word.
+        """
+        if pitch_accent:
+            lect = resolve_lect(dialect)
+            if lect not in _PITCH_ACCENT_LECTS:
+                raise ValueError(
+                    f"pitch_accent is only defined for Biscayan "
+                    f"({', '.join(_PITCH_ACCENT_LECTS)}); got {dialect!r} "
+                    f"({lect}). Coverage is scoped to the Northern Bizkaian "
+                    f"sub-area Hualde documents.")
+            return _annotate_pitch(
+                sentence,
+                lambda tok: _phonemize(tok, dialect, contact))
+        return _phonemize(sentence, dialect, contact)
+
+    @staticmethod
+    def is_supported(dialect: str) -> bool:
+        """Whether ``dialect`` names a known Basque lect."""
+        return _is_supported(dialect)
+
+
+__all__ = [
+    "EuskaPhonemizer",
+    "register_lexicon",
+    "register_toponyms",
+    "register_hitz_overlay",
+    "list_dialects",
+    "dialect_aliases",
+    "resolve_lect",
+    "AccentClass",
+    "PitchAccentLexicon",
+    "default_lexicon",
+    "__version__",
+]
+
+
+if __name__ == "__main__":
+    ph = EuskaPhonemizer()
+    sentences = [
+        "Gaur 1936ko gerra ikasi dugu.",
+        "Hogeita hamaika katu zuri ikusi ditut.",
+        "Bilbon 20:00etan hasiko da kontzertua.",
+    ]
+    for s in sentences:
+        print(s)
+        for code in ["eu", "eu-x-bizkaiera", "eu-x-zuberera"]:
+            print(f"{code} → {ph.phonemize_sentence(s, code)}")
+        print("######")

euskaphone/abbreviations.py

@@ -0,0 +1,87 @@
+"""Abbreviation (laburdura) expansion — the pre-lattice normalizer stage.
+
+Written Basque is full of period-marked abbreviations that a grapheme-to-phoneme
+lattice would mis-read letter by letter (``etab.`` is not "e-t-a-b", it is
+``eta abar``). This stage expands a curated, **cited** list of abbreviations to
+their full spoken form *before* the lattice — and before the Roman-numeral stage
+— so the trailing periods that mark an abbreviation are consumed here and never
+confused with the Roman-numeral ordinal period (``XX.``, see
+:mod:`euskaphone.romans`).
+
+The list itself is data, not code: it lives in ``data/abbreviations.tsv`` and is
+extended by adding a cited row, never by editing this module.
+
+Sources
+-------
+* **Euskaltzaindia**, **Araua 196**, EBO (I) *"Laburtzapenak: laburdurak eta
+  siglak"* — the Academy's norm fixing the standard abbreviation forms and the
+  rule that a laburdura is written with a period and read as the full word.
+* **EIMA / Basque Government**, J. R. Zubimendi, *Ortotipografia* (Eusko
+  Jaurlaritza, 2004), "Laburdurak" — spells the same list plus the ``K.a.`` /
+  ``K.o.`` era abbreviations.
+  (``papers/iberian/eima_ortotipografia_zubimendi.pdf``)
+* **Euskalterm**, *Laburtzapenen Hiztegia* (Gasteiz, 2010) — the exhaustive
+  reference dictionary of Basque abbreviations.
+  (``papers/iberian/euskalterm_laburtzapenen_hiztegia_2010.pdf``)
+"""
+from __future__ import annotations
+
+import os
+from typing import Dict
+
+_DATA = os.path.join(os.path.dirname(__file__), "data", "abbreviations.tsv")
+
+
+def _load(path: str = _DATA) -> Dict[str, str]:
+    """Load the abbreviation → expansion table (case-folded keys)."""
+    table: Dict[str, str] = {}
+    with open(path, encoding="utf-8") as fh:
+        for line in fh:
+            line = line.rstrip("\n")
+            if not line or line.startswith("#"):
+                continue
+            parts = line.split("\t")
+            if len(parts) < 2:
+                continue
+            abbr, expansion = parts[0], parts[1]
+            if abbr and expansion:
+                table[abbr.lower()] = expansion
+    return table
+
+
+#: abbreviation (as written, with its period) → full spoken expansion.
+ABBREVIATIONS: Dict[str, str] = _load()
+
+#: The bare (period-stripped) keys, for matching a token whose trailing period
+#: the tokenizer may have split off (``etab`` as well as ``etab.``).
+_BARE = {k.rstrip("."): v for k, v in ABBREVIATIONS.items()}
+
+
+def expand_token(token: str) -> str:
+    """Expand one *token* if it is a known abbreviation, else return it as-is.
+
+    Both the written form with its period (``etab.``) and the bare form
+    (``etab``) resolve; multi-dot abbreviations (``K.a.``, ``h.d.``) are matched
+    whole. Matching is case-insensitive but the lookup is exact on the letters.
+    """
+    key = token.lower()
+    if key in ABBREVIATIONS:
+        return ABBREVIATIONS[key]
+    bare = key.rstrip(".")
+    if bare in _BARE:
+        return _BARE[bare]
+    return token
+
+
+def expand_abbreviations(text: str) -> str:
+    """Expand every known abbreviation token in *text*.
+
+    Runs word by word so surrounding punctuation and spacing are preserved for
+    tokens that are not abbreviations. Multi-word expansions are inserted inline
+    (``etab.`` → ``eta abar``) and flow on to the pronunciation lattice as
+    ordinary Basque words.
+    """
+    out = []
+    for word in text.split():
+        out.append(expand_token(word))
+    return " ".join(out)

euskaphone/accent.py

@@ -0,0 +1,181 @@
+"""Accent forcing: re-transcribe or *respell* Batua text toward a target lect.
+
+Two modes, one entry point :func:`force_accent`:
+
+* ``mode="ipa"`` — the direct view. The text is transcribed straight through the
+  target lect's lattice, so the return is the target dialect's own IPA (the
+  "IPA delta" against Batua).
+
+* ``mode="respell"`` — the verification-gated respeller, ported from tugaphone's
+  accent-forcing architecture. It rewrites the *Batua orthography* using Basque
+  spelling conventions that push a **Batua** reader toward the target
+  pronunciation (``z``→``s`` for the peninsular seseo merger, ``il``→``ill`` for
+  palatalisation, ``h``-insertion for continental aspiration, …). Every edit is
+  **verification-gated**: an edit is kept only if re-transcribing the respelled
+  text *through the base (Batua) lattice* moves the result measurably closer to
+  the target lect's IPA. An edit the base lattice cannot cash out — Batua has a
+  silent ``h`` and no ``ü``, so continental aspiration and Souletin front
+  rounding are simply not recoverable through it — is rejected, and the honest
+  consequence is a near-zero respell gain for exactly those features.
+
+The gate is what keeps respell honest: it never invents an orthography whose
+Batua reading does not actually approximate the target. :func:`respell_report`
+exposes the per-edit accounting so a benchmark can publish the real ceiling.
+"""
+from __future__ import annotations
+
+import re
+from typing import Callable, List, NamedTuple, Optional
+
+from euskaphone.lattice_core import engine
+from euskaphone.registry import resolve_lect
+
+#: The base reading lect: respelled orthography is scored by *this* lattice.
+BASE_LECT = "eu"
+
+
+def _transcribe(text: str, lect: str) -> str:
+    """Pure-lattice transcription (no code-switch, no number stage)."""
+    return engine(lect).transcribe(text)
+
+
+def _levenshtein(a: str, b: str) -> int:
+    d = list(range(len(b) + 1))
+    for i in range(1, len(a) + 1):
+        prev, d[0] = d[0], i
+        for j in range(1, len(b) + 1):
+            cur = d[j]
+            d[j] = min(d[j] + 1, d[j - 1] + 1, prev + (a[i - 1] != b[j - 1]))
+            prev = cur
+    return d[len(b)]
+
+
+def _per(ref: str, hyp: str) -> float:
+    r, h = ref.replace(" ", ""), hyp.replace(" ", "")
+    return _levenshtein(r, h) / max(1, len(r))
+
+
+class RespellRule(NamedTuple):
+    """One candidate orthographic edit, tried against the verification gate."""
+
+    name: str
+    #: What phonological delta the edit chases (documentation only).
+    rationale: str
+    #: Rewrites Batua orthography toward the target spelling convention.
+    apply: Callable[[str], str]
+
+
+def _sub(pattern: str, repl: str) -> Callable[[str], str]:
+    rx = re.compile(pattern)
+    return lambda s: rx.sub(repl, s)
+
+
+#: The respell rule set. Each rule is a *proposal*; the verification gate decides
+#: per text and per target whether it survives. Rules whose delta the Batua
+#: lattice cannot represent (``h``-insertion, front rounding) are proposed too —
+#: they are simply gated out, which is how the honest ceiling is measured.
+RESPELL_RULES: List[RespellRule] = [
+    RespellRule(
+        "seseo_z_to_s",
+        "peninsular seseo: laminal /s̻/ (⟨z⟩) merges into apical /s̺/ (⟨s⟩)",
+        _sub(r"z", "s"),
+    ),
+    RespellRule(
+        "palatal_il",
+        "expressive/contextual palatalisation ⟨il⟩ → /ʎ/ (spelled ⟨ill⟩)",
+        _sub(r"il", "ill"),
+    ),
+    RespellRule(
+        "palatal_in",
+        "expressive/contextual palatalisation ⟨in⟩ → /ɲ/ (spelled ⟨iñ⟩)",
+        _sub(r"in", "iñ"),
+    ),
+    RespellRule(
+        "aspiration_h",
+        "continental aspiration: prothetic ⟨h⟩ on vowel-initial words",
+        _sub(r"(?<![^\W\d_])(?=[aeiouAEIOU])", "h"),
+    ),
+    RespellRule(
+        "front_rounding_u_ue",
+        "Souletin front rounding: ⟨u⟩ → ⟨ü⟩ [y]",
+        _sub(r"u", "ü"),
+    ),
+]
+
+
+class RespellResult(NamedTuple):
+    """The outcome of a respell pass over one text against one target."""
+
+    source: str
+    respelled: str
+    target_ipa: str
+    base_ipa: str
+    respelled_ipa: str
+    #: PER of the base reading before respelling; the starting distance.
+    per_before: float
+    #: PER after keeping every surviving edit; ``per_before - per_after`` is gain.
+    per_after: float
+    #: Names of the rules the verification gate accepted, in order.
+    accepted: List[str]
+
+    @property
+    def gain(self) -> float:
+        """PER reduction the respelling bought (>= 0)."""
+        return self.per_before - self.per_after
+
+
+def respell_report(text: str, dialect: str,
+                   rules: Optional[List[RespellRule]] = None,
+                   eps: float = 1e-9) -> RespellResult:
+    """Greedy, verification-gated respell of ``text`` toward ``dialect``.
+
+    For each rule in turn, the candidate respelling is scored by transcribing it
+    through the **base** (Batua) lattice and measuring PER against the target
+    lect's own transcription of the original text. The edit is kept only if it
+    strictly lowers that PER; otherwise it is discarded. The result carries the
+    full accounting for benchmarking.
+    """
+    rules = RESPELL_RULES if rules is None else rules
+    target = resolve_lect(dialect)
+    target_ipa = _transcribe(text, target)
+    base_ipa = _transcribe(text, BASE_LECT)
+
+    current = text
+    current_per = _per(target_ipa, base_ipa)
+    per_before = current_per
+    accepted: List[str] = []
+
+    for rule in rules:
+        candidate = rule.apply(current)
+        if candidate == current:
+            continue
+        cand_ipa = _transcribe(candidate, BASE_LECT)
+        cand_per = _per(target_ipa, cand_ipa)
+        if cand_per < current_per - eps:
+            current, current_per = candidate, cand_per
+            accepted.append(rule.name)
+
+    return RespellResult(
+        source=text,
+        respelled=current,
+        target_ipa=target_ipa,
+        base_ipa=base_ipa,
+        respelled_ipa=_transcribe(current, BASE_LECT),
+        per_before=per_before,
+        per_after=current_per,
+        accepted=accepted,
+    )
+
+
+def force_accent(text: str, dialect: str, mode: str = "ipa") -> str:
+    """Force ``text`` toward ``dialect``.
+
+    ``mode="ipa"`` returns the target lect's IPA directly. ``mode="respell"``
+    returns Batua orthography rewritten (verification-gated) so a Batua reader
+    approximates the target pronunciation.
+    """
+    if mode == "ipa":
+        return _transcribe(text, resolve_lect(dialect))
+    if mode == "respell":
+        return respell_report(text, dialect).respelled
+    raise ValueError(f"unknown mode {mode!r}; expected 'ipa' or 'respell'")