lexphon 0.1.0__py3-none-any.whl

lexphon/__init__.py

@@ -0,0 +1,32 @@
+"""Lexicon-driven phonemization on top of G2Lex."""
+
+from __future__ import annotations
+
+from ._version import __version__
+from .engine import Phonemizer
+from .errors import (
+    CatalogError,
+    DataIntegrityError,
+    LexiconNotInstalledError,
+    LexiconNotUsableError,
+    LexphonError,
+    UnknownWordError,
+    UnsupportedAlphabetError,
+)
+from .models import PhonemizationResult, PronunciationToken
+from .store import DataStore
+
+__all__ = [
+    "CatalogError",
+    "DataIntegrityError",
+    "DataStore",
+    "LexiconNotInstalledError",
+    "LexiconNotUsableError",
+    "LexphonError",
+    "PhonemizationResult",
+    "Phonemizer",
+    "PronunciationToken",
+    "UnknownWordError",
+    "UnsupportedAlphabetError",
+    "__version__",
+]

lexphon/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

lexphon/_version.py

@@ -0,0 +1,72 @@
+"""Small dependency-free Git-derived version helper used by setuptools."""
+
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+from email.parser import Parser
+from pathlib import Path
+
+_FALLBACK = "0.1.0"
+_TAG = re.compile(
+    r"^v?(?P<base>\d+\.\d+\.\d+)(?:-(?P<count>\d+)-g(?P<sha>[0-9a-f]+))?(?P<dirty>-dirty)?$"
+)
+
+
+def _sdist_version(root: Path) -> str | None:
+    try:
+        metadata = Parser().parsestr(
+            (root / "PKG-INFO").read_text(encoding="utf-8"), headersonly=True
+        )
+    except (OSError, UnicodeError):
+        return None
+    if metadata.get("Name") != "lexphon":
+        return None
+    value = metadata.get("Version")
+    return value.strip() if value and value.strip() else None
+
+
+def get_version() -> str:
+    override = os.environ.get("LEXPHON_VERSION")
+    if override:
+        return override
+    root = Path(__file__).resolve().parents[1]
+    try:
+        text = subprocess.check_output(
+            [
+                "git",
+                "-C",
+                str(root),
+                "describe",
+                "--tags",
+                "--long",
+                "--dirty",
+                "--match",
+                "v[0-9]*",
+            ],
+            stderr=subprocess.DEVNULL,
+            text=True,
+            timeout=2,
+        ).strip()
+    except (OSError, subprocess.SubprocessError):
+        text = ""
+    match = _TAG.match(text)
+    if not match:
+        return _sdist_version(root) or _FALLBACK
+    base = match.group("base")
+    count = int(match.group("count") or 0)
+    sha = match.group("sha")
+    dirty = bool(match.group("dirty"))
+    if count == 0 and not dirty:
+        return base
+    local = []
+    if sha:
+        local.append(f"g{sha}")
+    if dirty:
+        local.append("dirty")
+    version = f"{base}.post{count}" if count else f"{base}.dev0"
+    return version + ("+" + ".".join(local) if local else "")
+
+
+__version__ = get_version()

lexphon/alphabets.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import re
+import unicodedata
+
+from .errors import UnsupportedAlphabetError
+
+_ARPA_CONSONANTS = {
+    "B": "b",
+    "CH": "tʃ",
+    "D": "d",
+    "DH": "ð",
+    "F": "f",
+    "G": "ɡ",
+    "HH": "h",
+    "JH": "dʒ",
+    "K": "k",
+    "L": "l",
+    "M": "m",
+    "N": "n",
+    "NG": "ŋ",
+    "P": "p",
+    "R": "ɹ",
+    "S": "s",
+    "SH": "ʃ",
+    "T": "t",
+    "TH": "θ",
+    "V": "v",
+    "W": "w",
+    "Y": "j",
+    "Z": "z",
+    "ZH": "ʒ",
+}
+_ARPA_VOWELS = {
+    "AA": "ɑ",
+    "AE": "æ",
+    "AH": "ʌ",
+    "AO": "ɔ",
+    "AW": "aʊ",
+    "AY": "aɪ",
+    "EH": "ɛ",
+    "ER": "ɝ",
+    "EY": "eɪ",
+    "IH": "ɪ",
+    "IY": "i",
+    "OW": "oʊ",
+    "OY": "ɔɪ",
+    "UH": "ʊ",
+    "UW": "u",
+    "AX": "ə",
+    "AXR": "ɚ",
+    "IX": "ɨ",
+}
+_ARPA_TOKEN = re.compile(r"^(?P<phoneme>[A-Z]+)(?P<stress>[012])?$")
+
+
+def arpabet_to_ipa(value: str) -> str:
+    """Convert CMU-style ARPABET to deterministic broad IPA."""
+    if not isinstance(value, str) or not value.strip():
+        raise UnsupportedAlphabetError("ARPABET pronunciation must contain phone tokens")
+    segments: list[str] = []
+    vowel_indices: list[int] = []
+    stressed: list[tuple[int, str]] = []
+    for raw in value.split():
+        match = _ARPA_TOKEN.fullmatch(raw.upper())
+        if not match:
+            raise UnsupportedAlphabetError(f"invalid ARPABET token: {raw!r}")
+        symbol = match.group("phoneme")
+        stress = match.group("stress")
+        if symbol in _ARPA_VOWELS:
+            ipa = _ARPA_VOWELS[symbol]
+            if symbol == "AH" and stress == "0":
+                ipa = "ə"
+            elif symbol == "ER" and stress == "0":
+                ipa = "ɚ"
+            vowel_ordinal = len(vowel_indices)
+            if stress in {"1", "2"}:
+                stressed.append((vowel_ordinal, "ˈ" if stress == "1" else "ˌ"))
+            vowel_indices.append(len(segments))
+            segments.append(ipa)
+        elif symbol in _ARPA_CONSONANTS:
+            if stress is not None:
+                raise UnsupportedAlphabetError(f"stress on ARPABET consonant: {raw!r}")
+            segments.append(_ARPA_CONSONANTS[symbol])
+        else:
+            raise UnsupportedAlphabetError(f"unsupported ARPABET phoneme: {symbol}")
+
+    insertions: dict[int, list[str]] = {}
+    for vowel_ordinal, marker in stressed:
+        position = 0 if vowel_ordinal == 0 else vowel_indices[vowel_ordinal - 1] + 1
+        insertions.setdefault(position, []).append(marker)
+
+    output: list[str] = []
+    for index, segment in enumerate(segments):
+        output.extend(insertions.get(index, ()))
+        output.append(segment)
+    output.extend(insertions.get(len(segments), ()))
+    return unicodedata.normalize("NFC", "".join(output))
+
+
+def to_ipa(value: str, encoding: str) -> str:
+    if not isinstance(value, str) or not value:
+        raise UnsupportedAlphabetError("pronunciation must be a non-empty string")
+    key = encoding.casefold().replace("-", "")
+    if key in {"ipa", "unicodeipa"}:
+        return unicodedata.normalize("NFC", value)
+    if key in {"arpabet", "cmu", "cmudict"}:
+        return arpabet_to_ipa(value)
+    raise UnsupportedAlphabetError(f"unsupported pronunciation encoding: {encoding!r}")

lexphon/catalog.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+from .errors import CatalogError
+
+DEFAULT_CATALOG_URL = (
+    "https://raw.githubusercontent.com/buchwandler/g2lex-data/main/catalog/catalog.json"
+)
+_HASH_RE = re.compile(r"^[0-9a-f]{64}$")
+_ID_RE = re.compile(r"^[a-z]{2,3}(?:-[a-z0-9]{2,8})*:[a-z0-9][a-z0-9._-]*$")
+_LOCALE_RE = re.compile(r"^[a-z]{2,3}(?:-[a-z0-9]{2,8})*$")
+_SUPPORTED_KINDS = {"pronunciation", "membership"}
+_SUPPORTED_ENCODINGS = {"ipa", "arpabet", "none"}
+
+
+def _require_text(value: dict[str, Any], key: str, label: str) -> str:
+    result = value.get(key)
+    if not isinstance(result, str) or not result:
+        raise CatalogError(f"{label} field {key!r} must be a non-empty string")
+    return result
+
+
+def _require_hash(value: object, label: str) -> str:
+    if not isinstance(value, str) or not _HASH_RE.fullmatch(value):
+        raise CatalogError(f"{label} must be a lowercase SHA-256 hash")
+    return value
+
+
+def _require_size(value: object, label: str) -> int:
+    if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
+        raise CatalogError(f"{label} must be a positive integer")
+    return value
+
+
+def _validate_reference(value: object, label: str, *, require_size: bool) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        raise CatalogError(f"artifact {label} must be an object")
+    url = _require_text(value, "url", f"artifact {label}")
+    parsed = urlparse(url)
+    if parsed.scheme not in {"http", "https", "file"} and not parsed.scheme == "":
+        raise CatalogError(f"artifact {label} has unsupported URL scheme: {parsed.scheme!r}")
+    _require_text(value, "name", f"artifact {label}")
+    if Path(value["name"]).name != value["name"] or value["name"] in {".", ".."}:
+        raise CatalogError(f"artifact {label} name must be a plain filename")
+    _require_hash(value.get("sha256"), f"artifact {label}.sha256")
+    if require_size and "size" not in value:
+        raise CatalogError(f"artifact {label} requires size")
+    if "size" in value:
+        _require_size(value["size"], f"artifact {label}.size")
+    return value
+
+
+@dataclass(frozen=True, slots=True)
+class CatalogArtifact:
+    id: str
+    language: str
+    name: str
+    display_name: str
+    kind: str
+    phoneme_encoding: str
+    data_version: str
+    release_tag: str
+    manifest: dict[str, Any]
+    asset: dict[str, Any]
+    source: dict[str, Any]
+
+    @classmethod
+    def from_dict(cls, value: dict[str, Any]) -> CatalogArtifact:
+        if not isinstance(value, dict):
+            raise CatalogError("catalog artifacts must be objects")
+        identifier = _require_text(value, "id", "artifact")
+        if not _ID_RE.fullmatch(identifier):
+            raise CatalogError(f"artifact id is not a valid logical ID: {identifier!r}")
+        language = _require_text(value, "language", f"artifact {identifier}")
+        normalized_language = language.casefold().replace("_", "-")
+        id_language = identifier.split(":", 1)[0]
+        if not _LOCALE_RE.fullmatch(normalized_language) or normalized_language != id_language:
+            raise CatalogError(f"artifact {identifier} has inconsistent language metadata")
+        name = _require_text(value, "name", f"artifact {identifier}")
+        if Path(name).name != name:
+            raise CatalogError(f"artifact {identifier} name must be a plain name")
+        kind = _require_text(value, "kind", f"artifact {identifier}")
+        if kind not in _SUPPORTED_KINDS:
+            raise CatalogError(f"artifact {identifier} has unsupported kind: {kind!r}")
+        encoding = _require_text(value, "phoneme_encoding", f"artifact {identifier}").casefold()
+        if encoding not in _SUPPORTED_ENCODINGS:
+            raise CatalogError(
+                f"artifact {identifier} has unsupported phoneme_encoding: {encoding!r}"
+            )
+        if kind == "membership" and encoding != "none":
+            raise CatalogError(f"membership artifact {identifier} must use phoneme_encoding 'none'")
+        if kind == "pronunciation" and encoding == "none":
+            raise CatalogError(
+                f"pronunciation artifact {identifier} cannot use phoneme_encoding 'none'"
+            )
+        data_version = _require_text(value, "data_version", f"artifact {identifier}")
+        release_tag = _require_text(value, "release_tag", f"artifact {identifier}")
+        manifest = _validate_reference(value.get("manifest"), "manifest", require_size=False)
+        asset = _validate_reference(value.get("asset"), "asset", require_size=True)
+        if asset.get("format") not in {None, "g2lex.lexicon.v1", "g2lex"}:
+            raise CatalogError(f"artifact {identifier} has unsupported asset format")
+        source = value.get("source", {})
+        if not isinstance(source, dict):
+            raise CatalogError(f"artifact {identifier} source must be an object")
+        return cls(
+            id=identifier,
+            language=language,
+            name=name,
+            display_name=str(value.get("display_name") or name),
+            kind=kind,
+            phoneme_encoding=encoding,
+            data_version=data_version,
+            release_tag=release_tag,
+            manifest=manifest,
+            asset=asset,
+            source=source,
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class Catalog:
+    catalog_version: int
+    runtime_contract: str
+    artifacts: tuple[CatalogArtifact, ...]
+
+    @classmethod
+    def from_dict(cls, value: dict[str, Any]) -> Catalog:
+        if not isinstance(value, dict):
+            raise CatalogError("catalog root must be an object")
+        if value.get("catalog_version") != 1:
+            raise CatalogError(f"unsupported catalog_version: {value.get('catalog_version')!r}")
+        contract = value.get("runtime_contract")
+        if contract != "g2lex-data.catalog.v1":
+            raise CatalogError(f"unsupported runtime_contract: {contract!r}")
+        raw = value.get("artifacts")
+        if not isinstance(raw, list):
+            raise CatalogError("catalog artifacts must be a list")
+        artifacts = tuple(CatalogArtifact.from_dict(item) for item in raw)
+        ids = [item.id for item in artifacts]
+        if len(set(ids)) != len(ids):
+            raise CatalogError("catalog artifact ids must be unique")
+        return cls(1, contract, artifacts)
+
+    def artifact(self, identifier: str) -> CatalogArtifact:
+        for artifact in self.artifacts:
+            if artifact.id == identifier:
+                return artifact
+        raise CatalogError(f"unknown catalog artifact: {identifier}")
+
+    def for_language(self, language: str) -> tuple[CatalogArtifact, ...]:
+        key = language.casefold().replace("_", "-")
+        return tuple(
+            item for item in self.artifacts if item.language.casefold().replace("_", "-") == key
+        )
+
+
+def _read_location(location: str) -> bytes:
+    parsed = urlparse(location)
+    if parsed.scheme in {"http", "https", "file"}:
+        with urllib.request.urlopen(location, timeout=30) as response:
+            return response.read()
+    return Path(location).expanduser().read_bytes()
+
+
+def load_catalog(location: str | None = None) -> Catalog:
+    source = location or os.environ.get("LEXPHON_CATALOG_URL") or DEFAULT_CATALOG_URL
+    try:
+        value = json.loads(_read_location(source).decode("utf-8"))
+        return Catalog.from_dict(value)
+    except CatalogError:
+        raise
+    except Exception as exc:  # keep a stable public error at the catalog boundary
+        raise CatalogError(f"unable to load catalog from {source}: {exc}") from exc

lexphon/cli.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from . import __version__
+from .catalog import load_catalog
+from .engine import Phonemizer
+from .errors import LexphonError
+from .profiles import ProfileRegistry
+from .store import DataStore
+
+
+def _data_main(argv: list[str]) -> int:
+    parser = argparse.ArgumentParser(
+        prog="lexphon data", description="Manage installed G2Lex pronunciation data"
+    )
+    parser.add_argument("--catalog", help="catalog URL/path (default: g2lex-data main catalog)")
+    parser.add_argument("--data-home")
+    sub = parser.add_subparsers(dest="command", required=True)
+    p_install = sub.add_parser("install")
+    p_install.add_argument("id", nargs="+")
+    sub.add_parser("list")
+    p_verify = sub.add_parser("verify")
+    p_verify.add_argument("id", nargs="*")
+    p_remove = sub.add_parser("remove")
+    p_remove.add_argument("id", nargs="+")
+    p_info = sub.add_parser("info")
+    p_info.add_argument("id", nargs="+")
+    p_available = sub.add_parser("available")
+    p_available.add_argument("language", nargs="?")
+    args = parser.parse_args(argv)
+    store = DataStore(args.data_home)
+
+    if args.command == "install":
+        catalog = load_catalog(args.catalog)
+        for identifier in args.id:
+            path = store.install(catalog.artifact(identifier))
+            print(f"installed {identifier}: {path}")
+    elif args.command == "list":
+        for item in store.installed():
+            print(f"{item['id']}\t{item['data_version']}\t{item['phoneme_encoding']}")
+    elif args.command == "info":
+        for identifier in args.id:
+            print(json.dumps(store.metadata(identifier), ensure_ascii=False, sort_keys=True))
+    elif args.command == "verify":
+        ids = args.id or [item["id"] for item in store.installed()]
+        failed = False
+        for identifier in ids:
+            ok = store.verify(identifier)
+            print(f"{'OK' if ok else 'FAIL'}\t{identifier}")
+            failed |= not ok
+        return int(failed)
+    elif args.command == "remove":
+        for identifier in args.id:
+            store.remove(identifier)
+            print(f"removed {identifier}")
+    elif args.command == "available":
+        catalog = load_catalog(args.catalog)
+        artifacts = catalog.for_language(args.language) if args.language else catalog.artifacts
+        for item in artifacts:
+            print(f"{item.id}\t{item.language}\t{item.phoneme_encoding}\t{item.data_version}")
+    return 0
+
+
+def _voices_main() -> int:
+    for profile in ProfileRegistry().profiles:
+        print(profile.language)
+    return 0
+
+
+def _phonemize_main(argv: list[str]) -> int:
+    parser = argparse.ArgumentParser(prog="lexphon", description="Lexicon-driven IPA phonemizer")
+    parser.add_argument("--version", action="version", version=__version__)
+    parser.add_argument("-v", "--voice", "--language", dest="language", required=True)
+    parser.add_argument(
+        "--lexicon", action="append", dest="lexicons", help="installed lexicon id; repeatable"
+    )
+    parser.add_argument("--tag", help="selector tag for tagged G2Lex values")
+    parser.add_argument("--fallback", choices=["none", "espeak"], default="none")
+    parser.add_argument("--unknown", choices=["error", "keep", "skip"], default="error")
+    parser.add_argument("--punctuation", choices=["keep", "drop"], default="keep")
+    parser.add_argument("--json", action="store_true")
+    parser.add_argument("--data-home")
+    parser.add_argument("text", nargs="*")
+    args = parser.parse_args(argv)
+    text = " ".join(args.text) if args.text else sys.stdin.read().strip()
+    if not text:
+        parser.error("text argument or stdin is required")
+    with Phonemizer(
+        args.language,
+        lexicons=args.lexicons,
+        store=DataStore(args.data_home),
+        fallback=None if args.fallback == "none" else args.fallback,
+    ) as engine:
+        result = engine.phonemize_tokens(text, tag=args.tag)
+        if args.json:
+            print(
+                json.dumps(
+                    {
+                        "text": result.text,
+                        "language": result.language,
+                        "phonemes": result.render(
+                            unknown=args.unknown, punctuation=args.punctuation
+                        ),
+                        "tokens": [
+                            {
+                                "text": token.text,
+                                "original_token": token.text,
+                                "pronunciation": token.pronunciation,
+                                "source": token.source,
+                                "alphabet": token.alphabet,
+                                "known": token.known,
+                                "lexicon_id": token.lexicon_id,
+                                "matched_key": token.matched_key,
+                                "source_encoding": token.source_encoding,
+                                "variants": list(token.variants),
+                                "selector_tag": token.selector_tag,
+                                "punctuation": token.punctuation,
+                            }
+                            for token in result.tokens
+                        ],
+                    },
+                    ensure_ascii=False,
+                )
+            )
+        else:
+            print(result.render(unknown=args.unknown, punctuation=args.punctuation))
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = list(sys.argv[1:] if argv is None else argv)
+    try:
+        if args and args[0] == "data":
+            return _data_main(args[1:])
+        if args and args[0] == "voices":
+            return _voices_main()
+        return _phonemize_main(args)
+    except LexphonError as exc:
+        print(f"lexphon: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())