phonebox 0.1.0__py3-none-any.whl

phonebox/__init__.py

@@ -0,0 +1,41 @@
+"""
+Phonebox: grapheme-to-phoneme conversion package.
+
+A fast, lightweight grapheme-to-phoneme conversion system using
+decision trees, EM alignment, and optional joint multigram (n:m) models.
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+__package_name__ = "phonebox"
+
+# High-level API (recommended for most users)
+from .converter import G2P
+
+# Core classes (for advanced use)
+from .core.decision_tree import DecisionTree
+from .core.em_align import EMAlign
+from .core.multigram_g2p import MultigramG2P
+from .core.vectorizer import Vectorizer
+from .dictionary import Dictionary
+from .locales import (
+    canonical_locale,
+    load_locale_defaults,
+    supplement_defaults,
+)
+
+__all__ = [
+    # High-level API
+    "G2P",
+    "Dictionary",
+    # Core API
+    "DecisionTree",
+    "EMAlign",
+    "MultigramG2P",
+    "Vectorizer",
+    # Locale-aware defaults
+    "canonical_locale",
+    "load_locale_defaults",
+    "supplement_defaults",
+]

phonebox/bundler.py

@@ -0,0 +1,77 @@
+"""
+G2P model bundler - creates standalone Python executables with embedded models.
+"""
+
+from __future__ import annotations
+
+import tempfile
+from pathlib import Path
+
+from cartlet import bundle as cartlet_bundle
+
+from .constants import FILE_ENCODING
+
+
+def _ensure_cart_format(model_path: str) -> tuple[str, bool]:
+    """
+    Ensure model is in .cart format for bundling.
+
+    Returns:
+        (cart_path, needs_cleanup) - path to .cart file and whether to delete it
+    """
+    if model_path.endswith(".cart"):
+        return model_path, False
+
+    # Convert .g2p.gz or other formats to .cart
+    from .core.g2p_model import G2PDecisionTree
+
+    dt = G2PDecisionTree()
+    dt.load_model(model_path)
+
+    with tempfile.NamedTemporaryFile(suffix=".cart", delete=False) as tmp:
+        cart_path = tmp.name
+
+    dt._cart.export(cart_path, store_distributions=dt._cart.store_distributions)
+    return cart_path, True
+
+
+def bundle_g2p(model_path: str, output_path: str) -> None:
+    """
+    Bundle G2P model into a standalone Python executable.
+
+    Args:
+        model_path: Path to model file (.g2p.gz, .cart, etc.)
+        output_path: Output `.py` file path
+    """
+    cart_path, cleanup = _ensure_cart_format(model_path)
+
+    try:
+        with tempfile.NamedTemporaryFile(suffix=".py", delete=False, mode="w") as tmp:
+            tmp_path = tmp.name
+
+        try:
+            # library_only excludes cartlet's CLI; we provide our own G2P main.
+            cartlet_bundle(cart_path, tmp_path, library_only=True)
+
+            with open(tmp_path, encoding=FILE_ENCODING) as f:
+                cart_code = f.read()
+        finally:
+            Path(tmp_path).unlink(missing_ok=True)
+
+        g2p_template_path = Path(__file__).parent / "cart" / "g2p_predict.py"
+        with open(g2p_template_path, encoding=FILE_ENCODING) as f:
+            g2p_code = f.read()
+
+        # Strip the file prelude; keep everything from class G2PPredictor onward.
+        class_marker = "class G2PPredictor"
+        if class_marker in g2p_code:
+            idx = g2p_code.index(class_marker)
+            g2p_code = g2p_code[idx:]
+
+        output = cart_code + "\n\n\n" + g2p_code
+
+        with open(output_path, "w", encoding=FILE_ENCODING) as f:
+            f.write(output)
+    finally:
+        if cleanup:
+            Path(cart_path).unlink(missing_ok=True)

phonebox/cart/__init__.py

@@ -0,0 +1,11 @@
+"""
+G2P predictor template for bundling.
+
+The g2p_predict.py template is appended to cartlet's bundled Python
+predictor to create a standalone .py executable with the model embedded.
+
+Usage:
+    phonebox bundle model.g2p.gz -o g2p.py
+"""
+
+from __future__ import annotations

phonebox/cart/g2p_predict.py

@@ -0,0 +1,315 @@
+#!/usr/bin/env python
+"""
+G2P Predictor - extends cartlet's Predictor with phoneme-specific logic.
+
+This template is bundled with a model using:
+    phonebox bundle model.g2p.gz -o g2p.py
+
+The bundler:
+1. Uses cartlet's bundler with library_only=True
+2. Appends this G2P wrapper code (vectorization, uncooking, CLI)
+
+Usage (bundled):
+    python g2p.py "Hello, world!"     # Tokenizes and pronounces
+    from g2p import G2PPredictor; g2p = G2PPredictor.from_embedded()
+
+Zero dependencies beyond Python stdlib.
+"""
+
+from __future__ import annotations
+
+# NOTE: This file is appended to cartlet's predict.py during bundling.
+# The Predictor class, load_embedded(), etc. are defined above.
+#
+# The bundler slices source from the G2PPredictor class header to EOF, so
+# any module-level helpers MUST appear after the class definition (see the
+# helper section below) to be included in the bundled output.
+
+
+# =============================================================================
+# G2P Predictor - extends Predictor with phoneme handling
+# =============================================================================
+
+
+class G2PPredictor(Predictor):  # type: ignore[name-defined]  # noqa: F821
+    """
+    G2P predictor with word-to-phoneme conversion.
+
+    Extends cartlet's Predictor with:
+    - Word vectorization (letter context windows, including digraph joining)
+    - Phoneme uncooking (split joined phones, remove epsilon)
+    - OOV handling
+
+    Example:
+        g2p = G2PPredictor.from_embedded()
+        phones = g2p.pronounce("hello")  # ['HH', 'AH', 'L', 'OW']
+    """
+
+    # Defensive defaults: applied only when the .cart file ships with no
+    # metadata trailer at all. Phonebox always writes one; this fallback
+    # keeps the bundle predictable rather than blowing up.
+    DEFAULT_WIDTH = 7
+    DEFAULT_AETHER = "\u2022"  # •
+    DEFAULT_EPSILON = "\u2205"  # ∅
+    DEFAULT_JOIN_CHAR = "\u208a"  # ₊
+    DEFAULT_CASED = False
+
+    def __init__(self, path_or_bytes=None):
+        super().__init__(path_or_bytes)
+        self._apply_meta()
+
+    def _apply_meta(self):
+        """Apply g2p config from model metadata (or defaults when empty)."""
+        # Predictor.metadata reads self._model["metadata"] safely.
+        meta = self.metadata
+
+        self.width = meta.get("width", self.DEFAULT_WIDTH)
+        self.aether = meta.get("aether", self.DEFAULT_AETHER)
+        self.epsilon = meta.get("epsilon", self.DEFAULT_EPSILON)
+        self.join_char = meta.get("join_char", self.DEFAULT_JOIN_CHAR)
+        self.cased = meta.get("cased", self.DEFAULT_CASED)
+        self.exceptions = meta.get("exceptions", {})
+
+        self.center_position = (self.width - 1) // 2
+        self.padding = [self.aether] * self.center_position
+
+        letter_joins = meta.get("join", {}).get("letters", [])
+        self.lett_join_re = _make_join_re(letter_joins)
+
+    @classmethod
+    def from_embedded(cls):
+        """Create G2PPredictor from embedded model data."""
+        model_data = load_embedded()  # type: ignore[name-defined]  # noqa: F821 - defined by cartlet
+        if model_data is None:
+            raise ValueError("No embedded model found")
+        obj = cls()
+        obj._model = model_data
+        obj._apply_meta()
+        return obj
+
+    def vectorize_word(self, word):
+        """Convert word to context vectors (applies digraph joining)."""
+        if not self.cased:
+            word = word.lower()
+
+        letters = list(word)
+        if self.lett_join_re is not None:
+            letters = _join_seq(self.lett_join_re, letters, self.join_char)
+
+        padded = self.padding + letters + self.padding
+        return [padded[i : i + self.width] for i in range(len(letters))]
+
+    def uncook(self, phonemes):
+        """Split joined phones and remove epsilon."""
+        result = []
+        for phoneme in phonemes:
+            if phoneme == self.epsilon:
+                continue
+            result.extend(phoneme.split(self.join_char))
+        return result
+
+    def pronounce(self, word):
+        """Pronounce a word, returning list of phonemes."""
+        if self._model is None:
+            raise ValueError("No model loaded")
+
+        # Check exceptions
+        lookup_word = word if self.cased else word.lower()
+        if lookup_word in self.exceptions:
+            return self.exceptions[lookup_word]
+
+        # Vectorize and predict, dropping any position whose center letter was
+        # never seen in training (otherwise the tree falls through default
+        # branches and emits a random phoneme). Matches phonebox.runner.G2PRunner.
+        vectors = self.vectorize_word(word)
+        raw_phones = []
+        for vec in vectors:
+            center = vec[self.center_position]
+            if self.is_oov(self.center_position, center):
+                raw_phones.append(self.epsilon)
+            else:
+                raw_phones.append(self.predict(vec))
+
+        return self.uncook(raw_phones)
+
+    def pronounce_batch(self, words):
+        """Pronounce multiple words."""
+        return [(word, self.pronounce(word)) for word in words]
+
+    def tokenize(self, text):
+        """
+        Tokenize text into words for pronunciation.
+
+        Normalizes and extracts pronounceable tokens:
+        - NFC unicode normalization
+        - Strips punctuation/symbols from token edges (Unicode categories P, S, C, M, Z)
+        - Keeps internal punctuation (apostrophes, hyphens)
+        """
+        import unicodedata
+
+        # Unicode categories to strip from token edges
+        exclude_cats = {"P", "S", "C", "M", "Z"}
+
+        # Normalize unicode
+        text = unicodedata.normalize("NFC", text)
+
+        # Split and normalize each token
+        result = []
+        for token in text.split():
+            # Strip leading excluded characters
+            while token and unicodedata.category(token[0])[0] in exclude_cats:
+                token = token[1:]
+            # Strip trailing excluded characters
+            while token and unicodedata.category(token[-1])[0] in exclude_cats:
+                token = token[:-1]
+            if token:
+                result.append(token)
+
+        return result
+
+    def pronounce_text(self, text, raw=False):
+        """
+        Pronounce text, returning list of (word, phones) tuples.
+
+        Args:
+            text: Input text
+            raw: If True, skip normalization (just split on whitespace)
+        """
+        words = text.split() if raw else self.tokenize(text)
+        return [(word, self.pronounce(word)) for word in words]
+
+    def __call__(self, word):
+        """Allow direct calling: g2p('hello')"""
+        return self.pronounce(word)
+
+
+# =============================================================================
+# Letter-joining helpers (stdlib-only)
+#
+# Kept module-level so they stay easy to inline into the bundled standalone
+# script. The .cart metadata trailer is parsed by cartlet's loader and
+# exposed as Predictor.metadata; the helpers below only handle the digraph
+# joining that's specific to G2P (not part of cartlet's domain).
+# =============================================================================
+
+
+def _make_join_re(join_list):
+    """Compile a regex matching any space-separated joining sequence, or None."""
+    import re
+
+    if not join_list:
+        return None
+    items = sorted(join_list, key=len, reverse=True)
+    items = [re.escape(x) for x in items]
+    return re.compile(r" (" + r"|".join(items) + r") ")
+
+
+def _join_seq(regex, seq, join_char):
+    """Collapse adjacent tokens in seq matched by regex using join_char."""
+    if not regex:
+        return list(seq)
+    as_str = " " + " ".join(seq) + " "
+    m = regex.search(as_str)
+    while m:
+        s = m.group(0)
+        joined = s[1:-1].replace(" ", join_char)
+        as_str = as_str.replace(s, " " + joined + " ")
+        m = regex.search(as_str)
+    return as_str[1:-1].split()
+
+
+# =============================================================================
+# G2P CLI (replaces cartlet's main when bundled as G2P)
+# =============================================================================
+
+
+def g2p_main():
+    """G2P command-line interface."""
+    import argparse
+    import sys
+
+    embedded_model = load_embedded()  # type: ignore[name-defined]  # noqa: F821 - defined by cartlet
+    has_embedded = embedded_model is not None
+
+    parser = argparse.ArgumentParser(
+        description="G2P: Convert text to phonemes.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  %(prog)s "Hello, world!"        # Pronounce text (tokenizes automatically)
+  %(prog)s -f input.txt           # Read text from file, tokenize each line
+  echo "Hello world" | %(prog)s   # Read from stdin
+""",
+    )
+
+    if has_embedded:
+        parser.add_argument(
+            "-m", "--model", metavar="FILE", help="Model file (overrides embedded)"
+        )
+    else:
+        parser.add_argument("-m", "--model", metavar="FILE", help="Model file (.cart)")
+
+    parser.add_argument(
+        "-f",
+        "--file",
+        metavar="FILE",
+        help="Read text from file (- for stdin), tokenize each line",
+    )
+    parser.add_argument(
+        "-r",
+        "--raw",
+        action="store_true",
+        help="Disable normalization (no punctuation stripping)",
+    )
+    parser.add_argument(
+        "text", nargs="*", help="Text to pronounce (tokenized into words)"
+    )
+
+    args = parser.parse_args()
+
+    # Load model
+    if args.model:
+        g2p = G2PPredictor(args.model)
+    elif has_embedded:
+        g2p = G2PPredictor.from_embedded()
+    else:
+        parser.error("No model specified. Use -m MODEL or use a bundled version.")
+
+    # Process text and output pronunciations
+    def process_text(text):
+        for word, phones in g2p.pronounce_text(text, raw=args.raw):
+            print(f"{word}\t{' '.join(phones)}")
+
+    # Get input
+    if args.file:
+        f = sys.stdin if args.file == "-" else open(args.file, encoding="utf-8")  # noqa: SIM115
+        try:
+            for line in f:
+                process_text(line)
+        finally:
+            if args.file != "-":
+                f.close()
+    elif args.text:
+        # Join all positional args as one text string
+        process_text(" ".join(args.text))
+    elif not sys.stdin.isatty():
+        for line in sys.stdin:
+            process_text(line)
+    else:
+        # Interactive mode
+        print("G2P Predictor. Enter text (Ctrl+D to exit):", file=sys.stderr)
+        try:
+            while True:
+                line = input("> ")
+                process_text(line)
+        except (EOFError, KeyboardInterrupt):
+            print(file=sys.stderr)
+            return 0
+
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(g2p_main())

phonebox/cli/__init__.py

@@ -0,0 +1,8 @@
+"""
+Command-line interface for phonebox.
+
+This module provides the main CLI entry point and subcommand structure
+for the phonebox grapheme-to-phoneme conversion tool.
+"""
+
+from __future__ import annotations

phonebox/cli/commands/__init__.py

@@ -0,0 +1,3 @@
+"""CLI command groups."""
+
+from __future__ import annotations

phonebox/cli/commands/_common.py

@@ -0,0 +1,61 @@
+"""Shared argparse helpers for CLI subcommands."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from ...constants import DEFAULT_LOCALE, DEFAULT_PHONESET
+
+# Conventional exit code for a bad-input / usage error (missing file, etc.).
+EXIT_BAD_INPUT = 2
+
+
+def require_file(path: str | Path, label: str = "file") -> int | None:
+    """Return EXIT_BAD_INPUT (after printing a message) if *path* is missing.
+
+    Lets command handlers guard their inputs uniformly::
+
+        if (rc := require_file(args.dict, "dictionary")) is not None:
+            return rc
+
+    Returns None when the file exists, so callers proceed normally.
+    """
+    if not Path(path).is_file():
+        print(f"Error: {label} not found: {path}", file=sys.stderr)
+        return EXIT_BAD_INPUT
+    return None
+
+
+def add_vectorizer_args(
+    parser: argparse.ArgumentParser,
+    *,
+    cased: bool = False,
+    target_first: bool = False,
+) -> None:
+    """Add the locale/phoneset/stress options shared by vectorizer-driven commands.
+
+    ``cased`` and ``target_first`` add the corresponding optional flags only for
+    commands that consume them, keeping each command's surface identical to its
+    hand-written original while sharing one definition and one set of defaults.
+    """
+    parser.add_argument(
+        "--locale", default=DEFAULT_LOCALE, help="Language locale (e.g. en_US)"
+    )
+    parser.add_argument(
+        "--phoneset", default=DEFAULT_PHONESET, help="Phoneset name (e.g. cmu, ipa)"
+    )
+    parser.add_argument(
+        "--remove-stress", action="store_true", help="Remove stress markers"
+    )
+    if cased:
+        parser.add_argument(
+            "--cased", action="store_true", help="Case-sensitive letters"
+        )
+    if target_first:
+        parser.add_argument(
+            "--target-first",
+            action="store_true",
+            help="Place target column first (default: last)",
+        )

phonebox/cli/commands/align.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+"""Alignment operations: align letters to phonemes."""
+
+from __future__ import annotations
+
+import sys
+
+from ._common import add_vectorizer_args, require_file
+
+
+def setup_align_commands(subparsers):
+    """Setup align command."""
+    align_parser = subparsers.add_parser(
+        "align",
+        help="Align letters to phonemes",
+        description="Align letters to phonemes using EM algorithm",
+    )
+    align_parser.add_argument("dict", help="Dictionary file")
+    align_parser.add_argument("-o", "--output", help="Output alignment file")
+    add_vectorizer_args(align_parser)
+    align_parser.add_argument("--max-iterations", type=int, help="Max EM iterations")
+    align_parser.add_argument(
+        "--max-combinations", type=int, help="Max combinations per alignment"
+    )
+    align_parser.set_defaults(func=handle_align)
+
+
+def handle_align(args):
+    """Handle 'phonebox align' command."""
+    from ...constants import DICT_ENCODING
+    from ...core.em_align import EMAlign
+    from ...core.vectorizer import Vectorizer
+    from ...utils.io import is_dict_comment
+
+    if (rc := require_file(args.dict, "dictionary")) is not None:
+        return rc
+
+    print(f"Aligning dictionary: {args.dict}", file=sys.stderr)
+
+    # Create vectorizer with same settings that will be used for vectorization
+    vectorizer = Vectorizer(
+        locale=args.locale,
+        phoneset_name=args.phoneset,
+        remove_stress=args.remove_stress,
+        verbose=True,
+    )
+
+    # Create EM aligner
+    em = EMAlign(
+        vectorizer,
+        max_iterations=args.max_iterations,
+        max_combinations=args.max_combinations,
+        verbose=True,
+    )
+
+    # Load dictionary
+    with open(args.dict, encoding=DICT_ENCODING) as f:
+        for line in f:
+            if not is_dict_comment(line):
+                em.add_line(line)
+
+    # Run alignment
+    em.align()
+
+    # Write output
+    if args.output:
+        with open(args.output, "w", encoding=DICT_ENCODING) as f:
+            em.write(f)
+        print(f"Done: Alignments saved to {args.output}", file=sys.stderr)
+    else:
+        em.write(sys.stdout)
+
+    return 0

phonebox/cli/commands/bundle.py

@@ -0,0 +1,71 @@
+"""Bundle command: create standalone G2P predictor."""
+
+from __future__ import annotations
+
+import os
+import sys
+import tempfile
+
+
+def setup_bundle_command(subparsers):
+    """Setup bundle command."""
+    parser = subparsers.add_parser(
+        "bundle",
+        help="Create standalone Python G2P predictor with embedded model",
+        description="""Bundle G2P model into a standalone Python executable.
+
+Creates a self-contained .py file with the model embedded.
+
+Examples:
+  phonebox bundle model.g2p.gz -o g2p.py
+
+Usage of bundled file:
+  python g2p.py "Hello, world!"
+
+Library usage:
+  from g2p import G2PPredictor
+  g2p = G2PPredictor.from_embedded()
+  phones = g2p.pronounce_text('Hello, world!')""",
+    )
+    parser.add_argument("model", help="Model file (.g2p.gz, .cart, etc.)")
+    parser.add_argument("-o", "--output", required=True, help="Output .py file")
+    parser.set_defaults(func=handle_bundle)
+
+
+def handle_bundle(args):
+    """Handle 'phonebox bundle' command."""
+    from ...bundler import bundle_g2p
+    from ...core.g2p_model import G2PDecisionTree
+
+    model_path = args.model
+    output_path = args.output
+
+    # Convert to .cart format if needed (cartlet's bundler requires .cart)
+    needs_conversion = not model_path.endswith(".cart")
+
+    if needs_conversion:
+        print(f"Converting {model_path} to .cart for bundling...", file=sys.stderr)
+
+        dt = G2PDecisionTree(model=model_path)
+
+        with tempfile.NamedTemporaryFile(suffix=".cart", delete=False) as tmp:
+            cart_path = tmp.name
+
+        dt._cart.export(cart_path, store_distributions=dt._cart.store_distributions)
+        model_path = cart_path
+
+    try:
+        print(f"Bundling -> {output_path}", file=sys.stderr)
+        bundle_g2p(model_path, output_path)
+        print(f"Done: Created {output_path}", file=sys.stderr)
+
+        module_name = os.path.splitext(os.path.basename(output_path))[0]
+        print("\nUsage:", file=sys.stderr)
+        print(f'  CLI:     python {output_path} "Hello, world!"', file=sys.stderr)
+        print(f"  Library: from {module_name} import G2PPredictor", file=sys.stderr)
+        print("           g2p = G2PPredictor.from_embedded()", file=sys.stderr)
+        print("           g2p.pronounce_text('Hello, world!')", file=sys.stderr)
+        return 0
+    finally:
+        if needs_conversion:
+            os.unlink(cart_path)