g2p2 0.1.0__tar.gz

g2p2-0.1.0/Cargo.toml

@@ -0,0 +1,6 @@
+[workspace]
+members = ["g2p", "xtask"]
+# The Python binding pulls in pyo3; keep it out of the workspace so plain
+# `cargo build/test --workspace` and the zero-dep runtime stay clean.
+exclude = ["bindings/python"]
+resolver = "2"

g2p2-0.1.0/PKG-INFO

@@ -0,0 +1,45 @@
+Metadata-Version: 2.4
+Name: g2p2
+Version: 0.1.0
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: License :: OSI Approved :: MIT License
+Summary: Zero-dependency grapheme-to-phoneme for the 100 Whisper languages (Rust core)
+Keywords: g2p,grapheme-to-phoneme,phonemizer,ipa,whisper,phonetics
+Author: jqueguiner
+License: MIT OR Apache-2.0
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Repository, https://github.com/jqueguiner/g2p2
+
+# g2p2
+
+Zero-dependency grapheme-to-phoneme for the **100 Whisper languages**, with a
+Rust core exposed to Python.
+
+```bash
+pip install g2p2
+```
+
+```python
+from g2p2 import Model
+
+m = Model.load("fr.g2p")
+print(m.phonemize("bonjour"))            # bɔ̃ʒuʁ
+print(m.phonemize_many(["chat", "eau"])) # ['ʃa', 'o']
+```
+
+Model blobs (`*.g2p`) are built from WikiPron / epitran / LLM data — see the
+[main repo](https://github.com/jqueguiner/g2p2). Download a language's `.g2p`
+from the GitHub releases or build with `cargo run -p xtask -- build-all`.
+
+## API
+
+- `Model.load(path) -> Model` — load a `.g2p` file
+- `Model.from_bytes(bytes) -> Model` — load from raw bytes
+- `Model.phonemize(word) -> str` — IPA for one word
+- `Model.phonemize_many(words) -> list[str]` — IPA for many
+
+License: MIT OR Apache-2.0.
+

g2p2-0.1.0/README.md

@@ -0,0 +1,29 @@
+# g2p2
+
+Zero-dependency grapheme-to-phoneme for the **100 Whisper languages**, with a
+Rust core exposed to Python.
+
+```bash
+pip install g2p2
+```
+
+```python
+from g2p2 import Model
+
+m = Model.load("fr.g2p")
+print(m.phonemize("bonjour"))            # bɔ̃ʒuʁ
+print(m.phonemize_many(["chat", "eau"])) # ['ʃa', 'o']
+```
+
+Model blobs (`*.g2p`) are built from WikiPron / epitran / LLM data — see the
+[main repo](https://github.com/jqueguiner/g2p2). Download a language's `.g2p`
+from the GitHub releases or build with `cargo run -p xtask -- build-all`.
+
+## API
+
+- `Model.load(path) -> Model` — load a `.g2p` file
+- `Model.from_bytes(bytes) -> Model` — load from raw bytes
+- `Model.phonemize(word) -> str` — IPA for one word
+- `Model.phonemize_many(words) -> list[str]` — IPA for many
+
+License: MIT OR Apache-2.0.

g2p2-0.1.0/bindings/python/Cargo.lock

@@ -0,0 +1,185 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+
+[[package]]
+name = "g2p"
+version = "0.1.0"
+
+[[package]]
+name = "g2p2"
+version = "0.1.0"
+dependencies = [
+ "g2p",
+ "pyo3",
+]
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "indoc"
+version = "2.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.186"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+
+[[package]]
+name = "memoffset"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "portable-atomic"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884"
+dependencies = [
+ "cfg-if",
+ "indoc",
+ "libc",
+ "memoffset",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.22.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rustversion"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.12.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "unindent"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"

g2p2-0.1.0/bindings/python/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "g2p2"
+version = "0.1.0"
+edition = "2021"
+description = "Python bindings for g2p — zero-dependency grapheme-to-phoneme for the 100 Whisper languages"
+license = "MIT OR Apache-2.0"
+readme = "README.md"
+
+[lib]
+name = "g2p2"
+crate-type = ["cdylib"]
+
+[dependencies]
+g2p = { path = "../../g2p" }
+pyo3 = { version = "0.22", features = ["extension-module", "abi3-py38"] }

g2p2-0.1.0/bindings/python/README.md

@@ -0,0 +1,29 @@
+# g2p2
+
+Zero-dependency grapheme-to-phoneme for the **100 Whisper languages**, with a
+Rust core exposed to Python.
+
+```bash
+pip install g2p2
+```
+
+```python
+from g2p2 import Model
+
+m = Model.load("fr.g2p")
+print(m.phonemize("bonjour"))            # bɔ̃ʒuʁ
+print(m.phonemize_many(["chat", "eau"])) # ['ʃa', 'o']
+```
+
+Model blobs (`*.g2p`) are built from WikiPron / epitran / LLM data — see the
+[main repo](https://github.com/jqueguiner/g2p2). Download a language's `.g2p`
+from the GitHub releases or build with `cargo run -p xtask -- build-all`.
+
+## API
+
+- `Model.load(path) -> Model` — load a `.g2p` file
+- `Model.from_bytes(bytes) -> Model` — load from raw bytes
+- `Model.phonemize(word) -> str` — IPA for one word
+- `Model.phonemize_many(words) -> list[str]` — IPA for many
+
+License: MIT OR Apache-2.0.

g2p2-0.1.0/bindings/python/src/lib.rs

@@ -0,0 +1,58 @@
+//! Python bindings for the `g2p` engine (PyO3 + maturin, abi3).
+//!
+//! ```python
+//! from g2p2 import Model
+//! m = Model.load("fr.g2p")
+//! m.phonemize("bonjour")          # 'bɔ̃ʒuʁ'
+//! m.phonemize_many(["a", "b"])    # ['...', '...']
+//! ```
+
+use pyo3::exceptions::PyIOError;
+use pyo3::prelude::*;
+
+/// A loaded G2P model. Construct with `Model.load(path)` or `Model.from_bytes(b)`.
+#[pyclass]
+struct Model {
+    inner: g2p::Model,
+}
+
+#[pymethods]
+impl Model {
+    /// Load a compiled `.g2p` blob from disk.
+    #[staticmethod]
+    fn load(path: &str) -> PyResult<Self> {
+        let bytes = std::fs::read(path).map_err(|e| PyIOError::new_err(e.to_string()))?;
+        Ok(Self {
+            inner: g2p::Model::from_bytes(&bytes),
+        })
+    }
+
+    /// Load a model from raw `.g2p` bytes.
+    #[staticmethod]
+    fn from_bytes(bytes: &[u8]) -> Self {
+        Self {
+            inner: g2p::Model::from_bytes(bytes),
+        }
+    }
+
+    /// Phonemize one word -> IPA string.
+    fn phonemize(&self, word: &str) -> String {
+        g2p::phonemize(&self.inner, word)
+    }
+
+    /// Phonemize many words -> list of IPA strings.
+    fn phonemize_many(&self, words: Vec<String>) -> Vec<String> {
+        words.iter().map(|w| g2p::phonemize(&self.inner, w)).collect()
+    }
+
+    fn __repr__(&self) -> String {
+        format!("<g2p2.Model {} tokens>", self.inner.tokens.len())
+    }
+}
+
+#[pymodule]
+fn g2p2(m: &Bound<'_, PyModule>) -> PyResult<()> {
+    m.add("__version__", env!("CARGO_PKG_VERSION"))?;
+    m.add_class::<Model>()?;
+    Ok(())
+}

g2p2-0.1.0/g2p/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "g2p"
+version = "0.1.0"
+edition = "2021"
+description = "Zero-dependency grapheme-to-phoneme for the 101 Whisper languages"
+
+[dependencies]

g2p2-0.1.0/g2p/src/bin/g2p.rs

@@ -0,0 +1,39 @@
+//! `g2p` — command-line grapheme-to-phoneme.
+//!
+//!   g2p <model.g2p> <word>...      phonemize each word
+//!   echo "words" | g2p <model.g2p> read whitespace-separated words from stdin
+//!
+//! Models are the `.g2p` blobs produced by the build tool. Zero dependencies.
+
+use std::io::Read;
+use std::process::ExitCode;
+
+fn main() -> ExitCode {
+    let args: Vec<String> = std::env::args().skip(1).collect();
+    let Some(model_path) = args.first() else {
+        eprintln!("usage: g2p <model.g2p> [word ...]   (words also read from stdin)");
+        return ExitCode::from(2);
+    };
+
+    let bytes = match std::fs::read(model_path) {
+        Ok(b) => b,
+        Err(e) => {
+            eprintln!("g2p: cannot read {model_path}: {e}");
+            return ExitCode::from(1);
+        }
+    };
+    let model = g2p::Model::from_bytes(&bytes);
+
+    let mut words: Vec<String> = args[1..].to_vec();
+    if words.is_empty() {
+        let mut buf = String::new();
+        if std::io::stdin().read_to_string(&mut buf).is_ok() {
+            words = buf.split_whitespace().map(String::from).collect();
+        }
+    }
+
+    for w in words {
+        println!("{}", g2p::phonemize(&model, &w));
+    }
+    ExitCode::SUCCESS
+}

g2p2-0.1.0/g2p/src/decode.rs

@@ -0,0 +1,279 @@
+//! Beam-search decoder over the joint n-gram model.
+//!
+//! Walk the input grapheme by grapheme; at each position try every grapheme
+//! chunk of length `1..=max_chunk`, expand each candidate joint token, score
+//! with `logp`, keep the top-`BEAM` hypotheses. Close with an EOS transition.
+
+use crate::model::{Model, EOS};
+use crate::normalize::{graphemes, lower};
+
+const BEAM: usize = 8;
+/// Penalty for skipping an unknown grapheme (no chunk matched at that position).
+const SKIP_PEN: f32 = -20.0;
+
+struct Hyp {
+    pos: usize,
+    hist: Vec<u32>,
+    out: Vec<u32>,
+    score: f32,
+}
+
+impl Hyp {
+    fn dup(&self) -> Hyp {
+        Hyp {
+            pos: self.pos,
+            hist: self.hist.clone(),
+            out: self.out.clone(),
+            score: self.score,
+        }
+    }
+}
+
+/// Phonemize a single word.
+///
+/// Tiers: exact lexicon (word, then lowercased) -> n-gram beam ->
+/// per-character lexicon fallback (logographic OOV).
+pub fn phonemize(m: &Model, word: &str) -> String {
+    // 1. exact lexicon (logographic words / irregulars)
+    if let Some(ipa) = m.lexicon.get(word) {
+        return ipa.to_string();
+    }
+    let lw = lower(word);
+    if lw != word {
+        if let Some(ipa) = m.lexicon.get(lw.as_str()) {
+            return ipa.to_string();
+        }
+    }
+    // Logographic: per-character lexicon before the (unreliable) n-gram.
+    // Alphabetic: n-gram before the char fallback.
+    if m.logo {
+        let r = char_fallback(m, word);
+        if !r.is_empty() {
+            return r;
+        }
+        if !m.ngram.is_empty() {
+            return beam(m, &lw);
+        }
+        String::new()
+    } else {
+        if !m.ngram.is_empty() {
+            let r = beam(m, &lw);
+            if !r.is_empty() {
+                return r;
+            }
+        }
+        char_fallback(m, word)
+    }
+}
+
+/// Concatenate per-grapheme lexicon readings (logographic OOV). Empty if none hit.
+fn char_fallback(m: &Model, word: &str) -> String {
+    let mut out = String::new();
+    let mut any = false;
+    for ch in graphemes(word) {
+        if let Some(ipa) = m.lexicon.get(&*ch) {
+            out.push_str(ipa);
+            any = true;
+        }
+    }
+    if any {
+        out
+    } else {
+        String::new()
+    }
+}
+
+fn beam(m: &Model, lw: &str) -> String {
+    let g = graphemes(lw);
+    let n = g.len();
+    if n == 0 {
+        return String::new();
+    }
+    let ord = (m.order as usize).max(1);
+
+    let mut beam = vec![Hyp {
+        pos: 0,
+        hist: Vec::new(),
+        out: Vec::new(),
+        score: 0.0,
+    }];
+
+    while !beam.iter().all(|h| h.pos == n) {
+        let mut next: Vec<Hyp> = Vec::new();
+        for h in &beam {
+            if h.pos == n {
+                next.push(h.dup());
+                continue;
+            }
+            let maxk = m.max_chunk.min(n - h.pos);
+            let mut matched = false;
+            for k in 1..=maxk {
+                let chunk: String = g[h.pos..h.pos + k].concat();
+                if let Some(ids) = m.by_graph.get(chunk.as_str()) {
+                    matched = true;
+                    for &tok in ids {
+                        let lp = m.logp(&h.hist, tok);
+                        let mut hist = h.hist.clone();
+                        hist.push(tok);
+                        if ord > 1 && hist.len() > ord - 1 {
+                            let d = hist.len() - (ord - 1);
+                            hist.drain(0..d);
+                        } else if ord == 1 {
+                            hist.clear();
+                        }
+                        let mut out = h.out.clone();
+                        out.push(tok);
+                        next.push(Hyp {
+                            pos: h.pos + k,
+                            hist,
+                            out,
+                            score: h.score + lp,
+                        });
+                    }
+                }
+            }
+            if !matched {
+                // Unknown grapheme: skip one unit with a penalty, emit nothing.
+                next.push(Hyp {
+                    pos: h.pos + 1,
+                    hist: h.hist.clone(),
+                    out: h.out.clone(),
+                    score: h.score + SKIP_PEN,
+                });
+            }
+        }
+        if next.is_empty() {
+            break;
+        }
+        next.sort_by(|a, b| b.score.total_cmp(&a.score));
+        next.truncate(BEAM);
+        beam = next;
+    }
+
+    // Close each finished hypothesis with an EOS transition, pick the best.
+    let best = beam
+        .iter()
+        .filter(|h| h.pos == n)
+        .map(|h| (h.score + m.logp(&h.hist, EOS), h))
+        .max_by(|a, b| a.0.total_cmp(&b.0));
+
+    match best {
+        Some((_, h)) => h.out.iter().map(|&t| &*m.tokens[t as usize].1).collect(),
+        None => String::new(),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::collections::HashMap;
+
+    /// Toy model: graphemes a->ɑ, b->b, with a unigram n-gram so the beam runs.
+    fn toy(logo: bool) -> Model {
+        let mut m = Model {
+            tokens: vec![
+                (String::new().into(), String::new().into()), // EOS
+                ("a".into(), "ɑ".into()),
+                ("b".into(), "b".into()),
+            ],
+            order: 2,
+            logo,
+            max_chunk: 0,
+            by_graph: HashMap::new(),
+            ngram: HashMap::from([
+                (Box::from(&[0u32][..]), -0.5f32),
+                (Box::from(&[1u32][..]), -0.5f32),
+                (Box::from(&[2u32][..]), -0.5f32),
+            ]),
+            backoff: HashMap::new(),
+            unk: -5.0,
+            lexicon: HashMap::new(),
+        };
+        m.index();
+        m
+    }
+
+    #[test]
+    fn empty_word_empty_output() {
+        assert_eq!(phonemize(&toy(false), ""), "");
+    }
+
+    #[test]
+    fn beam_decodes_known_graphemes() {
+        assert_eq!(phonemize(&toy(false), "ab"), "ɑb");
+    }
+
+    #[test]
+    fn uppercase_lowercased_before_decode() {
+        assert_eq!(phonemize(&toy(false), "AB"), "ɑb");
+    }
+
+    #[test]
+    fn unknown_grapheme_skipped_to_empty() {
+        // 'z' matches nothing -> skipped -> no output
+        assert_eq!(phonemize(&toy(false), "z"), "");
+    }
+
+    #[test]
+    fn exact_lexicon_wins() {
+        let mut m = toy(false);
+        m.lexicon.insert("ab".into(), "EXACT".into());
+        assert_eq!(phonemize(&m, "ab"), "EXACT");
+    }
+
+    #[test]
+    fn lexicon_lowercased_lookup() {
+        let mut m = toy(false);
+        m.lexicon.insert("cat".into(), "kat".into());
+        assert_eq!(phonemize(&m, "CAT"), "kat");
+    }
+
+    #[test]
+    fn logographic_char_fallback() {
+        let mut m = toy(true);
+        m.lexicon.insert("中".into(), "tʂʊŋ".into());
+        m.lexicon.insert("国".into(), "kwɔ".into());
+        // whole word absent -> per-char fallback concatenates
+        assert_eq!(phonemize(&m, "中国"), "tʂʊŋkwɔ");
+    }
+
+    #[test]
+    fn logographic_falls_through_to_ngram() {
+        // logo model, char fallback empty (a/b not in lexicon) -> n-gram beam
+        assert_eq!(phonemize(&toy(true), "ab"), "ɑb");
+    }
+
+    #[test]
+    fn no_ngram_no_lexicon_is_empty() {
+        let mut m = toy(false);
+        m.ngram.clear();
+        assert_eq!(phonemize(&m, "ab"), "");
+    }
+
+    #[test]
+    fn logo_all_miss_is_empty() {
+        let mut m = toy(true);
+        m.ngram.clear();
+        assert_eq!(phonemize(&m, "z"), "");
+    }
+
+    #[test]
+    fn multi_grapheme_token_and_dup() {
+        // add a 2-grapheme token so one hypothesis finishes before another,
+        // exercising the carry-forward (dup) path.
+        let mut m = toy(false);
+        m.tokens.push(("ab".into(), "AB".into()));
+        m.ngram.insert(Box::from(&[3u32][..]), -0.4);
+        m.index();
+        // both "ɑb" (a+b) and "AB" (ab) are valid; decoder returns one of them
+        let out = phonemize(&m, "ab");
+        assert!(out == "AB" || out == "ɑb");
+    }
+
+    #[test]
+    fn order_one_model_clears_history() {
+        let mut m = toy(false);
+        m.order = 1;
+        assert_eq!(phonemize(&m, "ab"), "ɑb");
+    }
+}

g2p2-0.1.0/g2p/src/lang.rs

@@ -0,0 +1,163 @@
+//! The 100 Whisper languages mapped to their WikiPron source.
+//!
+//! `pat` is an extra substring the WikiPron filename must contain, used to
+//! disambiguate multi-script / multi-dialect languages (e.g. pick Latin pinyin
+//! for Chinese, Cyrillic for Serbian, US English). Empty `pat` = take the
+//! best-ranked file for the ISO code.
+//!
+//! `logo` marks logographic languages whose grapheme->phoneme mapping is not an
+//! n-gram problem (kanji/hanzi need a word->reading lexicon); they are trained
+//! for completeness but should be served from the lexicon tier.
+
+pub struct Lang {
+    pub whisper: &'static str,
+    pub iso: &'static str,
+    pub pat: &'static str,
+    pub logo: bool,
+}
+
+const fn l(whisper: &'static str, iso: &'static str, pat: &'static str, logo: bool) -> Lang {
+    Lang {
+        whisper,
+        iso,
+        pat,
+        logo,
+    }
+}
+
+/// Whisper large-v3 language set (100). Five have no WikiPron data
+/// (sn/so/tt/ln/su) — resolved via epitran silver.
+pub const LANGS: &[Lang] = &[
+    l("en", "eng", "us", false),
+    l("zh", "cmn", "hani", true),
+    l("de", "deu", "", false),
+    l("es", "spa", "", false),
+    l("ru", "rus", "", false),
+    l("ko", "kor", "", false),
+    l("fr", "fra", "", false),
+    l("ja", "jpn", "", true),
+    l("pt", "por", "", false),
+    l("tr", "tur", "", false),
+    l("pl", "pol", "", false),
+    l("ca", "cat", "", false),
+    l("nl", "nld", "", false),
+    l("ar", "ara", "", false),
+    l("sv", "swe", "", false),
+    l("it", "ita", "", false),
+    l("id", "ind", "", false),
+    l("hi", "hin", "", false),
+    l("fi", "fin", "", false),
+    l("vi", "vie", "", false),
+    l("he", "heb", "", false),
+    l("uk", "ukr", "", false),
+    l("el", "ell", "", false),
+    l("ms", "msa", "", false),
+    l("cs", "ces", "", false),
+    l("ro", "ron", "", false),
+    l("da", "dan", "", false),
+    l("hu", "hun", "", false),
+    l("ta", "tam", "", false),
+    l("no", "nor", "", false),
+    l("th", "tha", "", false),
+    l("ur", "urd", "arab", false),
+    l("hr", "hbs", "latn", false),
+    l("bg", "bul", "", false),
+    l("lt", "lit", "", false),
+    l("la", "lat", "", false),
+    l("mi", "mri", "", false),
+    l("ml", "mal", "", false),
+    l("cy", "cym", "", false),
+    l("sk", "slk", "", false),
+    l("te", "tel", "", false),
+    l("fa", "fas", "", false),
+    l("lv", "lav", "", false),
+    l("bn", "ben", "", false),
+    l("sr", "hbs", "cyrl", false),
+    l("az", "aze", "latn", false),
+    l("sl", "slv", "", false),
+    l("kn", "kan", "", false),
+    l("et", "est", "", false),
+    l("mk", "mkd", "", false),
+    l("br", "bre", "", false),
+    l("eu", "eus", "", false),
+    l("is", "isl", "", false),
+    l("hy", "hye", "", false),
+    l("ne", "nep", "", false),
+    l("mn", "mon", "cyrl", false),
+    l("bs", "hbs", "latn", false),
+    l("kk", "kaz", "cyrl", false),
+    l("sq", "sqi", "", false),
+    l("sw", "swa", "", false),
+    l("gl", "glg", "", false),
+    l("mr", "mar", "", false),
+    l("pa", "pan", "guru", false),
+    l("si", "sin", "", false),
+    l("km", "khm", "", false),
+    l("sn", "sna", "", false), // no wikipron -> silver
+    l("yo", "yor", "", false),
+    l("so", "som", "", false), // no wikipron -> silver
+    l("af", "afr", "", false),
+    l("oc", "oci", "", false),
+    l("ka", "kat", "", false),
+    l("be", "bel", "", false),
+    l("tg", "tgk", "cyrl", false),
+    l("sd", "snd", "arab", false),
+    l("gu", "guj", "", false),
+    l("am", "amh", "", false),
+    l("yi", "yid", "", false),
+    l("lo", "lao", "", false),
+    l("uz", "uzb", "latn", false),
+    l("fo", "fao", "", false),
+    l("ht", "hat", "", false),
+    l("ps", "pus", "", false),
+    l("tk", "tuk", "latn", false),
+    l("nn", "nno", "", false),
+    l("mt", "mlt", "", false),
+    l("sa", "san", "", false),
+    l("lb", "ltz", "", false),
+    l("my", "mya", "", false),
+    l("bo", "bod", "", false),
+    l("tl", "tgl", "", false),
+    l("mg", "mlg", "", false),
+    l("as", "asm", "", false),
+    l("tt", "tat", "", false), // no wikipron -> silver
+    l("haw", "haw", "", false),
+    l("ln", "lin", "", false), // no wikipron -> silver
+    l("ha", "hau", "", false),
+    l("ba", "bak", "", false),
+    l("jw", "jav", "", false),
+    l("su", "sun", "", false), // no wikipron -> silver
+    l("yue", "yue", "", true),
+];
+
+/// Look up a language entry by Whisper code.
+pub fn by_whisper(code: &str) -> Option<&'static Lang> {
+    LANGS.iter().find(|l| l.whisper == code)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn covers_100_whisper_langs() {
+        assert_eq!(LANGS.len(), 100);
+    }
+
+    #[test]
+    fn lookup_hit_and_miss() {
+        let fr = by_whisper("fr").expect("fr present");
+        assert_eq!(fr.iso, "fra");
+        assert!(!fr.logo);
+        assert!(by_whisper("zh").unwrap().logo); // logographic flag set
+        assert!(by_whisper("nonsense").is_none());
+    }
+
+    #[test]
+    fn whisper_codes_unique() {
+        let mut seen = std::collections::HashSet::new();
+        for l in LANGS {
+            assert!(seen.insert(l.whisper), "duplicate {}", l.whisper);
+        }
+    }
+}

g2p2-0.1.0/g2p/src/lexicon.rs

@@ -0,0 +1,56 @@
+//! Exact-match lexicon tier. Checked before the n-gram decoder for irregular
+//! words, and used as the primary engine for logographic languages
+//! (zh/ja/ko/...) where grapheme->phoneme n-grams do not apply.
+
+use std::collections::HashMap;
+
+#[derive(Default)]
+pub struct Lexicon {
+    map: HashMap<Box<str>, Box<str>>,
+}
+
+impl Lexicon {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn insert(&mut self, word: &str, ipa: &str) {
+        self.map.insert(word.into(), ipa.into());
+    }
+
+    /// Exact lookup. Returns `None` on miss (caller falls through to n-gram).
+    pub fn get(&self, word: &str) -> Option<&str> {
+        self.map.get(word).map(|b| &**b)
+    }
+
+    pub fn len(&self) -> usize {
+        self.map.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.map.is_empty()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn insert_get_miss() {
+        let mut lex = Lexicon::new();
+        assert!(lex.is_empty());
+        lex.insert("cat", "k a t");
+        assert_eq!(lex.len(), 1);
+        assert!(!lex.is_empty());
+        assert_eq!(lex.get("cat"), Some("k a t"));
+        assert_eq!(lex.get("dog"), None);
+    }
+
+    #[test]
+    fn default_is_empty() {
+        let lex = Lexicon::default();
+        assert!(lex.is_empty());
+        assert_eq!(lex.len(), 0);
+    }
+}

g2p2-0.1.0/g2p/src/lib.rs

@@ -0,0 +1,14 @@
+//! Zero-dependency grapheme-to-phoneme (G2P) for the 101 Whisper languages.
+//!
+//! Runtime engine: pair joint n-gram + beam decode. Pure `std`, CPU.
+//! Models are compiled offline by `xtask` into `.g2p` blobs and loaded via
+//! [`Model::from_bytes`].
+
+pub mod decode;
+pub mod lang;
+pub mod lexicon;
+pub mod model;
+pub mod normalize;
+
+pub use decode::phonemize;
+pub use model::Model;

g2p2-0.1.0/g2p/src/model.rs

@@ -0,0 +1,315 @@
+//! The n-gram G2P model + its binary `.g2p` format.
+//!
+//! One source of truth for serialization: `xtask` writes via [`Model::to_bytes`],
+//! the runtime reads via [`Model::from_bytes`]. Same struct both sides -> the
+//! blob format cannot drift.
+//!
+//! N-gram probabilities are stored **fully interpolated** (Witten-Bell) per seen
+//! gram, so runtime lookup is a clean backoff recursion with no interpolation
+//! math: exact gram hit returns the stored logprob (lower orders already folded
+//! in); a miss multiplies by the history backoff weight and drops the oldest
+//! token.
+
+use std::collections::HashMap;
+
+use crate::normalize::graphemes;
+
+/// A joint alignment token: (grapheme side, phoneme side).
+/// Token id 0 is reserved for [`EOS`] and has empty sides.
+pub type Tok = (Box<str>, Box<str>);
+
+/// End-of-sequence token id. Appended during training so the model learns
+/// word-final phonotactics; the decoder must take it to close a word.
+pub const EOS: u32 = 0;
+
+const MAGIC: &[u8; 4] = b"G2P2";
+/// Fixed-point scale for quantized ln-probabilities (i16). 0.001 resolution is
+/// far finer than argmax decoding needs.
+const QSCALE: f32 = 1000.0;
+
+pub struct Model {
+    /// id -> token. index 0 = EOS ("", "").
+    pub tokens: Vec<Tok>,
+    /// n-gram order (e.g. 6).
+    pub order: u8,
+    /// logographic language: prefer the exact/per-char lexicon over the n-gram,
+    /// which is unreliable on ideographs.
+    pub logo: bool,
+    /// max grapheme-cluster length of any token's grapheme side (derived).
+    pub max_chunk: usize,
+    /// grapheme chunk -> candidate token ids (derived, not serialized).
+    pub by_graph: HashMap<Box<str>, Vec<u32>>,
+    /// exact gram (history ++ token) -> ln p, fully interpolated.
+    pub ngram: HashMap<Box<[u32]>, f32>,
+    /// history -> ln backoff weight (1 - lambda).
+    pub backoff: HashMap<Box<[u32]>, f32>,
+    /// ln floor for an unseen unigram.
+    pub unk: f32,
+    /// exact word -> IPA (continuous). Primary tier for logographic languages
+    /// (zh/ja/yue) and irregulars; checked before the n-gram in `phonemize`.
+    /// Single-character entries double as the per-character OOV fallback.
+    pub lexicon: HashMap<Box<str>, Box<str>>,
+}
+
+impl Model {
+    /// Rebuild the derived `by_graph` index and `max_chunk` from `tokens`.
+    /// Empty-grapheme specials (EOS) are skipped — they are never decode
+    /// candidates mid-word.
+    pub fn index(&mut self) {
+        self.by_graph.clear();
+        self.max_chunk = 1;
+        for (id, (g, _)) in self.tokens.iter().enumerate() {
+            if g.is_empty() {
+                continue;
+            }
+            self.max_chunk = self.max_chunk.max(graphemes(g).len());
+            self.by_graph.entry(g.clone()).or_default().push(id as u32);
+        }
+    }
+
+    /// `ln p(t | hist)`. `hist` is the last `order-1` token ids, most-recent
+    /// last. Interpolation is baked into stored grams, so: exact gram hit
+    /// returns it; otherwise back off by the history weight and drop the oldest
+    /// token.
+    pub fn logp(&self, hist: &[u32], t: u32) -> f32 {
+        let mut key: Vec<u32> = Vec::with_capacity(hist.len() + 1);
+        key.extend_from_slice(hist);
+        key.push(t);
+        if let Some(&lp) = self.ngram.get(key.as_slice()) {
+            return lp;
+        }
+        if hist.is_empty() {
+            return self.unk; // unseen unigram
+        }
+        let bow = self.backoff.get(hist).copied().unwrap_or(0.0); // ln(1) if history unseen
+        bow + self.logp(&hist[1..], t)
+    }
+
+    // ---- binary format ----
+
+    pub fn to_bytes(&self) -> Vec<u8> {
+        let mut o = Vec::new();
+        o.extend_from_slice(MAGIC);
+        o.push(self.order);
+        o.push(self.logo as u8);
+        o.extend_from_slice(&[0, 0]); // pad
+        put_u32(&mut o, self.tokens.len() as u32);
+        for (g, p) in &self.tokens {
+            put_str(&mut o, g);
+            put_str(&mut o, p);
+        }
+        put_map(&mut o, &self.ngram);
+        put_map(&mut o, &self.backoff);
+        o.extend_from_slice(&self.unk.to_le_bytes());
+        put_u32(&mut o, self.lexicon.len() as u32);
+        for (k, v) in &self.lexicon {
+            put_str(&mut o, k);
+            put_str(&mut o, v);
+        }
+        o
+    }
+
+    pub fn from_bytes(b: &[u8]) -> Model {
+        let mut c = Cur { b, i: 0 };
+        assert_eq!(c.take(4), MAGIC, "bad magic");
+        let order = c.u8();
+        let logo = c.u8() != 0;
+        c.take(2); // pad
+        let nt = c.u32() as usize;
+        let tokens: Vec<Tok> = (0..nt).map(|_| (c.str(), c.str())).collect();
+        let ngram = c.map();
+        let backoff = c.map();
+        let unk = c.f32();
+        let nlex = c.u32() as usize;
+        let mut lexicon = HashMap::with_capacity(nlex);
+        for _ in 0..nlex {
+            lexicon.insert(c.str(), c.str());
+        }
+        let mut m = Model {
+            tokens,
+            order,
+            logo,
+            max_chunk: 0,
+            by_graph: HashMap::new(),
+            ngram,
+            backoff,
+            unk,
+            lexicon,
+        };
+        m.index();
+        m
+    }
+}
+
+// ---- little-endian helpers, no deps ----
+
+fn put_u16(o: &mut Vec<u8>, v: u16) {
+    o.extend_from_slice(&v.to_le_bytes());
+}
+fn put_u32(o: &mut Vec<u8>, v: u32) {
+    o.extend_from_slice(&v.to_le_bytes());
+}
+fn put_str(o: &mut Vec<u8>, s: &str) {
+    put_u16(o, s.len() as u16);
+    o.extend_from_slice(s.as_bytes());
+}
+/// LEB128 unsigned varint — token ids are small, so most take 1-2 bytes.
+fn put_varint(o: &mut Vec<u8>, mut v: u32) {
+    loop {
+        let b = (v & 0x7f) as u8;
+        v >>= 7;
+        if v != 0 {
+            o.push(b | 0x80);
+        } else {
+            o.push(b);
+            break;
+        }
+    }
+}
+fn put_map(o: &mut Vec<u8>, m: &HashMap<Box<[u32]>, f32>) {
+    put_u32(o, m.len() as u32);
+    for (k, v) in m {
+        o.push(k.len() as u8);
+        for &id in k.iter() {
+            put_varint(o, id);
+        }
+        let q = (v * QSCALE).round().clamp(i16::MIN as f32, i16::MAX as f32) as i16;
+        o.extend_from_slice(&q.to_le_bytes());
+    }
+}
+
+struct Cur<'a> {
+    b: &'a [u8],
+    i: usize,
+}
+impl<'a> Cur<'a> {
+    fn take(&mut self, n: usize) -> &'a [u8] {
+        let s = &self.b[self.i..self.i + n];
+        self.i += n;
+        s
+    }
+    fn u8(&mut self) -> u8 {
+        let v = self.b[self.i];
+        self.i += 1;
+        v
+    }
+    fn u16(&mut self) -> u16 {
+        u16::from_le_bytes(self.take(2).try_into().unwrap())
+    }
+    fn u32(&mut self) -> u32 {
+        u32::from_le_bytes(self.take(4).try_into().unwrap())
+    }
+    fn f32(&mut self) -> f32 {
+        f32::from_le_bytes(self.take(4).try_into().unwrap())
+    }
+    fn i16(&mut self) -> i16 {
+        i16::from_le_bytes(self.take(2).try_into().unwrap())
+    }
+    fn varint(&mut self) -> u32 {
+        let mut v = 0u32;
+        let mut s = 0;
+        loop {
+            let b = self.u8();
+            v |= ((b & 0x7f) as u32) << s;
+            if b & 0x80 == 0 {
+                break;
+            }
+            s += 7;
+        }
+        v
+    }
+    fn str(&mut self) -> Box<str> {
+        let n = self.u16() as usize;
+        String::from_utf8(self.take(n).to_vec()).unwrap().into()
+    }
+    fn map(&mut self) -> HashMap<Box<[u32]>, f32> {
+        let n = self.u32() as usize;
+        let mut m = HashMap::with_capacity(n);
+        for _ in 0..n {
+            let l = self.u8() as usize;
+            let k: Box<[u32]> = (0..l).map(|_| self.varint()).collect();
+            m.insert(k, self.i16() as f32 / QSCALE);
+        }
+        m
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn roundtrip_preserves_lexicon_and_logo() {
+        let mut m = Model {
+            tokens: vec![
+                (String::new().into(), String::new().into()),
+                ("a".into(), "ɑ".into()),
+            ],
+            order: 6,
+            logo: true,
+            max_chunk: 0,
+            by_graph: HashMap::new(),
+            ngram: HashMap::from([(Box::from(&[1u32][..]), -1.5f32)]),
+            backoff: HashMap::from([(Box::from(&[][..]), -0.3f32)]),
+            unk: -9.0,
+            lexicon: HashMap::from([("中".into(), "ʈʂʊŋ".into())]),
+        };
+        m.index();
+        let back = Model::from_bytes(&m.to_bytes());
+        assert!(back.logo);
+        assert_eq!(back.order, 6);
+        assert_eq!(back.lexicon.get("中").map(|s| &**s), Some("ʈʂʊŋ"));
+        assert_eq!(back.ngram.get(&[1u32][..]).copied(), Some(-1.5));
+        assert_eq!(back.by_graph.get("a").map(|s| s.len()), Some(1));
+        // quantized logprob round-trips within i16 resolution
+        assert!((back.unk - (-9.0)).abs() < 0.01);
+    }
+
+    #[test]
+    fn logp_backoff_and_unk() {
+        let m = Model {
+            tokens: vec![
+                (String::new().into(), String::new().into()),
+                ("a".into(), "ɑ".into()),
+            ],
+            order: 3,
+            logo: false,
+            max_chunk: 0,
+            by_graph: HashMap::new(),
+            ngram: HashMap::from([(Box::from(&[1u32][..]), -1.5f32)]), // only unigram [1]
+            backoff: HashMap::from([(Box::from(&[2u32][..]), -0.7f32)]),
+            unk: -8.0,
+            lexicon: HashMap::new(),
+        };
+        // exact unigram hit
+        assert!((m.logp(&[], 1) - (-1.5)).abs() < 1e-6);
+        // bigram [2,1] missing -> backoff(hist=[2]) + logp([],1) = -0.7 + -1.5
+        assert!((m.logp(&[2], 1) - (-2.2)).abs() < 1e-6);
+        // unseen unigram -> unk
+        assert!((m.logp(&[], 99) - (-8.0)).abs() < 1e-6);
+        // missing history backoff weight treated as ln(1)=0
+        assert!((m.logp(&[7], 1) - (-1.5)).abs() < 1e-6);
+    }
+
+    #[test]
+    fn varint_multibyte_roundtrip() {
+        // token id > 127 forces a multi-byte LEB128 varint on both encode/decode
+        let mut tokens: Vec<Tok> = (0..300)
+            .map(|i| (i.to_string().into(), format!("p{i}").into()))
+            .collect();
+        tokens[0] = (String::new().into(), String::new().into());
+        let m = Model {
+            tokens,
+            order: 2,
+            logo: false,
+            max_chunk: 0,
+            by_graph: HashMap::new(),
+            ngram: HashMap::from([(Box::from(&[257u32][..]), -2.0f32)]),
+            backoff: HashMap::new(),
+            unk: -7.0,
+            lexicon: HashMap::new(),
+        };
+        let back = Model::from_bytes(&m.to_bytes());
+        assert!((back.ngram.get(&[257u32][..]).copied().unwrap() - (-2.0)).abs() < 0.01);
+    }
+}

g2p2-0.1.0/g2p/src/normalize.rs

@@ -0,0 +1,82 @@
+//! Grapheme splitting + casing. Pure `std`, no Unicode-table crates.
+//!
+//! `graphemes` clusters a base char with its trailing combining marks so that
+//! e.g. a base + combining tilde stays one unit. The combining-mark ranges
+//! below cover Latin/Cyrillic/Greek/Arabic/Hebrew diacritics and common
+//! symbol/Indic combining blocks. It is an approximation of full UAX #29
+//! grapheme clustering — good enough for word-level G2P; extend the ranges if a
+//! target language needs finer clustering.
+
+/// Split a string into grapheme clusters (base char + trailing combining marks).
+pub fn graphemes(s: &str) -> Vec<Box<str>> {
+    let mut out: Vec<Box<str>> = Vec::new();
+    let mut cur = String::new();
+    for ch in s.chars() {
+        if is_combining(ch) && !cur.is_empty() {
+            cur.push(ch); // attach to current base
+        } else {
+            if !cur.is_empty() {
+                out.push(cur.as_str().into());
+                cur.clear();
+            }
+            cur.push(ch);
+        }
+    }
+    if !cur.is_empty() {
+        out.push(cur.as_str().into());
+    }
+    out
+}
+
+/// Unicode-aware lowercasing (`std`). Note: Turkish dotless-i and a few
+/// language-specific casings are not special-cased here.
+pub fn lower(s: &str) -> String {
+    s.to_lowercase()
+}
+
+#[inline]
+fn is_combining(c: char) -> bool {
+    matches!(c as u32,
+        0x0300..=0x036F | // combining diacritical marks
+        0x0483..=0x0489 | // Cyrillic
+        0x0591..=0x05BD | 0x05BF | 0x05C1..=0x05C2 | 0x05C4..=0x05C5 | 0x05C7 | // Hebrew
+        0x0610..=0x061A | 0x064B..=0x065F | 0x0670 | 0x06D6..=0x06DC | 0x06DF..=0x06E4 | // Arabic
+        0x0900..=0x0903 | 0x093A..=0x094F | 0x0951..=0x0957 | // Devanagari (approx)
+        0x1AB0..=0x1AFF | 0x1DC0..=0x1DFF | 0x20D0..=0x20FF | 0xFE20..=0xFE2F)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn ascii_splits_per_char() {
+        let g = graphemes("chat");
+        assert_eq!(g.len(), 4);
+    }
+
+    #[test]
+    fn combining_mark_attaches() {
+        // "a" + U+0303 combining tilde -> one cluster
+        let g = graphemes("a\u{0303}bc");
+        assert_eq!(g.len(), 3);
+        assert_eq!(&*g[0], "a\u{0303}");
+    }
+
+    #[test]
+    fn empty_string_no_clusters() {
+        assert!(graphemes("").is_empty());
+    }
+
+    #[test]
+    fn leading_combining_stands_alone() {
+        // combining mark with no preceding base starts its own cluster
+        let g = graphemes("\u{0303}a");
+        assert_eq!(g.len(), 2);
+    }
+
+    #[test]
+    fn lower_is_unicode_aware() {
+        assert_eq!(lower("ÀÉ"), "àé");
+    }
+}

g2p2-0.1.0/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["maturin>=1.5,<2.0"]
+build-backend = "maturin"
+
+[project]
+name = "g2p2"
+version = "0.1.0"
+description = "Zero-dependency grapheme-to-phoneme for the 100 Whisper languages (Rust core)"
+readme = "README.md"
+requires-python = ">=3.8"
+license = { text = "MIT OR Apache-2.0" }
+authors = [{ name = "jqueguiner" }]
+keywords = ["g2p", "grapheme-to-phoneme", "phonemizer", "ipa", "whisper", "phonetics"]
+classifiers = [
+  "Programming Language :: Rust",
+  "Programming Language :: Python :: 3",
+  "Topic :: Text Processing :: Linguistic",
+  "License :: OSI Approved :: MIT License",
+]
+
+[project.urls]
+Repository = "https://github.com/jqueguiner/g2p2"
+
+[tool.maturin]
+module-name = "g2p2"
+features = ["pyo3/extension-module"]
+manifest-path = "bindings/python/Cargo.toml"