zyphra-textnorm 0.1.0__tar.gz

zyphra_textnorm-0.1.0/Cargo.lock

@@ -0,0 +1,218 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "itoa"
+version = "1.0.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
+
+[[package]]
+name = "libc"
+version = "0.2.191"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "portable-atomic"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.30.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4"
+dependencies = [
+ "libc",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.30.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700"
+dependencies = [
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.30.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.30.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.30.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "ruzstd"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69e4ce4a039f42bf25920b95667e4d789e98749ab11bbb8d2c2936f788421bb5"
+dependencies = [
+ "twox-hash",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.154"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
+[[package]]
+name = "twox-hash"
+version = "2.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
+
+[[package]]
+name = "zmij"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
+
+[[package]]
+name = "zyphra-textnorm"
+version = "0.1.0"
+dependencies = [
+ "ruzstd",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "zyphra-textnorm-py"
+version = "0.1.0"
+dependencies = [
+ "pyo3",
+ "zyphra-textnorm",
+]

zyphra_textnorm-0.1.0/Cargo.toml

@@ -0,0 +1,87 @@
+[package]
+name = "zyphra-textnorm"
+version.workspace = true
+description = "Text normalization for TTS (written form to spoken form) in nine languages: hand-written Rust ports of the NeMo r1.2.0 grammars, no FST runtime"
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+readme = "README.md"
+keywords = ["tts", "text-normalization", "speech", "nemo"]
+categories = ["text-processing", "internationalization"]
+# The oracle, its corpus and goldens (≈ 15 MB) stay in the repository: the crate ships the library, its tables and
+# the tests that need no golden.
+include = [
+    "/src/**",
+    "/data/**",
+    "/include/**",
+    "/examples/**",
+    "/benches/**",
+    "/tests/wrapper.rs",
+    "/README.md",
+    "/LICENSE",
+    "/NOTICE",
+    "/licenses/**",
+]
+
+[workspace]
+members = ["python"]
+
+[workspace.package]
+version = "0.1.0"
+edition = "2024"
+# let chains
+rust-version = "1.88"
+license = "MIT AND Apache-2.0"
+repository = "https://github.com/Zyphra/zyphra-textnorm"
+
+[lib]
+name = "zyphra_textnorm"
+
+[features]
+# One feature per grammar package; a port is built and tested alone with `--no-default-features --features lang-<l>`.
+default = ["lang-en", "lang-de", "lang-es", "lang-fr", "lang-it", "lang-pt", "lang-ja", "lang-zh", "lang-ko"]
+lang-en = []
+lang-de = []
+lang-es = []
+lang-fr = []
+lang-it = []
+lang-pt = []
+lang-ja = []
+lang-zh = []
+lang-ko = []
+# The C ABI (include/zyphra_textnorm.h). Build the static library with
+# `cargo rustc --release --lib --features capi --crate-type staticlib`.
+capi = []
+
+[dependencies]
+
+[dev-dependencies]
+# The goldens are committed zstd-compressed (tests/golden/*.jsonl.zst); pure-Rust decoder.
+ruzstd = "0.9"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+
+[[bench]]
+name = "normalize"
+harness = false
+
+# The differential tests read tests/golden/ (repository only).
+[[test]]
+name = "golden"
+path = "tests/golden.rs"
+
+[[test]]
+name = "en_cardinal"
+path = "tests/en_cardinal.rs"
+
+[profile.release]
+lto = "fat"
+codegen-units = 1
+opt-level = 3
+# unwind, not abort: a grammar panic becomes a grammar failure (the sentence kept as typed). Under panic = "abort" it
+# aborts the process instead.
+panic = "unwind"
+
+[profile.bench]
+inherits = "release"

zyphra_textnorm-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Zyphra, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

zyphra_textnorm-0.1.0/NOTICE

@@ -0,0 +1,32 @@
+zyphra-textnorm
+Copyright (c) 2026 Zyphra, Inc.
+
+The original code of this project is under the MIT License (LICENSE). It includes the third-party components below;
+the full text of each license is in licenses/.
+
+================================================================================
+1. NVIDIA NeMo-text-processing
+   https://github.com/NVIDIA/NeMo-text-processing
+   Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES
+   Portions Copyright 2015 and onwards Google, Inc.
+   SPDX-License-Identifier: Apache-2.0
+   Full license: licenses/nemo-Apache-2.0.txt
+
+   The Rust grammars in src/lang/ are a port of the r1.2.0 text_normalization
+   grammars, and the tables in data/ (except data/perluniprops/ and
+   data/unicode/) are copies of their data files. The r1.2.0 package itself is
+   vendored under oracle/nemo_text_processing/ (the differential-test oracle;
+   not part of the library or its packages), with its copyright and Apache-2.0
+   headers retained in each file. oracle/upstream_cases/ holds the r1.2.0 test
+   cases.
+
+================================================================================
+2. sacremoses
+   https://github.com/hplt-project/sacremoses
+   Copyright (c) 2020 alvations
+   SPDX-License-Identifier: MIT
+   Full license: licenses/sacremoses-MIT.txt
+
+   MosesDetokenizer and the perluniprops IsAlpha/IsSc tables are ported in
+   src/moses.rs and data/perluniprops/.
+================================================================================

zyphra_textnorm-0.1.0/PKG-INFO

@@ -0,0 +1,156 @@
+Metadata-Version: 2.4
+Name: zyphra-textnorm
+Version: 0.1.0
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Topic :: Text Processing :: Linguistic
+License-File: LICENSE
+License-File: NOTICE
+License-File: licenses/nemo-Apache-2.0.txt
+License-File: licenses/sacremoses-MIT.txt
+Summary: Text normalization for TTS (written form to spoken form) in nine languages: Rust ports of the NeMo r1.2.0 grammars
+Keywords: tts,text-normalization,speech,nemo
+License-Expression: MIT AND Apache-2.0
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Repository, https://github.com/Zyphra/zyphra-textnorm
+
+# zyphra-textnorm
+
+Text normalization for text-to-speech: written text to its spoken form ("$5.32" → "five dollars thirty two cents"),
+for English, German, Spanish, French, Italian, Portuguese, Japanese, Mandarin Chinese and Korean. These are
+hand-written Rust ports of the [NeMo-text-processing](https://github.com/NVIDIA/NeMo-text-processing) r1.2.0
+grammars. There is no FST runtime and no pynini: every grammar is Rust code, every table is embedded at compile
+time, and nothing is loaded at runtime. One library serves Rust, Python and C, with a 1024-entry memo per normalizer.
+
+Every grammar call and every end-to-end output equals the NeMo pipeline on a differential corpus of 92,953 lines
+(see [Parity](#parity)), at 13-20 µs per 100 characters.
+
+## Use
+
+```rust
+use zyphra_textnorm::Normalizer;
+
+assert_eq!(zyphra_textnorm::normalize("It costs $5.32.", "en"), "It costs five dollars thirty two cents.");
+let tn = Normalizer::with_markup_tags(["laugh", "shouting"]);
+assert_eq!(tn.normalize("So I said <laugh/> no way, $5.", "en"), "So I said <laugh/> no way, five dollars.");
+```
+
+```python
+import zyphra_textnorm
+
+zyphra_textnorm.normalize("Am 3. Oktober kostete es 5,32 €.", "de")
+# 'Am dritter oktober kostete es fünf komma drei zwei €.'
+tn = zyphra_textnorm.Normalizer(markup_tags=["laugh"])
+tn.normalize("So I said <laugh/> no way, $5.", "en")
+```
+
+```c
+#include "zyphra_textnorm.h"
+
+zyphra_textnorm *tn = zyphra_textnorm_new("laugh\nshouting", 14);
+char *out = zyphra_textnorm_normalize(tn, text, text_len, "en", 2);
+/* ... */
+zyphra_textnorm_free(out);
+zyphra_textnorm_delete(tn);
+```
+
+- **Languages**: `LANGUAGES` = `en de es fr it pt ja zh ko` (NeMo's names; `zh` is Mandarin). `supported(lang)` says
+  whether this build has a grammar. One cargo feature per language (`lang-en` …), all on by default.
+- **Never fails**: a language without a grammar comes back with its line breaks collapsed, and a sentence the
+  grammar fails on is kept as typed (the rest of the text is still normalized).
+- **Line breaks**: every whitespace run holding a line break, tab or other non-space whitespace becomes one space
+  first (`collapse_line_breaks`); runs of plain spaces are kept.
+- **Markup** (`Normalizer::with_markup_tags`): the markup of the given tag names stays as typed: point tags
+  (`<laugh/>`, `<laugh>`) and span tags (`<shouting>…</shouting>`, `\s*` allowed before `/` or `>`), and the text
+  between markers is normalized piece by piece. Without tags, `<…>` is text like any other.
+- **Verbatim spans**: anything between U+E000 and U+E001 (e.g. a pronunciation) is kept as typed.
+- **Memo**: each `Normalizer` remembers its last 1024 results, shared by every thread that calls it;
+  `normalize(text, lang)` uses one shared normalizer without markup tags.
+
+The output is the spoken form the NeMo grammars produce, quirks included: they are ported as they are, and parity is
+the contract. On top of the grammars come the fixes of the wrapper the goldens were dumped with
+(`oracle/reference_textnorm.py`, the wrapper of [ZONOS3](https://github.com/Zyphra/ZONOS3)): sentence punctuation
+after a digit is spaced off where it breaks a grammar (pt, de, es, fr, it, ko, en), ja drops the ASCII space between
+CJK characters, and en reads `24/7` and `$20k` as speech.
+
+## Install
+
+| | |
+| --- | --- |
+| Rust | `cargo add zyphra-textnorm` (MSRV 1.88) |
+| Python (≥ 3.9) | `pip install zyphra-textnorm`: abi3 wheels for Linux (manylinux, x86_64 and aarch64), macOS (x86_64, arm64) and Windows (x64, arm64); elsewhere pip builds the sdist, which needs cargo |
+| C / C++ | `cargo rustc --release --lib --features capi --crate-type staticlib` → `target/release/libzyphra_textnorm.a` (`zyphra_textnorm.lib` on MSVC), header `include/zyphra_textnorm.h`. Link the platform's Rust std libraries (`-lpthread -ldl -lm` on Linux, `ws2_32 userenv ntdll bcrypt` on MSVC). |
+
+The release profile keeps `panic = "unwind"`: a panic inside a grammar is a grammar failure (the sentence kept as
+typed). A build with `panic = "abort"` aborts on one instead.
+
+## Layout
+
+| Path | Contents |
+| --- | --- |
+| `src/` | The library: `lib.rs` (API), `wrapper.rs` (line breaks, markup, verbatim spans, per-language passes, sentence retry, memo), `pipeline.rs` (NeMo's `Normalizer.normalize`), `lattice.rs` (the tagger's Viterbi), `tag.rs`, `permute.rs`, `moses.rs`, `punct_post.rs`, `ffi.rs` (C ABI) |
+| `src/lang/<l>/` | One grammar package: `taggers/` and `verbalizers/` mirror the upstream files one to one |
+| `data/` | The upstream `.tsv` tables verbatim, the sacremoses perluniprops tables, frozen Unicode 15.0 tables |
+| `include/` | The C header |
+| `python/` | The Python extension `zyphra_textnorm` (PyO3; `pyproject.toml` at the root) |
+| `oracle/` | The differential-test oracle: the vendored NeMo r1.2.0 package, the frozen reference wrapper, the Docker image, corpus and golden dump scripts |
+| `tests/` | Differential tests; the corpus (`corpus/`), the oracle's outputs (`golden/*.jsonl.zst`) and the oracle-confirmed equal-cost ties the port resolves differently (`ties/`) |
+| `PORTING.md` | How a NeMo grammar becomes Rust, with the verified pynini semantics |
+
+## Develop
+
+```sh
+cargo test --release --features capi       # unit tests + differential tests against the goldens
+cargo run --release --example normalize -- en 'Save $20k.'
+cargo bench                                # median µs per call and per 100 chars, allocations
+cargo bench --no-default-features --features lang-de   # one language, faster to build
+uv venv && uv pip install -e . && .venv/bin/python -c 'import zyphra_textnorm'
+```
+
+The bench warms every case before timing it and reports the median of 101 calls on a cleared memo. Run it at load
+below 4: medians swing by 1.5× on a busy machine. The target is ≤ 20 µs per 100 characters on every language's line.
+
+## Release
+
+The crate and the Python package share the version (`[workspace.package]` in `Cargo.toml`). Pushing a `v<version>`
+tag runs `.github/workflows/release.yml`, which builds the wheels and the sdist and installs and runs each wheel on
+its platform. Then:
+
+```sh
+cargo publish -p zyphra-textnorm                                  # crates.io
+gh run download <run id> -D dist && uv publish dist/*/*           # PyPI
+```
+
+## Parity
+
+The goldens in `tests/golden/` are the outputs of the NeMo r1.2.0 pipeline (pynini) behind the reference wrapper
+(`oracle/reference_textnorm.py`), committed `zstd -19`-compressed (the tests read the `.zst` directly; the plain
+`.jsonl` the dump scripts write are gitignored). The corpus is generated from every table of each language
+(`oracle/gen_corpus.py`) plus the upstream r1.2.0 test cases and handcrafted lines. The `calls` test checks every
+NeMo grammar call the wrapper made, the `golden` test every end-to-end row. The only exceptions are 735 calls where
+the oracle picks among paths of equal float32 cost by its compiled arc order (`tests/ties/`, PORTING.md "Ties");
+a listed call that no longer differs fails as stale. The English number-name FARs, which have no source, are checked
+against a 457k-row dump of the FSTs (`tests/en_cardinal.rs`).
+
+pynini has no macOS wheel, so the oracle runs in a linux/amd64 container (Rosetta on Apple silicon):
+
+```sh
+colima start --vz-rosetta --cpu 4 --memory 8
+docker build --platform linux/amd64 -f oracle/Dockerfile -t zyphra-textnorm-oracle .
+docker run --rm -v "$PWD":/repo -v "$HOME/.cache/zyphra-textnorm-oracle":/cache -w /repo zyphra-textnorm-oracle \
+    python oracle/dump_golden.py --cache /cache --lang en --corpus tests/corpus/en.jsonl --out tests/golden/en.jsonl
+zstd -19 -f tests/golden/en.jsonl -o tests/golden/en.jsonl.zst
+docker run --rm -v "$PWD":/repo -v "$HOME/.cache/zyphra-textnorm-oracle":/cache -w /repo zyphra-textnorm-oracle \
+    python oracle/triage.py --lang en --text 'Call me at 123 Main Street, please.'
+```
+
+`zstd -d -k tests/golden/<l>.jsonl.zst` restores the plain file for inspection. `oracle/gen_corpus.py` (host Python)
+regenerates `tests/corpus/`; `oracle/gen_unicode_tables.py` the Unicode tables (must run on CPython 3.12 / Unicode
+15.0, like the oracle).
+
+## License
+
+MIT (`LICENSE`) for the original code; the grammars and tables ported from NeMo-text-processing are Apache-2.0, and
+the Moses detokenizer port is MIT (sacremoses). See `NOTICE` and `licenses/`.
+

zyphra_textnorm-0.1.0/README.md

@@ -0,0 +1,138 @@
+# zyphra-textnorm
+
+Text normalization for text-to-speech: written text to its spoken form ("$5.32" → "five dollars thirty two cents"),
+for English, German, Spanish, French, Italian, Portuguese, Japanese, Mandarin Chinese and Korean. These are
+hand-written Rust ports of the [NeMo-text-processing](https://github.com/NVIDIA/NeMo-text-processing) r1.2.0
+grammars. There is no FST runtime and no pynini: every grammar is Rust code, every table is embedded at compile
+time, and nothing is loaded at runtime. One library serves Rust, Python and C, with a 1024-entry memo per normalizer.
+
+Every grammar call and every end-to-end output equals the NeMo pipeline on a differential corpus of 92,953 lines
+(see [Parity](#parity)), at 13-20 µs per 100 characters.
+
+## Use
+
+```rust
+use zyphra_textnorm::Normalizer;
+
+assert_eq!(zyphra_textnorm::normalize("It costs $5.32.", "en"), "It costs five dollars thirty two cents.");
+let tn = Normalizer::with_markup_tags(["laugh", "shouting"]);
+assert_eq!(tn.normalize("So I said <laugh/> no way, $5.", "en"), "So I said <laugh/> no way, five dollars.");
+```
+
+```python
+import zyphra_textnorm
+
+zyphra_textnorm.normalize("Am 3. Oktober kostete es 5,32 €.", "de")
+# 'Am dritter oktober kostete es fünf komma drei zwei €.'
+tn = zyphra_textnorm.Normalizer(markup_tags=["laugh"])
+tn.normalize("So I said <laugh/> no way, $5.", "en")
+```
+
+```c
+#include "zyphra_textnorm.h"
+
+zyphra_textnorm *tn = zyphra_textnorm_new("laugh\nshouting", 14);
+char *out = zyphra_textnorm_normalize(tn, text, text_len, "en", 2);
+/* ... */
+zyphra_textnorm_free(out);
+zyphra_textnorm_delete(tn);
+```
+
+- **Languages**: `LANGUAGES` = `en de es fr it pt ja zh ko` (NeMo's names; `zh` is Mandarin). `supported(lang)` says
+  whether this build has a grammar. One cargo feature per language (`lang-en` …), all on by default.
+- **Never fails**: a language without a grammar comes back with its line breaks collapsed, and a sentence the
+  grammar fails on is kept as typed (the rest of the text is still normalized).
+- **Line breaks**: every whitespace run holding a line break, tab or other non-space whitespace becomes one space
+  first (`collapse_line_breaks`); runs of plain spaces are kept.
+- **Markup** (`Normalizer::with_markup_tags`): the markup of the given tag names stays as typed: point tags
+  (`<laugh/>`, `<laugh>`) and span tags (`<shouting>…</shouting>`, `\s*` allowed before `/` or `>`), and the text
+  between markers is normalized piece by piece. Without tags, `<…>` is text like any other.
+- **Verbatim spans**: anything between U+E000 and U+E001 (e.g. a pronunciation) is kept as typed.
+- **Memo**: each `Normalizer` remembers its last 1024 results, shared by every thread that calls it;
+  `normalize(text, lang)` uses one shared normalizer without markup tags.
+
+The output is the spoken form the NeMo grammars produce, quirks included: they are ported as they are, and parity is
+the contract. On top of the grammars come the fixes of the wrapper the goldens were dumped with
+(`oracle/reference_textnorm.py`, the wrapper of [ZONOS3](https://github.com/Zyphra/ZONOS3)): sentence punctuation
+after a digit is spaced off where it breaks a grammar (pt, de, es, fr, it, ko, en), ja drops the ASCII space between
+CJK characters, and en reads `24/7` and `$20k` as speech.
+
+## Install
+
+| | |
+| --- | --- |
+| Rust | `cargo add zyphra-textnorm` (MSRV 1.88) |
+| Python (≥ 3.9) | `pip install zyphra-textnorm`: abi3 wheels for Linux (manylinux, x86_64 and aarch64), macOS (x86_64, arm64) and Windows (x64, arm64); elsewhere pip builds the sdist, which needs cargo |
+| C / C++ | `cargo rustc --release --lib --features capi --crate-type staticlib` → `target/release/libzyphra_textnorm.a` (`zyphra_textnorm.lib` on MSVC), header `include/zyphra_textnorm.h`. Link the platform's Rust std libraries (`-lpthread -ldl -lm` on Linux, `ws2_32 userenv ntdll bcrypt` on MSVC). |
+
+The release profile keeps `panic = "unwind"`: a panic inside a grammar is a grammar failure (the sentence kept as
+typed). A build with `panic = "abort"` aborts on one instead.
+
+## Layout
+
+| Path | Contents |
+| --- | --- |
+| `src/` | The library: `lib.rs` (API), `wrapper.rs` (line breaks, markup, verbatim spans, per-language passes, sentence retry, memo), `pipeline.rs` (NeMo's `Normalizer.normalize`), `lattice.rs` (the tagger's Viterbi), `tag.rs`, `permute.rs`, `moses.rs`, `punct_post.rs`, `ffi.rs` (C ABI) |
+| `src/lang/<l>/` | One grammar package: `taggers/` and `verbalizers/` mirror the upstream files one to one |
+| `data/` | The upstream `.tsv` tables verbatim, the sacremoses perluniprops tables, frozen Unicode 15.0 tables |
+| `include/` | The C header |
+| `python/` | The Python extension `zyphra_textnorm` (PyO3; `pyproject.toml` at the root) |
+| `oracle/` | The differential-test oracle: the vendored NeMo r1.2.0 package, the frozen reference wrapper, the Docker image, corpus and golden dump scripts |
+| `tests/` | Differential tests; the corpus (`corpus/`), the oracle's outputs (`golden/*.jsonl.zst`) and the oracle-confirmed equal-cost ties the port resolves differently (`ties/`) |
+| `PORTING.md` | How a NeMo grammar becomes Rust, with the verified pynini semantics |
+
+## Develop
+
+```sh
+cargo test --release --features capi       # unit tests + differential tests against the goldens
+cargo run --release --example normalize -- en 'Save $20k.'
+cargo bench                                # median µs per call and per 100 chars, allocations
+cargo bench --no-default-features --features lang-de   # one language, faster to build
+uv venv && uv pip install -e . && .venv/bin/python -c 'import zyphra_textnorm'
+```
+
+The bench warms every case before timing it and reports the median of 101 calls on a cleared memo. Run it at load
+below 4: medians swing by 1.5× on a busy machine. The target is ≤ 20 µs per 100 characters on every language's line.
+
+## Release
+
+The crate and the Python package share the version (`[workspace.package]` in `Cargo.toml`). Pushing a `v<version>`
+tag runs `.github/workflows/release.yml`, which builds the wheels and the sdist and installs and runs each wheel on
+its platform. Then:
+
+```sh
+cargo publish -p zyphra-textnorm                                  # crates.io
+gh run download <run id> -D dist && uv publish dist/*/*           # PyPI
+```
+
+## Parity
+
+The goldens in `tests/golden/` are the outputs of the NeMo r1.2.0 pipeline (pynini) behind the reference wrapper
+(`oracle/reference_textnorm.py`), committed `zstd -19`-compressed (the tests read the `.zst` directly; the plain
+`.jsonl` the dump scripts write are gitignored). The corpus is generated from every table of each language
+(`oracle/gen_corpus.py`) plus the upstream r1.2.0 test cases and handcrafted lines. The `calls` test checks every
+NeMo grammar call the wrapper made, the `golden` test every end-to-end row. The only exceptions are 735 calls where
+the oracle picks among paths of equal float32 cost by its compiled arc order (`tests/ties/`, PORTING.md "Ties");
+a listed call that no longer differs fails as stale. The English number-name FARs, which have no source, are checked
+against a 457k-row dump of the FSTs (`tests/en_cardinal.rs`).
+
+pynini has no macOS wheel, so the oracle runs in a linux/amd64 container (Rosetta on Apple silicon):
+
+```sh
+colima start --vz-rosetta --cpu 4 --memory 8
+docker build --platform linux/amd64 -f oracle/Dockerfile -t zyphra-textnorm-oracle .
+docker run --rm -v "$PWD":/repo -v "$HOME/.cache/zyphra-textnorm-oracle":/cache -w /repo zyphra-textnorm-oracle \
+    python oracle/dump_golden.py --cache /cache --lang en --corpus tests/corpus/en.jsonl --out tests/golden/en.jsonl
+zstd -19 -f tests/golden/en.jsonl -o tests/golden/en.jsonl.zst
+docker run --rm -v "$PWD":/repo -v "$HOME/.cache/zyphra-textnorm-oracle":/cache -w /repo zyphra-textnorm-oracle \
+    python oracle/triage.py --lang en --text 'Call me at 123 Main Street, please.'
+```
+
+`zstd -d -k tests/golden/<l>.jsonl.zst` restores the plain file for inspection. `oracle/gen_corpus.py` (host Python)
+regenerates `tests/corpus/`; `oracle/gen_unicode_tables.py` the Unicode tables (must run on CPython 3.12 / Unicode
+15.0, like the oracle).
+
+## License
+
+MIT (`LICENSE`) for the original code; the grammars and tables ported from NeMo-text-processing are Apache-2.0, and
+the Moses detokenizer port is MIT (sacremoses). See `NOTICE` and `licenses/`.