afrispeech-synth 0.1.0__tar.gz

afrispeech_synth-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,78 @@
+# Publish to PyPI when a version tag is pushed.
+#
+# Uses PyPI Trusted Publishing (OIDC), so there is no API token in this repository's secrets.
+# GitHub mints a short-lived identity token that PyPI verifies against a publisher configured at
+# https://pypi.org/manage/account/publishing/ with owner AfriSpeech, repo afrispeech-synth,
+# workflow release.yml, environment pypi. Because the project does not exist on PyPI yet, that
+# has to be added as a *pending* publisher — a normal one can only be attached to a project that
+# already exists. The first successful run creates the project and converts it automatically.
+#
+# Tag and version must agree. A tag of v0.2.0 with `version = "0.1.0"` in pyproject.toml would
+# upload the wrong release and PyPI does not allow re-uploading a version, so the check below
+# fails the job before the build rather than after the upload.
+name: release
+
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:          # lets you dry-run the build without tagging
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Tag matches pyproject version
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          tag="${GITHUB_REF_NAME#v}"
+          ver=$(python -c "import tomllib,pathlib; \
+            print(tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project']['version'])")
+          echo "tag=$tag pyproject=$ver"
+          [ "$tag" = "$ver" ] || { echo "::error::tag $tag != version $ver"; exit 1; }
+
+      - run: pip install build twine
+      - run: python -m build
+
+      # `twine check` catches a malformed README before upload. PyPI renders the long
+      # description itself and rejects the whole upload if it cannot.
+      - run: twine check dist/*
+
+      - name: Import the built wheel
+        run: |
+          pip install dist/*.whl
+          python -c "import afrispeech_synth; print(afrispeech_synth.__version__)"
+          afrispeech-synth --help > /dev/null
+          afrispeech-synth voices > /dev/null
+
+      # Run the suite against the *installed wheel*, from outside the source tree: the
+      # package data and entry points are the things most likely to be missing from a
+      # build, and a test run from the repo root would find the source on disk whether
+      # or not it was packaged.
+      - name: Test the installed wheel
+        run: |
+          pip install pytest
+          cd /tmp && python -m pytest "$GITHUB_WORKSPACE/tests" -q
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    environment: pypi          # must match the environment name set on PyPI's publisher form
+    permissions:
+      id-token: write          # required for OIDC; without it the upload cannot authenticate
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

afrispeech_synth-0.1.0/.gitignore

@@ -0,0 +1,17 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+.venv/
+venv/
+.env
+work/
+out/
+corpora/
+data/
+space/
+*.wav
+*.mp3
+*.parquet
+.pytest_cache/

afrispeech_synth-0.1.0/LICENSE

@@ -0,0 +1,28 @@
+MIT License
+
+Copyright (c) 2026 AfriSpeech
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+---
+
+The MIT license above covers the afriso source code.
+
+The bundled datasets (src/afriso/data/) are derived from third-party sources
+and remain subject to their own licenses — see DATA_LICENSE.md.

afrispeech_synth-0.1.0/PKG-INFO

@@ -0,0 +1,405 @@
+Metadata-Version: 2.5
+Name: afrispeech-synth
+Version: 0.1.0
+Summary: Build synthetic speech datasets for African languages with Google Gemini TTS: corpus text, phoneme-coverage sentence selection, africa-g2p orthography normalisation, and a training-ready HuggingFace dataset.
+Project-URL: Homepage, https://github.com/AfriSpeech/afrispeech-synth
+Project-URL: Issues, https://github.com/AfriSpeech/afrispeech-synth/issues
+Author: AfriSpeech
+License: MIT
+License-File: LICENSE
+Keywords: african-languages,asr,dataset,g2p,gemini,speech-synthesis,tts
+Requires-Python: >=3.10
+Requires-Dist: africa-g2p>=0.2.3
+Requires-Dist: huggingface-hub>=0.20
+Requires-Dist: pyarrow>=12.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Provides-Extra: gemini
+Requires-Dist: google-genai>=0.3.0; extra == 'gemini'
+Provides-Extra: hf
+Requires-Dist: datasets>=2.14; extra == 'hf'
+Description-Content-Type: text/markdown
+
+# afrispeech-synth
+
+**Turn [Google Gemini TTS](https://ai.google.dev/gemini-api/docs/speech-generation) into a
+speech-dataset factory for any African language — from raw text to a training-ready
+HuggingFace dataset, with one command.**
+
+Most African languages have no recorded speech corpus. Gemini TTS can speak many of them
+well enough to bootstrap one, but a usable dataset is not just API calls: you need text in
+the language, the *right* sentences rather than all of them, orthography the model reads
+correctly, and output packaged so a trainer can consume it. This does all four around Gemini.
+
+```bash
+pip install "afrispeech-synth[gemini]"
+export GEMINI_API_KEY=...        # https://aistudio.google.com/apikey
+
+afrispeech-synth run examples/twi.yaml
+```
+
+**[Hear what it produces](https://huggingface.co/spaces/AfriSpeech/afrispeech-synth-samples)** —
+one sample per language, 215 languages, a different voice each.
+
+## What's around Gemini
+
+| | |
+|---|---|
+| **The voice** | **[Google Gemini TTS](https://ai.google.dev/gemini-api/docs/speech-generation)** — `gemini-3.1-flash-tts-preview`, [30 voices](#4--synthesise), a paid Google API you bring your own key to. Every dataset built with this tool so far was spoken by it. |
+| **The text** | [africa-corpus-builder](https://github.com/AfriSpeech/africa-corpus-builder) — source text for **693 African languages**, so a language with no corpus of its own still has a starting point |
+| **The orthography** | [africa-g2p](https://github.com/AfriSpeech/africa-g2p) — phoneme tables for **400 languages**. Feeding Gemini a language's raw orthography gets you its guess at `ɔ`, `ɛ` and `ŋ`; feeding it the universal grapheme set gets you the sound |
+| **The names** | [afriso](https://github.com/AfriSpeech/afriso) — resolves `Twi`, `tw`, `aka`, `Asante Twi` to one code all of the above agree on |
+| **The selection** | Greedy set cover over phoneme units — on Twi, 4,141 candidate sentences reduced to 115 at full phoneme coverage. 97% fewer Gemini calls for the same coverage |
+
+Gemini is the reference backend, not a dependency of the design — [swapping it
+out](#adding-a-tts-backend) is one method and one line of config — but it is what this was
+built on and tuned against, and the parts above exist because raw API calls alone did not
+produce a dataset worth training on.
+
+For **recorded** African speech rather than synthetic, use
+[afrispeech-selector](https://github.com/AfriSpeech/afrispeech-selector).
+
+---
+
+## Which languages work
+
+| | Languages | What you need to do |
+|---|--:|---|
+| **Ready** | **215** | Nothing. Name the language and run. |
+| **Bring your own text** | 185 | Point a `file:` or `hf:` source at your own sentences. |
+| **No G2P table** | 478 | Text is available; run with `--normalise none`. |
+
+africa-g2p has phoneme tables for **400** African languages, africa-corpus-builder has text
+for **693**, and **215 are in both** — those need nothing from you but a name:
+
+```bash
+afrispeech-synth run --lang Zulu --source corpus:zul --out out/zul
+```
+
+**If your language has no corpus text, it still works — supply your own:**
+
+```bash
+afrispeech-synth run --lang Afar --source file:my_afar_sentences.txt --out out/aar
+afrispeech-synth run --lang Afar --source hf:my-org/my-dataset#text  --out out/aar
+```
+
+Any source works for any language, and you can mix them. If there's no G2P table either, add
+`--normalise none` and the TTS model gets your raw text — everything else in the pipeline is
+unchanged.
+
+Check where yours stands:
+
+```bash
+afrispeech-synth langs --search zulu     # zul  Zulu  ready  Atlantic-Congo
+afrispeech-synth langs --ready           # the 215 that need nothing from you
+```
+
+Languages are matched by exact code only. Matching by name would add ~73 more, but it pairs
+different languages that share an alternative name — Tunisian Arabic text under an Algerian
+Arabic table, Basa of Cameroon under Basa of Nigeria — so those are left out rather than
+shipped wrong.
+
+---
+
+## The pipeline
+
+```
+ source ──► select ──► normalise ──► synthesise ──► package
+  text      phoneme      africa-g2p      TTS API     parquet + card
+   │         cover          │              │            │
+   └ corpus:twi             └ grapheme     └ resumable  └ push to the Hub
+     hf:org/ds#col            or IPA         + retries
+     file:x.txt
+```
+
+**1 · Source.** Combine any number of text sources; duplicates are dropped in first-seen order.
+
+```yaml
+sources:
+  - corpus:twi                                               # africa-corpus-builder
+  - hf:ghanaopenai/Ghana_English-Twi_Code-switching_Speech#transcript
+  - file:my_sentences.txt
+```
+
+**2 · Select.** Greedy set cover over **phoneme** units (default) or **word** units: the fewest
+sentences that still contain every sound. Every sentence dropped is a TTS call you don't pay for —
+on Twi it cut a 4,141-sentence pool to 115 at full phoneme coverage.
+
+```
+  covered 208/208 phoneme units (100.0%) with 115 sentences
+```
+
+**3 · Normalise.** `africa-g2p` rewrites each sentence, then punctuation is reduced to `.` `?`
+`!` `,` — the marks a voice uses for phrasing. Everything else (apostrophes, asterisks marking
+proper nouns, hyphens, colons, quotes) is read as a pause or spelled out, so it goes. Stored as
+`normalised_text` — it's what the TTS model is actually asked to speak.
+
+| `--normalise` | Twi example | When |
+|---|---|---|
+| `universal` *(default)* | `ho bobea onyankopon` | Every phoneme written with the letter most African languages use for it (`ɔ`→`o`, `ɛ`→`e`). Plain `a-z` only — needs africa-g2p ≥ 0.2.3. |
+| `grapheme` | `hɔ bɔbea onyankopɔn` | The language's own phoneme units, multigraphs (`ny`, `kp`) kept whole and special characters preserved. |
+| `ipa` | `hɔ bɔbea oɲankʰopʰɔn` | Phonetic symbols — for phoneme-level ASR work, not for speech generation. |
+| `none` | `hɔ bɔbea Onyankopɔn` | Send the raw text. Works for any language, G2P table or not. |
+
+**Whichever you pick, look at the output before a full run.** Neither transform is right for every
+language:
+
+```
+           universal                    grapheme
+twi        na ho bobea            ✓     na hɔ bɔbea           ɔ/ɛ may be mispronounced
+yor        ngi˥ i˩bɛ˩rɛ˩          ✗     ní ìbẹ̀rẹ̀              ✓
+afr        eng die aarde khemaak  ✗     en die aarde gemaak   ✓
+```
+
+Universal is what the Ghana Twi dataset was built with and is the default, but it rewrites more
+than it should in some languages. `--dry-run` prints the selection without spending any API
+calls; `afrispeech-synth card config.yaml` shows the transform that will be recorded:
+
+```bash
+afrispeech-synth run --lang yor --source corpus:yor --normalise grapheme --dry-run
+```
+
+**4 · Synthesise.** Google Gemini TTS speaks the normalised transcript, in any of its
+**30 voices**:
+
+```bash
+afrispeech-synth voices            # Zephyr Bright, Kore Firm, Sulafat Warm, …
+--voices Zephyr                       # one speaker
+--voices Zephyr,Kore,Sulafat          # rotated across utterances, so the dataset has three
+```
+
+Async, rate-limited, and **resumable**: every finished clip writes its own
+audio file plus a sidecar record, so an interrupted run restarts where it stopped. Retries back
+off on 429s and empty responses.
+
+**5 · Package.** Parquet shards with audio bytes embedded (the viewer plays them inline), a
+`metadata.jsonl` manifest, an LJSpeech export for Piper/VITS/MeloTTS, and a dataset card
+generated from the run config.
+
+---
+
+## Install
+
+```bash
+pip install afrispeech-synth            # core
+pip install "afrispeech-synth[gemini]"  # + the Gemini TTS backend
+pip install "afrispeech-synth[hf]"      # + hf: text sources (pulls datasets)
+```
+
+`afriso` and `africa-corpus-builder` are not on PyPI yet:
+
+```bash
+pip install "afriso @ git+https://github.com/AfriSpeech/afriso"
+
+git clone https://github.com/AfriSpeech/africa-corpus-builder
+export AFRICA_CORPUS_PATH=$PWD/africa-corpus-builder
+```
+
+Both are optional. Without `afriso` you pass codes rather than names; without
+africa-corpus-builder every source except `corpus:` still works.
+
+---
+
+## Usage
+
+### Start from nothing
+
+```bash
+afrispeech-synth init Yoruba          # writes yor.yaml
+afrispeech-synth run yor.yaml --dry-run   # select sentences, call no API
+afrispeech-synth run yor.yaml
+```
+
+### Or stay on the command line
+
+```bash
+afrispeech-synth run \
+  --lang Twi \
+  --source corpus:twi \
+  --cover phoneme \
+  --max-sentences 2000 \
+  --voices Zephyr,Puck \
+  --out out/twi \
+  --repo AfriSpeech/twi-synthetic-speech
+```
+
+### Samples gallery
+
+**[Hear it: AfriSpeech/afrispeech-synth-samples](https://huggingface.co/spaces/AfriSpeech/afrispeech-synth-samples)**
+— one clip per language, each in a different voice. The page is built from this repo, so it
+has no repo of its own:
+
+```bash
+afrispeech-synth samples --limit 20          # generate clips into space/
+afrispeech-synth space                       # build space/index.html, preview locally
+afrispeech-synth space --repo org/my-samples # publish it
+```
+
+`samples` covers every ready language by default and spreads the 30 voices evenly across them,
+so the gallery is also the voice catalogue. Clips are compressed to MP3 if `ffmpeg` is on PATH.
+
+### One stage at a time
+
+```bash
+afrispeech-synth select  config.yaml   # source + cover, writes sentences.txt
+afrispeech-synth synth   config.yaml   # synthesise (resumes by default)
+afrispeech-synth package config.yaml   # parquet + manifest + card
+afrispeech-synth push    config.yaml --repo org/name
+
+afrispeech-synth langs --ready         # languages that need no text from you
+afrispeech-synth langs --search yor    # what one language needs
+afrispeech-synth voices                # the 30 voices you can pick from
+```
+
+Interrupted? Run the same command again — finished clips are skipped.
+
+### As a library
+
+```python
+from afrispeech_synth import RunConfig, run
+
+config = RunConfig(language="Twi", sources=["corpus:twi"], out="out/twi")
+config.select.cover = "phoneme"
+config.select.max_sentences = 2000
+config.tts.voices = ["Zephyr", "Puck"]
+
+run(config)
+```
+
+Individual stages are importable too:
+
+```python
+from afrispeech_synth import resolve, Normaliser, stage_sources, stage_select
+
+language   = resolve("Twi")
+normaliser = Normaliser(language, "grapheme")
+sentences  = stage_sources(config, language)
+selection  = stage_select(config, language, sentences, normaliser)
+
+print(selection.coverage, len(selection.sentences))
+```
+
+---
+
+## Configuration
+
+```yaml
+language: Twi                 # name, ISO 639-1/2/3 code, or alternative name
+sources:                      # corpus: | hf: | file:
+  - corpus:twi
+normalise: universal          # universal | grapheme | ipa | none
+out: out/twi
+
+select:
+  cover: phoneme              # phoneme | word | none
+  min_freq: 1                 # only target units seen at least this often
+  max_sentences: 12000
+  min_chars: 20
+  max_chars: 240
+  seed: 0
+
+tts:
+  backend: gemini
+  model: gemini-3.1-flash-tts-preview
+  voices: [Zephyr]            # round-robined across utterances
+  context: speak in {language} accent
+  concurrency: 10
+  rpm: 200                    # requests per minute, enforced
+  max_retries: 5
+  sample_rate: 24000
+  api_key_env: GEMINI_API_KEY # the key is read from the environment, never the file
+
+package:
+  formats: [parquet, ljspeech]
+  shard_target_mb: 190
+  push_to: AfriSpeech/twi-synthetic-speech
+  private: false
+```
+
+CLI flags override any field. API keys are read only from the environment — never
+put one in a config file you intend to commit.
+
+---
+
+## Output
+
+```
+out/twi/
+├── data/train-00000-of-00019.parquet   # audio bytes embedded, viewer-playable
+├── metadata.jsonl                      # index, text, normalised_text, voice, shard
+├── sentences.txt                       # the selected source sentences
+├── README.md                           # generated dataset card
+└── work/                               # per-clip audio + sidecars (resume state)
+```
+
+```python
+from datasets import load_dataset
+ds = load_dataset("parquet", data_files="out/twi/data/*.parquet", split="train")
+ds[0]["audio"]["array"], ds[0]["text"], ds[0]["normalised_text"]
+```
+
+---
+
+## Adding a TTS backend
+
+A backend is one method. Register it and it becomes available as `tts.backend`:
+
+```python
+from afrispeech_synth import tts
+from afrispeech_synth.tts.base import Clip, TTSBackend
+
+class MyTTS(TTSBackend):
+    name = "mytts"
+    async def synth(self, text: str, voice: str) -> Clip:
+        audio = await my_api(text, voice)          # bytes
+        return Clip(audio=audio, mime_type="audio/wav",
+                    sample_rate=self.config.sample_rate, voice=voice)
+
+tts.register("mytts", lambda: MyTTS)
+```
+
+Raise `RetryableTTSError` for rate limits and transient failures; the runner backs off
+and retries. Raise `TTSError` for anything permanent.
+
+---
+
+## A note on synthetic speech
+
+Synthetic audio inherits the TTS model's accent and pronunciation errors, and a model trained
+only on it learns those too. Listen to a sample before training, and mix in real recordings from
+[afrispeech-selector](https://github.com/AfriSpeech/afrispeech-selector) where they exist. The
+generated dataset card says plainly that the audio is model-generated — leave that in.
+
+---
+
+## Development
+
+```bash
+git clone https://github.com/AfriSpeech/afrispeech-synth
+cd afrispeech-synth
+python3 -m venv .venv && source .venv/bin/activate
+pip install -e ".[dev]"
+pytest
+```
+
+The test suite runs the whole pipeline end to end against a mock backend, so it needs
+no API key and no network.
+
+## Acknowledgements
+
+- **[Google Gemini TTS](https://ai.google.dev/gemini-api/docs/speech-generation)** generates the
+  audio. Clips produced with it are subject to
+  [Google's API terms](https://ai.google.dev/gemini-api/terms); check them before publishing a
+  dataset, and say plainly in the dataset card that the audio is model-generated — the generated
+  card does.
+- **[africa-g2p](https://github.com/AfriSpeech/africa-g2p)**, built on Hartell's
+  *Alphabets of Africa* (UNESCO, 1993) and Omniglot, for phonemisation.
+- **[africa-corpus-builder](https://github.com/AfriSpeech/africa-corpus-builder)** and
+  **[afriso](https://github.com/AfriSpeech/afriso)** (SIL ISO 639-3 + Glottolog) for text and
+  language metadata.
+
+## License
+
+MIT — the pipeline. Generated audio is yours subject to your TTS provider's terms, and source
+text keeps the licence of its own corpus.