samvada 20261111.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- samvada-20261111.1/.gitignore +13 -0
- samvada-20261111.1/LICENSE +21 -0
- samvada-20261111.1/PKG-INFO +101 -0
- samvada-20261111.1/README.md +75 -0
- samvada-20261111.1/pyproject.toml +59 -0
- samvada-20261111.1/release/bundle.py +189 -0
- samvada-20261111.1/release/lock_voices.py +125 -0
- samvada-20261111.1/src/samvada/__init__.py +6 -0
- samvada-20261111.1/src/samvada/__main__.py +3 -0
- samvada-20261111.1/src/samvada/agents/__init__.py +1 -0
- samvada-20261111.1/src/samvada/agents/hooks.py +105 -0
- samvada-20261111.1/src/samvada/agents/rule.md +22 -0
- samvada-20261111.1/src/samvada/agents/setup.py +239 -0
- samvada-20261111.1/src/samvada/agents/skills/samvada/SKILL.md +88 -0
- samvada-20261111.1/src/samvada/cli.py +246 -0
- samvada-20261111.1/src/samvada/engines.py +130 -0
- samvada-20261111.1/src/samvada/locks/piper-plus.txt +74 -0
- samvada-20261111.1/src/samvada/locks/piper.txt +120 -0
- samvada-20261111.1/src/samvada/packs.py +306 -0
- samvada-20261111.1/src/samvada/paths.py +51 -0
- samvada-20261111.1/src/samvada/player.py +22 -0
- samvada-20261111.1/src/samvada/presence.py +326 -0
- samvada-20261111.1/src/samvada/speak.py +75 -0
- samvada-20261111.1/src/samvada/state.py +166 -0
- samvada-20261111.1/src/samvada/voices.lock +155 -0
- samvada-20261111.1/src/samvada/voices.toml +29 -0
- samvada-20261111.1/tests/conftest.py +102 -0
- samvada-20261111.1/tests/test_agents.py +137 -0
- samvada-20261111.1/tests/test_cli.py +155 -0
- samvada-20261111.1/tests/test_packs.py +237 -0
- samvada-20261111.1/tests/test_parity.py +52 -0
- samvada-20261111.1/tests/test_presence.py +181 -0
- samvada-20261111.1/uv.lock +852 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Cong Zhang
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: samvada
|
|
3
|
+
Version: 20261111.1
|
|
4
|
+
Summary: Samvada: a voice and the habits of a language tutor for AI agents, offline first
|
|
5
|
+
Project-URL: Homepage, https://github.com/congzhangzh/samvada
|
|
6
|
+
Project-URL: Source, https://github.com/congzhangzh/samvada
|
|
7
|
+
Project-URL: Releases, https://github.com/congzhangzh/samvada/releases
|
|
8
|
+
Author-email: Cong Zhang <13283869+congzhangzh@users.noreply.github.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: ai-agent,language-learning,offline,piper,text-to-speech,tts
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
|
|
22
|
+
Requires-Python: >=3.11
|
|
23
|
+
Provides-Extra: online
|
|
24
|
+
Requires-Dist: edge-tts>=7; extra == 'online'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# Samvada
|
|
28
|
+
|
|
29
|
+
**A voice for AI agents, offline first.** The agent decides what to say; Samvada decides
|
|
30
|
+
whether, where and how it is heard. *Samvada* (संवाद, *saṃvāda*) means dialogue.
|
|
31
|
+
|
|
32
|
+
- **Offline first.** Local voices (Piper) for English and Chinese, French and Japanese as
|
|
33
|
+
optional packs. Online voices (edge-tts) are **off by default**, so your text never leaves
|
|
34
|
+
the machine unless you turn them on.
|
|
35
|
+
- **Plays only where you are.** Silent when the screen is locked, when you are muted, or when
|
|
36
|
+
your last input came from your phone instead of this desk. Silence is never an error.
|
|
37
|
+
- **One call, every language.** The caller passes all its languages in order; Samvada skips
|
|
38
|
+
the ones that are off or have no voice installed.
|
|
39
|
+
- **No dependencies.** The package itself is pure Python; each voice engine runs in its own
|
|
40
|
+
folder, with pinned versions.
|
|
41
|
+
|
|
42
|
+
Status: alpha. Linux (GNOME) is the tested platform.
|
|
43
|
+
|
|
44
|
+
## Install
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
uv tool install samvada # or: pipx install samvada, pip install samvada
|
|
48
|
+
samvada voices add en zh # once: about 290 MB, downloaded and verified
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## Use
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
samvada speak en "Hello" zh "你好" # plays in this order, then exits
|
|
55
|
+
samvada status # where the voice goes now, and why
|
|
56
|
+
samvada mute "watching TV" # silent until `samvada unmute`
|
|
57
|
+
samvada lang off fr # skip a language (the caller still passes it)
|
|
58
|
+
samvada --session mute # the same, for this terminal session only
|
|
59
|
+
samvada online fallback # allow edge-tts for a language that fails offline
|
|
60
|
+
# (needs: pip install 'samvada[online]')
|
|
61
|
+
samvada presence phone # a chat bridge says: the owner just wrote from the phone
|
|
62
|
+
samvada voices verify # check the installed voices against their checksums
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Exit codes: `0` played, or silent by decision; `1` a language failed; `2` bad usage.
|
|
66
|
+
|
|
67
|
+
## Voice packs
|
|
68
|
+
|
|
69
|
+
Every file a pack needs is pinned in `src/samvada/voices.lock`: a commit on the
|
|
70
|
+
[Hugging Face Hub](https://huggingface.co) (or a URL at a commit), with its size and sha256.
|
|
71
|
+
`samvada voices add` downloads from the Hub's resolve URLs, resumes partial downloads, and
|
|
72
|
+
checks every file before putting it in place. Nothing needs configuring; optionally
|
|
73
|
+
`HF_ENDPOINT` points at a mirror and `HF_TOKEN` lifts anonymous rate limits.
|
|
74
|
+
|
|
75
|
+
| Pack | Voice | Engine | Source |
|
|
76
|
+
|---|---|---|---|
|
|
77
|
+
| en | `en_US-lessac-high` | Piper | `rhasspy/piper-voices` |
|
|
78
|
+
| zh | `zh_CN-xiao_ya-medium` + g2pW + BERT tokenizer | Piper | `rhasspy/piper-voices`, `rhasspy/piper-checkpoints`, `google-bert/bert-base-chinese` |
|
|
79
|
+
| ja | Tsukuyomi-chan | piper-plus | `ayousanz/piper-plus-tsukuyomi-chan` + NLTK data |
|
|
80
|
+
| fr | `fr_FR-siwis-medium` | Piper | `rhasspy/piper-voices` |
|
|
81
|
+
|
|
82
|
+
Each voice keeps its own license; see its model card on the Hub before you redistribute it.
|
|
83
|
+
|
|
84
|
+
## Where things live
|
|
85
|
+
|
|
86
|
+
| What | Where |
|
|
87
|
+
|---|---|
|
|
88
|
+
| Settings (mute, languages, online mode, sessions) | `$XDG_STATE_HOME/samvada/` |
|
|
89
|
+
| Voices, engines | `$XDG_DATA_HOME/samvada/{voices,engines}/` (or `$SAMVADA_VOICES`, `$SAMVADA_ENGINES`) |
|
|
90
|
+
|
|
91
|
+
## Develop
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
uv run pytest -q # fake engines, player, desktop and Hub: no audio, no network
|
|
95
|
+
uv run python release/bundle.py <version> --test # the standalone bundle and its offline test
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
## License
|
|
99
|
+
|
|
100
|
+
MIT, see [LICENSE](LICENSE). The voices are not part of this package; they are downloaded
|
|
101
|
+
from their own repositories under their own licenses.
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# Samvada
|
|
2
|
+
|
|
3
|
+
**A voice for AI agents, offline first.** The agent decides what to say; Samvada decides
|
|
4
|
+
whether, where and how it is heard. *Samvada* (संवाद, *saṃvāda*) means dialogue.
|
|
5
|
+
|
|
6
|
+
- **Offline first.** Local voices (Piper) for English and Chinese, French and Japanese as
|
|
7
|
+
optional packs. Online voices (edge-tts) are **off by default**, so your text never leaves
|
|
8
|
+
the machine unless you turn them on.
|
|
9
|
+
- **Plays only where you are.** Silent when the screen is locked, when you are muted, or when
|
|
10
|
+
your last input came from your phone instead of this desk. Silence is never an error.
|
|
11
|
+
- **One call, every language.** The caller passes all its languages in order; Samvada skips
|
|
12
|
+
the ones that are off or have no voice installed.
|
|
13
|
+
- **No dependencies.** The package itself is pure Python; each voice engine runs in its own
|
|
14
|
+
folder, with pinned versions.
|
|
15
|
+
|
|
16
|
+
Status: alpha. Linux (GNOME) is the tested platform.
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
uv tool install samvada # or: pipx install samvada, pip install samvada
|
|
22
|
+
samvada voices add en zh # once: about 290 MB, downloaded and verified
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Use
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
samvada speak en "Hello" zh "你好" # plays in this order, then exits
|
|
29
|
+
samvada status # where the voice goes now, and why
|
|
30
|
+
samvada mute "watching TV" # silent until `samvada unmute`
|
|
31
|
+
samvada lang off fr # skip a language (the caller still passes it)
|
|
32
|
+
samvada --session mute # the same, for this terminal session only
|
|
33
|
+
samvada online fallback # allow edge-tts for a language that fails offline
|
|
34
|
+
# (needs: pip install 'samvada[online]')
|
|
35
|
+
samvada presence phone # a chat bridge says: the owner just wrote from the phone
|
|
36
|
+
samvada voices verify # check the installed voices against their checksums
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Exit codes: `0` played, or silent by decision; `1` a language failed; `2` bad usage.
|
|
40
|
+
|
|
41
|
+
## Voice packs
|
|
42
|
+
|
|
43
|
+
Every file a pack needs is pinned in `src/samvada/voices.lock`: a commit on the
|
|
44
|
+
[Hugging Face Hub](https://huggingface.co) (or a URL at a commit), with its size and sha256.
|
|
45
|
+
`samvada voices add` downloads from the Hub's resolve URLs, resumes partial downloads, and
|
|
46
|
+
checks every file before putting it in place. Nothing needs configuring; optionally
|
|
47
|
+
`HF_ENDPOINT` points at a mirror and `HF_TOKEN` lifts anonymous rate limits.
|
|
48
|
+
|
|
49
|
+
| Pack | Voice | Engine | Source |
|
|
50
|
+
|---|---|---|---|
|
|
51
|
+
| en | `en_US-lessac-high` | Piper | `rhasspy/piper-voices` |
|
|
52
|
+
| zh | `zh_CN-xiao_ya-medium` + g2pW + BERT tokenizer | Piper | `rhasspy/piper-voices`, `rhasspy/piper-checkpoints`, `google-bert/bert-base-chinese` |
|
|
53
|
+
| ja | Tsukuyomi-chan | piper-plus | `ayousanz/piper-plus-tsukuyomi-chan` + NLTK data |
|
|
54
|
+
| fr | `fr_FR-siwis-medium` | Piper | `rhasspy/piper-voices` |
|
|
55
|
+
|
|
56
|
+
Each voice keeps its own license; see its model card on the Hub before you redistribute it.
|
|
57
|
+
|
|
58
|
+
## Where things live
|
|
59
|
+
|
|
60
|
+
| What | Where |
|
|
61
|
+
|---|---|
|
|
62
|
+
| Settings (mute, languages, online mode, sessions) | `$XDG_STATE_HOME/samvada/` |
|
|
63
|
+
| Voices, engines | `$XDG_DATA_HOME/samvada/{voices,engines}/` (or `$SAMVADA_VOICES`, `$SAMVADA_ENGINES`) |
|
|
64
|
+
|
|
65
|
+
## Develop
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
uv run pytest -q # fake engines, player, desktop and Hub: no audio, no network
|
|
69
|
+
uv run python release/bundle.py <version> --test # the standalone bundle and its offline test
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## License
|
|
73
|
+
|
|
74
|
+
MIT, see [LICENSE](LICENSE). The voices are not part of this package; they are downloaded
|
|
75
|
+
from their own repositories under their own licenses.
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
# The import is samvada and the command samvada. A release stamps its version, <YYYYMMDD>.<N>,
|
|
3
|
+
# here and in src/samvada/__init__.py.
|
|
4
|
+
name = "samvada"
|
|
5
|
+
version = "20261111.1"
|
|
6
|
+
description = "Samvada: a voice and the habits of a language tutor for AI agents, offline first"
|
|
7
|
+
readme = "README.md"
|
|
8
|
+
authors = [
|
|
9
|
+
{ name = "Cong Zhang", email = "13283869+congzhangzh@users.noreply.github.com" }
|
|
10
|
+
]
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
requires-python = ">=3.11"
|
|
14
|
+
keywords = ["tts", "text-to-speech", "offline", "piper", "ai-agent", "language-learning"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: POSIX :: Linux",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Programming Language :: Python :: 3.14",
|
|
25
|
+
"Topic :: Multimedia :: Sound/Audio :: Speech",
|
|
26
|
+
]
|
|
27
|
+
# No dependency by default: the voices run as subprocesses in their own environments, and
|
|
28
|
+
# the default install cannot reach the network (docs/DESIGN.md 4.3).
|
|
29
|
+
dependencies = []
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/congzhangzh/samvada"
|
|
33
|
+
Source = "https://github.com/congzhangzh/samvada"
|
|
34
|
+
Releases = "https://github.com/congzhangzh/samvada/releases"
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
# edge-tts: only used after `samvada online fallback|always`
|
|
38
|
+
online = ["edge-tts>=7"]
|
|
39
|
+
|
|
40
|
+
[project.scripts]
|
|
41
|
+
samvada = "samvada.cli:main"
|
|
42
|
+
# The `speak` alias (DESIGN 11) is registered at the migration step, not before: installing it
|
|
43
|
+
# now would collide with the bash wrapper that is still in use.
|
|
44
|
+
|
|
45
|
+
[dependency-groups]
|
|
46
|
+
dev = ["pytest>=8"]
|
|
47
|
+
|
|
48
|
+
[build-system]
|
|
49
|
+
requires = ["hatchling>=1.25"]
|
|
50
|
+
build-backend = "hatchling.build"
|
|
51
|
+
|
|
52
|
+
[tool.hatch.build.targets.wheel]
|
|
53
|
+
packages = ["src/samvada"]
|
|
54
|
+
|
|
55
|
+
[tool.hatch.build.targets.sdist]
|
|
56
|
+
include = ["src/samvada", "tests", "release", "README.md", "LICENSE", "pyproject.toml", "uv.lock"]
|
|
57
|
+
|
|
58
|
+
[tool.pytest.ini_options]
|
|
59
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""Build the standalone bundle: Samvada with its own Python, engines and voice packs (DESIGN 8.2).
|
|
2
|
+
|
|
3
|
+
uv run python release/bundle.py <version> [--langs en,zh] [--out DIR] [--work DIR]
|
|
4
|
+
[--cache DIR] [--test]
|
|
5
|
+
|
|
6
|
+
The result works on a machine with no network: unpack it anywhere and run bin/samvada.
|
|
7
|
+
Layout of samvada-<version>-linux-x86_64/:
|
|
8
|
+
|
|
9
|
+
bin/samvada the launcher (sh): sets the folders below, runs python -m samvada
|
|
10
|
+
python/ a standalone CPython (python-build-standalone, through uv)
|
|
11
|
+
lib/ samvada itself, from the wheel
|
|
12
|
+
engines/<name>/ each engine's pinned requirements (separate: both install `piper`)
|
|
13
|
+
voices/ the voice packs, verified against voices.lock
|
|
14
|
+
BUNDLE.json what is inside and where it came from
|
|
15
|
+
|
|
16
|
+
A version is never overwritten: building one that exists stops. Heavy work goes in --work and
|
|
17
|
+
--cache (default under $SAMVADA_BUILD_DIR, else ~/.cache/samvada-build; never /tmp, which is
|
|
18
|
+
often tmpfs, i.e. RAM). --test unpacks the archive
|
|
19
|
+
in a new folder and speaks en and zh inside a container with no network (podman or docker).
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import argparse
|
|
25
|
+
import hashlib
|
|
26
|
+
import json
|
|
27
|
+
import os
|
|
28
|
+
import platform
|
|
29
|
+
import shutil
|
|
30
|
+
import subprocess
|
|
31
|
+
import sys
|
|
32
|
+
import tarfile
|
|
33
|
+
import time
|
|
34
|
+
import wave
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
|
|
37
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
38
|
+
PYTHON = "3.12" # the engines' wheels (onnxruntime, tokenizers) are proven on it
|
|
39
|
+
SCRATCH = Path(os.environ.get("SAMVADA_BUILD_DIR") or Path.home() / ".cache" / "samvada-build")
|
|
40
|
+
|
|
41
|
+
LAUNCHER = """#!/bin/sh
|
|
42
|
+
# Samvada standalone bundle: everything it needs is in this folder; no network needed.
|
|
43
|
+
here=$(cd "$(dirname "$(readlink -f "$0")")/.." && pwd)
|
|
44
|
+
export SAMVADA_VOICES="${SAMVADA_VOICES:-$here/voices}"
|
|
45
|
+
export SAMVADA_ENGINES="${SAMVADA_ENGINES:-$here/engines}"
|
|
46
|
+
export PYTHONPATH="$here/lib${PYTHONPATH:+:$PYTHONPATH}"
|
|
47
|
+
exec "$here/python/bin/python3" -s -m samvada "$@"
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
TEST_TEXT = {"en": "Samvada speaks offline.", "zh": "Samvada 离线说话。",
|
|
51
|
+
"ja": "サンヴァーダはオフラインで話します。", "fr": "Samvada parle hors ligne."}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def run(cmd: list[str], **kw) -> subprocess.CompletedProcess:
|
|
55
|
+
print("+ " + " ".join(str(c) for c in cmd), flush=True)
|
|
56
|
+
return subprocess.run(cmd, check=True, **kw)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def git_commit() -> str:
|
|
60
|
+
r = subprocess.run(["git", "-C", str(ROOT), "rev-parse", "HEAD"], capture_output=True, text=True)
|
|
61
|
+
dirty = subprocess.run(["git", "-C", str(ROOT), "status", "--porcelain", "--", "."],
|
|
62
|
+
capture_output=True, text=True).stdout.strip()
|
|
63
|
+
return r.stdout.strip() + ("+dirty" if dirty else "")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def sha256(path: Path) -> str:
|
|
67
|
+
h = hashlib.sha256()
|
|
68
|
+
with path.open("rb") as fh:
|
|
69
|
+
while chunk := fh.read(1 << 20):
|
|
70
|
+
h.update(chunk)
|
|
71
|
+
return h.hexdigest()
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def du(path: Path) -> int:
|
|
75
|
+
return sum(p.stat().st_size for p in path.rglob("*") if p.is_file() and not p.is_symlink())
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def build(version: str, langs: list[str], out: Path, work: Path, cache: Path) -> Path:
|
|
79
|
+
plat = f"{platform.system().lower()}-{platform.machine()}"
|
|
80
|
+
name = f"samvada-{version}-{plat}"
|
|
81
|
+
archive = out / f"{name}.tar.gz"
|
|
82
|
+
if archive.exists() or (out / name).exists():
|
|
83
|
+
sys.exit(f"bundle: {archive} exists; a version is never overwritten (pick the next one)")
|
|
84
|
+
stage = work / name
|
|
85
|
+
shutil.rmtree(stage, ignore_errors=True)
|
|
86
|
+
stage.mkdir(parents=True)
|
|
87
|
+
t0 = time.time()
|
|
88
|
+
|
|
89
|
+
wheels = work / "wheel"
|
|
90
|
+
shutil.rmtree(wheels, ignore_errors=True)
|
|
91
|
+
run(["uv", "build", "--wheel", "--out-dir", str(wheels), str(ROOT)])
|
|
92
|
+
wheel = next(wheels.glob("samvada-*.whl"))
|
|
93
|
+
|
|
94
|
+
pys = work / "python"
|
|
95
|
+
run(["uv", "python", "install", "--install-dir", str(pys), PYTHON])
|
|
96
|
+
src = sorted(p for p in pys.glob(f"cpython-{PYTHON}.*") if p.is_dir() and not p.is_symlink())[-1]
|
|
97
|
+
shutil.copytree(src, stage / "python", symlinks=True)
|
|
98
|
+
py = stage / "python" / "bin" / "python3"
|
|
99
|
+
|
|
100
|
+
run(["uv", "pip", "install", "--quiet", "--python", str(py), "--target", str(stage / "lib"),
|
|
101
|
+
"--no-deps", str(wheel)])
|
|
102
|
+
(stage / "bin").mkdir()
|
|
103
|
+
launcher = stage / "bin" / "samvada"
|
|
104
|
+
launcher.write_text(LAUNCHER)
|
|
105
|
+
launcher.chmod(0o755)
|
|
106
|
+
|
|
107
|
+
env = {**os.environ, "SAMVADA_DOWNLOAD_CACHE": str(cache),
|
|
108
|
+
"XDG_STATE_HOME": str(work / "state"), "XDG_DATA_HOME": str(work / "data")}
|
|
109
|
+
run([str(launcher), "voices", "add", *langs, "--engine", "always"], env=env)
|
|
110
|
+
run([str(launcher), "voices", "verify", *langs], env=env)
|
|
111
|
+
shutil.rmtree(work / "state", ignore_errors=True) # never ship a builder's state
|
|
112
|
+
|
|
113
|
+
info = {
|
|
114
|
+
"name": name, "version": version, "languages": langs, "python": src.name,
|
|
115
|
+
"source_commit": git_commit(), "voices_lock_sha256": sha256(ROOT / "src/samvada/voices.lock"),
|
|
116
|
+
"built": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "size_bytes": du(stage),
|
|
117
|
+
}
|
|
118
|
+
(stage / "BUNDLE.json").write_text(json.dumps(info, indent=1, ensure_ascii=False) + "\n")
|
|
119
|
+
|
|
120
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
121
|
+
tmp = archive.with_name(archive.name + ".part")
|
|
122
|
+
with tarfile.open(tmp, "w:gz", compresslevel=6) as tar:
|
|
123
|
+
tar.add(stage, arcname=name)
|
|
124
|
+
tmp.replace(archive)
|
|
125
|
+
(out / f"{name}.tar.gz.sha256").write_text(f"{sha256(archive)} {archive.name}\n")
|
|
126
|
+
print(f"bundle: {archive} ({archive.stat().st_size / 1e6:.0f} MB packed, "
|
|
127
|
+
f"{info['size_bytes'] / 1e6:.0f} MB unpacked) in {time.time() - t0:.0f}s")
|
|
128
|
+
return archive
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def wav_seconds(path: Path) -> float:
|
|
132
|
+
with wave.open(str(path)) as w:
|
|
133
|
+
return w.getnframes() / w.getframerate()
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def test(archive: Path, langs: list[str], work: Path) -> int:
|
|
137
|
+
"""Unpack in a new folder (another path: it must be relocatable) and speak in a container
|
|
138
|
+
with no network, on an image with no Python at all."""
|
|
139
|
+
engine = shutil.which("podman") or shutil.which("docker")
|
|
140
|
+
if not engine:
|
|
141
|
+
sys.exit("bundle test: needs podman or docker")
|
|
142
|
+
where = work / f"test-{time.strftime('%Y%m%d-%H%M%S')}"
|
|
143
|
+
where.mkdir(parents=True)
|
|
144
|
+
with tarfile.open(archive) as tar:
|
|
145
|
+
tar.extractall(where, filter="tar")
|
|
146
|
+
folder = next(p for p in where.iterdir() if p.is_dir())
|
|
147
|
+
out = where / "out"
|
|
148
|
+
out.mkdir()
|
|
149
|
+
pairs = [x for l in langs for x in (l, TEST_TEXT[l])]
|
|
150
|
+
cmd = [engine, "run", "--rm", "--network", "none",
|
|
151
|
+
"-v", f"{folder}:/opt/samvada:ro,Z", "-v", f"{out}:/out:Z",
|
|
152
|
+
"-e", "HOME=/tmp", "debian:testing",
|
|
153
|
+
"sh", "-c", 'command -v python3 && echo "python3 found: not a clean machine" && exit 9; '
|
|
154
|
+
'/opt/samvada/bin/samvada voices verify && '
|
|
155
|
+
'/opt/samvada/bin/samvada speak --save /out "$@"', "sh", *pairs]
|
|
156
|
+
r = subprocess.run(cmd)
|
|
157
|
+
print(f"bundle test: container exit {r.returncode}")
|
|
158
|
+
ok = r.returncode == 0
|
|
159
|
+
for i, lang in enumerate(langs, 1):
|
|
160
|
+
clip = out / f"{i}-{lang}.wav"
|
|
161
|
+
if clip.is_file():
|
|
162
|
+
print(f"bundle test: {lang}: {clip.name} {wav_seconds(clip):.2f}s of audio")
|
|
163
|
+
ok &= wav_seconds(clip) > 0.5
|
|
164
|
+
else:
|
|
165
|
+
print(f"bundle test: {lang}: no clip")
|
|
166
|
+
ok = False
|
|
167
|
+
print(f"bundle test: {'PASSED' if ok else 'FAILED'} (network none; clips in {out})")
|
|
168
|
+
return 0 if ok else 1
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def main() -> int:
|
|
172
|
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
173
|
+
ap.add_argument("version")
|
|
174
|
+
ap.add_argument("--langs", default="en,zh", help="the packs inside (default: en,zh)")
|
|
175
|
+
ap.add_argument("--out", type=Path, default=SCRATCH / "bundles")
|
|
176
|
+
ap.add_argument("--work", type=Path, default=SCRATCH / "work")
|
|
177
|
+
ap.add_argument("--cache", type=Path, default=SCRATCH / "download-cache")
|
|
178
|
+
ap.add_argument("--test", action="store_true", help="also run the offline container test")
|
|
179
|
+
ap.add_argument("--test-only", type=Path, metavar="ARCHIVE", help="only test this archive")
|
|
180
|
+
a = ap.parse_args()
|
|
181
|
+
langs = [l for l in a.langs.split(",") if l]
|
|
182
|
+
if a.test_only:
|
|
183
|
+
return test(a.test_only, langs, a.work)
|
|
184
|
+
archive = build(a.version, langs, a.out, a.work, a.cache)
|
|
185
|
+
return test(archive, langs, a.work) if a.test else 0
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
if __name__ == "__main__":
|
|
189
|
+
sys.exit(main())
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Write src/samvada/voices.lock: every file a voice pack needs, pinned (DESIGN 8.3, 8.4).
|
|
2
|
+
|
|
3
|
+
The sources live here, each pinned to a commit. For each file the Hub's API gives the size
|
|
4
|
+
and, for files in LFS, the sha256; a small file outside LFS is downloaded once to hash it.
|
|
5
|
+
Also writes the engines' requirement locks (src/samvada/locks/*.txt) with `uv pip compile`.
|
|
6
|
+
|
|
7
|
+
uv run python release/lock_voices.py # network needed; nothing is installed
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import subprocess
|
|
15
|
+
import sys
|
|
16
|
+
import urllib.request
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
20
|
+
PKG = ROOT / "src" / "samvada"
|
|
21
|
+
HF = "https://huggingface.co"
|
|
22
|
+
|
|
23
|
+
PIPER_VOICES = ("rhasspy/piper-voices", "model", "cf910fb59f1d79ab3b625a63993a108751c6c25d")
|
|
24
|
+
CHECKPOINTS = ("rhasspy/piper-checkpoints", "dataset", "8d418f0288a50aed603547ee751193fa7910bfe3")
|
|
25
|
+
BERT_ZH = ("google-bert/bert-base-chinese", "model", "8f23c25b06e129b6c986331a13d8d025a92cf0ea")
|
|
26
|
+
TSUKUYOMI = ("ayousanz/piper-plus-tsukuyomi-chan", "model", "619ad26fcbd5635b3d70a27eac919f4c2ce892b1")
|
|
27
|
+
NLTK = "https://raw.githubusercontent.com/nltk/nltk_data/550b6625bcef1f2abff2ff770a5a0d272c9c6b2a"
|
|
28
|
+
|
|
29
|
+
# The tokenizer goes into a Hugging Face cache layout under the voices folder, named as
|
|
30
|
+
# Piper asks for it ("bert-base-chinese"), so from_pretrained finds it with HF_HUB_OFFLINE=1.
|
|
31
|
+
BERT_SNAP = f"hf/hub/models--bert-base-chinese/snapshots/{BERT_ZH[2]}"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def piper_voice(pack: str, folder: str, name: str) -> list[dict]:
|
|
35
|
+
return [{"pack": pack, "hf": PIPER_VOICES, "path": f"{folder}/{name}{ext}", "dest": name + ext}
|
|
36
|
+
for ext in (".onnx", ".onnx.json")]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
SOURCES: list[dict] = [
|
|
40
|
+
*piper_voice("en", "en/en_US/lessac/high", "en_US-lessac-high"),
|
|
41
|
+
*piper_voice("zh", "zh/zh_CN/xiao_ya/medium", "zh_CN-xiao_ya-medium"),
|
|
42
|
+
{"pack": "zh", "hf": CHECKPOINTS, "path": "zh/zh_CN/_resources/g2pw.tar.gz",
|
|
43
|
+
"dest": "g2pW", "unpack": "tar"},
|
|
44
|
+
*[{"pack": "zh", "hf": BERT_ZH, "path": f, "dest": f"{BERT_SNAP}/{f}"}
|
|
45
|
+
for f in ("tokenizer_config.json", "tokenizer.json", "vocab.txt")],
|
|
46
|
+
{"pack": "ja", "hf": TSUKUYOMI, "path": "tsukuyomi-chan-6lang-fp16.onnx",
|
|
47
|
+
"dest": "tsukuyomi-chan-6lang-fp16.onnx"},
|
|
48
|
+
{"pack": "ja", "hf": TSUKUYOMI, "path": "config.json",
|
|
49
|
+
"dest": "tsukuyomi-chan-6lang-fp16.onnx.json"},
|
|
50
|
+
{"pack": "ja", "url": f"{NLTK}/packages/taggers/averaged_perceptron_tagger_eng.zip",
|
|
51
|
+
"dest": "nltk_data/taggers", "unpack": "zip"},
|
|
52
|
+
{"pack": "ja", "url": f"{NLTK}/packages/corpora/cmudict.zip",
|
|
53
|
+
"dest": "nltk_data/corpora", "unpack": "zip"},
|
|
54
|
+
*piper_voice("fr", "fr/fr_FR/siwis/medium", "fr_FR-siwis-medium"),
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
ENGINES = {
|
|
58
|
+
"piper": "piper-tts[zh]==1.8.0",
|
|
59
|
+
"piper-plus": "piper-plus==1.13.0",
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def get(url: str) -> bytes:
|
|
64
|
+
req = urllib.request.Request(url, headers={"User-Agent": "samvada-lock"})
|
|
65
|
+
with urllib.request.urlopen(req, timeout=120) as r:
|
|
66
|
+
return r.read()
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def tree(repo: str, kind: str, rev: str, folder: str) -> dict[str, dict]:
|
|
70
|
+
prefix = "datasets/" if kind == "dataset" else "models/"
|
|
71
|
+
url = f"{HF}/api/{prefix}{repo}/tree/{rev}/{folder}".rstrip("/")
|
|
72
|
+
return {f["path"]: f for f in json.loads(get(url))}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def resolve(src: dict) -> dict:
|
|
76
|
+
out = {"pack": src["pack"]}
|
|
77
|
+
if "hf" in src:
|
|
78
|
+
repo, kind, rev = src["hf"]
|
|
79
|
+
folder = src["path"].rpartition("/")[0]
|
|
80
|
+
info = tree(repo, kind, rev, folder)[src["path"]]
|
|
81
|
+
out |= {"repo": repo, "repo_type": kind, "revision": rev, "path": src["path"]}
|
|
82
|
+
size, sha = info["size"], (info.get("lfs") or {}).get("oid")
|
|
83
|
+
if not sha: # a small file outside LFS: hash it
|
|
84
|
+
prefix = "datasets/" if kind == "dataset" else ""
|
|
85
|
+
data = get(f"{HF}/{prefix}{repo}/resolve/{rev}/{src['path']}")
|
|
86
|
+
assert len(data) == size, src
|
|
87
|
+
sha = hashlib.sha256(data).hexdigest()
|
|
88
|
+
else:
|
|
89
|
+
data = get(src["url"])
|
|
90
|
+
out["url"] = src["url"]
|
|
91
|
+
size, sha = len(data), hashlib.sha256(data).hexdigest()
|
|
92
|
+
out |= {"dest": src["dest"], "unpack": src.get("unpack", ""), "size": size, "sha256": sha}
|
|
93
|
+
return out
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def toml_value(v) -> str:
|
|
97
|
+
return str(v) if isinstance(v, int) else json.dumps(v, ensure_ascii=False)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def main() -> int:
|
|
101
|
+
lines = ["# Generated by release/lock_voices.py: do not edit by hand.",
|
|
102
|
+
"# Every file a voice pack needs, pinned: a Hub commit (or a URL at a commit),",
|
|
103
|
+
"# its size and sha256. `samvada voices add` downloads and verifies against it.", ""]
|
|
104
|
+
for src in SOURCES:
|
|
105
|
+
f = resolve(src)
|
|
106
|
+
print(f"{f['pack']} {f['size']:>11} {f['sha256'][:12]} {f['dest']}", file=sys.stderr)
|
|
107
|
+
lines.append("[[file]]")
|
|
108
|
+
lines += [f"{k} = {toml_value(v)}" for k, v in f.items()]
|
|
109
|
+
lines.append("")
|
|
110
|
+
lines += ["[engines]"] + [f"{k} = {toml_value(v)}" for k, v in ENGINES.items()]
|
|
111
|
+
(PKG / "voices.lock").write_text("\n".join(lines) + "\n")
|
|
112
|
+
|
|
113
|
+
locks = PKG / "locks"
|
|
114
|
+
locks.mkdir(exist_ok=True)
|
|
115
|
+
for name, req in ENGINES.items():
|
|
116
|
+
out = locks / f"{name}.txt"
|
|
117
|
+
subprocess.run(["uv", "pip", "compile", "--universal", "--quiet", "--no-header",
|
|
118
|
+
"--python-version", "3.12", "-o", str(out), "-"],
|
|
119
|
+
input=req + "\n", text=True, check=True)
|
|
120
|
+
print(f"engine {name}: {out.relative_to(ROOT)}", file=sys.stderr)
|
|
121
|
+
return 0
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
if __name__ == "__main__":
|
|
125
|
+
sys.exit(main())
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""The agent layer: the rule, the skill, the hooks and `samvada setup` (DESIGN 7)."""
|