desphere 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- desphere-0.1.0/.gitignore +42 -0
- desphere-0.1.0/CHANGELOG.md +45 -0
- desphere-0.1.0/LICENSE +21 -0
- desphere-0.1.0/PKG-INFO +180 -0
- desphere-0.1.0/PROVENANCE.md +90 -0
- desphere-0.1.0/README.md +139 -0
- desphere-0.1.0/pyproject.toml +52 -0
- desphere-0.1.0/src/desphere/__init__.py +49 -0
- desphere-0.1.0/src/desphere/cli.py +155 -0
- desphere-0.1.0/src/desphere/codecs.py +217 -0
- desphere-0.1.0/src/desphere/errors.py +34 -0
- desphere-0.1.0/src/desphere/g711.py +49 -0
- desphere-0.1.0/src/desphere/shorten.py +339 -0
- desphere-0.1.0/src/desphere/sphere.py +238 -0
- desphere-0.1.0/src/desphere/transcode.py +97 -0
- desphere-0.1.0/src/desphere/wav.py +73 -0
- desphere-0.1.0/tests/conftest.py +60 -0
- desphere-0.1.0/tests/fixtures/alaw_allcodes.sph +0 -0
- desphere-0.1.0/tests/fixtures/manifest.json +70 -0
- desphere-0.1.0/tests/fixtures/pcm16be_mono.sph +0 -0
- desphere-0.1.0/tests/fixtures/pcm16be_stereo.sph +0 -0
- desphere-0.1.0/tests/fixtures/pcm16le_mono.sph +0 -0
- desphere-0.1.0/tests/fixtures/pcm16le_stereo.sph +0 -0
- desphere-0.1.0/tests/fixtures/pcm32be_mono.sph +0 -0
- desphere-0.1.0/tests/fixtures/pcm32le_mono.sph +0 -0
- desphere-0.1.0/tests/fixtures/pcm8_mono.sph +0 -0
- desphere-0.1.0/tests/fixtures/qlpc_ar2.shn +0 -0
- desphere-0.1.0/tests/fixtures/qlpc_ar2.wav +0 -0
- desphere-0.1.0/tests/fixtures/qlpc_ar2_stereo.shn +0 -0
- desphere-0.1.0/tests/fixtures/qlpc_ar2_stereo.wav +0 -0
- desphere-0.1.0/tests/fixtures/qlpc_hi.shn +0 -0
- desphere-0.1.0/tests/fixtures/qlpc_hi.wav +0 -0
- desphere-0.1.0/tests/fixtures/shorten_gate.sph +0 -0
- desphere-0.1.0/tests/fixtures/ulaw_allcodes.sph +0 -0
- desphere-0.1.0/tests/fixtures/ulaw_bitshift.shn +0 -0
- desphere-0.1.0/tests/fixtures/ulaw_bitshift.ulaw +0 -0
- desphere-0.1.0/tests/test_api.py +81 -0
- desphere-0.1.0/tests/test_failures.py +82 -0
- desphere-0.1.0/tests/test_fixtures.py +74 -0
- desphere-0.1.0/tests/test_g711.py +43 -0
- desphere-0.1.0/tests/test_pcm.py +80 -0
- desphere-0.1.0/tests/test_qlpc.py +125 -0
- desphere-0.1.0/tests/test_robustness.py +126 -0
- desphere-0.1.0/tests/test_shorten_local.py +71 -0
- desphere-0.1.0/tests/test_sphere.py +81 -0
- desphere-0.1.0/tests/test_ulaw_bitshift.py +66 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
build/
|
|
6
|
+
dist/
|
|
7
|
+
.venv
|
|
8
|
+
.venv/
|
|
9
|
+
venv/
|
|
10
|
+
.pytest_cache/
|
|
11
|
+
.ruff_cache/
|
|
12
|
+
|
|
13
|
+
# Oracle / scratch output (regenerated, not committed)
|
|
14
|
+
*.ref.wav
|
|
15
|
+
/scratch/
|
|
16
|
+
|
|
17
|
+
# Real corpus test files — license-restricted (TIMIT/Switchboard/etc).
|
|
18
|
+
# Synced via Syncthing for cross-machine local testing; never committed.
|
|
19
|
+
/local-fixtures/
|
|
20
|
+
|
|
21
|
+
# Black-box oracle BINARIES — used to validate decode output only; we never read
|
|
22
|
+
# their source. License-restricted, synced via Syncthing, not committed. The
|
|
23
|
+
# build SCRIPTS + README ARE committed as clean-room paper-trail evidence
|
|
24
|
+
# (compiling != reading source); see PROVENANCE.md.
|
|
25
|
+
/oracles/sph2pipe
|
|
26
|
+
/oracles/w_decode
|
|
27
|
+
/oracles/shorten
|
|
28
|
+
|
|
29
|
+
# Local working state — Syncthing-synced, not committed (see CLAUDE.md)
|
|
30
|
+
/memories/
|
|
31
|
+
/plans/
|
|
32
|
+
|
|
33
|
+
# Editor
|
|
34
|
+
*.swp
|
|
35
|
+
.DS_Store
|
|
36
|
+
|
|
37
|
+
# Built WASM for the web page (regenerated; deployed by CI)
|
|
38
|
+
/web/pkg/
|
|
39
|
+
|
|
40
|
+
# Rust build dir (the crate is a workspace member under rust/; cargo builds into
|
|
41
|
+
# the workspace-root target/). Cargo.lock at the root IS committed.
|
|
42
|
+
/target/
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to desphere (the Python package and the Rust crate) are
|
|
4
|
+
recorded here. Format loosely follows [Keep a Changelog]; versions are shared
|
|
5
|
+
across the Python `desphere`, the Rust crate `desphere`, and the pyo3 module
|
|
6
|
+
`desphere-native`.
|
|
7
|
+
|
|
8
|
+
## [0.1.0] — 2026-06-28 (first release)
|
|
9
|
+
|
|
10
|
+
First public release. NIST SPHERE → RIFF/WAV transcoder, clean-room and
|
|
11
|
+
MIT-licensed (see `PROVENANCE.md`).
|
|
12
|
+
|
|
13
|
+
### Decoding (Python reference + Rust port, byte-for-byte identical)
|
|
14
|
+
- PCM 16/32-bit, byte order `01`/`10`, mono & multi-channel.
|
|
15
|
+
- G.711 μ-law / a-law (ITU-T G.711).
|
|
16
|
+
- Embedded-shorten v2:
|
|
17
|
+
- 16-bit PCM (DIFF0–3, ZERO, VERBATIM, BLOCKSIZE).
|
|
18
|
+
- Lossless μ-law (type 8), including the non-linear **BITSHIFT** code-space
|
|
19
|
+
remap (validated on real CALLHOME audio).
|
|
20
|
+
- **QLPC** (LPC-predicted) blocks (validated vs the shorten encoder + ffmpeg,
|
|
21
|
+
orders 1–20, mono & stereo).
|
|
22
|
+
- Fail-loud on everything unvalidated (8/24-bit PCM, QLPC-less unknown codings,
|
|
23
|
+
corrupt/truncated streams) — never emits a plausible-but-wrong WAV, never
|
|
24
|
+
panics (safe for WASM).
|
|
25
|
+
|
|
26
|
+
### Packaging
|
|
27
|
+
- Python package `desphere` with the `sph2wav` CLI (pure Python, zero deps). The
|
|
28
|
+
whole decode path — the streaming `transcode()`, the one-shot `transcode_bytes()`,
|
|
29
|
+
and the CLI — transparently uses the optional Rust accelerator when installed
|
|
30
|
+
(`pip install desphere[fast]` → `desphere-native`) and falls back to pure Python
|
|
31
|
+
otherwise — same bytes either way. The accelerator delegates the heavy kernels
|
|
32
|
+
(shorten decode, G.711 expansion); typed error checks stay in Python.
|
|
33
|
+
- Rust crate `desphere` (dependency-free core); Rust clients (e.g. praatfan)
|
|
34
|
+
import it via a path/git dependency.
|
|
35
|
+
- WASM bindings via `wasm-bindgen` (opt-in `wasm` feature) + a self-contained
|
|
36
|
+
client-side `sph2wav` web page (`web/`, deployed to GitHub Pages) — converts in
|
|
37
|
+
the browser, nothing uploaded.
|
|
38
|
+
- Python-native module `desphere-native` via pyo3/maturin (opt-in `python`
|
|
39
|
+
feature).
|
|
40
|
+
- The CLI and web page pass a stray RIFF/WAV input through unchanged, with a
|
|
41
|
+
warning (the library API stays strict: SPHERE in, fail loud).
|
|
42
|
+
- Not published to crates.io / npm (consumers depend on the repo directly); PyPI
|
|
43
|
+
is the target registry.
|
|
44
|
+
|
|
45
|
+
[Keep a Changelog]: https://keepachangelog.com/en/1.1.0/
|
desphere-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Uriel Cohen Priva
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
desphere-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: desphere
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Clean-room NIST SPHERE (and Shorten) to RIFF/WAV transcoder
|
|
5
|
+
Project-URL: Homepage, https://github.com/ucpresearch/desphere
|
|
6
|
+
Author: Uriel Cohen Priva
|
|
7
|
+
License: MIT License
|
|
8
|
+
|
|
9
|
+
Copyright (c) 2026 Uriel Cohen Priva
|
|
10
|
+
|
|
11
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
13
|
+
in the Software without restriction, including without limitation the rights
|
|
14
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
15
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
16
|
+
furnished to do so, subject to the following conditions:
|
|
17
|
+
|
|
18
|
+
The above copyright notice and this permission notice shall be included in all
|
|
19
|
+
copies or substantial portions of the Software.
|
|
20
|
+
|
|
21
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
22
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
23
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
24
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
25
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
26
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
27
|
+
SOFTWARE.
|
|
28
|
+
License-File: LICENSE
|
|
29
|
+
Keywords: audio,nist,riff,shorten,sph2pipe,sphere,transcode,wav
|
|
30
|
+
Classifier: Development Status :: 3 - Alpha
|
|
31
|
+
Classifier: Intended Audience :: Science/Research
|
|
32
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
33
|
+
Classifier: Programming Language :: Python :: 3
|
|
34
|
+
Classifier: Topic :: Multimedia :: Sound/Audio :: Conversion
|
|
35
|
+
Requires-Python: >=3.9
|
|
36
|
+
Provides-Extra: dev
|
|
37
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
38
|
+
Provides-Extra: fast
|
|
39
|
+
Requires-Dist: desphere-native; extra == 'fast'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# desphere
|
|
43
|
+
|
|
44
|
+
**Flatten a sphere.** `desphere` is a clean-room, MIT-licensed, zero-dependency
|
|
45
|
+
transcoder from **NIST SPHERE** audio (the `.sph` format used by TIMIT, WSJ,
|
|
46
|
+
Switchboard, and friends) to plain **RIFF/WAV**. The CLI is `sph2wav`.
|
|
47
|
+
|
|
48
|
+
> **No tooling, no upload:** there's a browser page that converts a `.sph` to
|
|
49
|
+
> `.wav` entirely client-side (WASM) — see [`web/`](web/index.html). Prefer
|
|
50
|
+
> `sph2pipe`/`ffmpeg`/`sox` if you have them; the page is for when you don't.
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
pip install desphere # pure Python, zero deps, works anywhere
|
|
54
|
+
pip install "desphere[fast]" # + optional Rust accelerator (big shorten files)
|
|
55
|
+
|
|
56
|
+
sph2wav utterance.sph # -> utterance.wav
|
|
57
|
+
sph2wav utterance.sph out.wav
|
|
58
|
+
sph2wav --info utterance.sph # inspect the SPHERE header
|
|
59
|
+
sph2wav utterance.sph - # WAV to stdout
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from desphere import read_sphere, transcode, transcode_bytes
|
|
64
|
+
|
|
65
|
+
# Streaming API (the reference):
|
|
66
|
+
header, data = read_sphere("utterance.sph")
|
|
67
|
+
with open("utterance.wav", "wb") as f:
|
|
68
|
+
transcode(header, data, f)
|
|
69
|
+
|
|
70
|
+
# One-shot bytes API:
|
|
71
|
+
wav_bytes = transcode_bytes(open("utterance.sph", "rb").read())
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
**Both APIs transparently use the Rust accelerator when installed** (and fall
|
|
75
|
+
back to pure Python) — same bytes out either way. With `desphere[fast]`, the
|
|
76
|
+
heavy kernels (shorten decode, G.711 expansion) run in Rust; pure-Python PCM is
|
|
77
|
+
already C-speed. The CLI passes a stray `.wav` through unchanged (with a warning),
|
|
78
|
+
so pointing it at an already-decoded file is harmless.
|
|
79
|
+
|
|
80
|
+
## Why
|
|
81
|
+
|
|
82
|
+
`libsndfile`/`soundfile` cannot read SPHERE (especially shorten-compressed
|
|
83
|
+
SPHERE), and the tools that can — `sph2pipe`, the original `shorten`, FFmpeg's
|
|
84
|
+
decoder — are GPL/LGPL or otherwise awkwardly licensed. `desphere` is a
|
|
85
|
+
permissively licensed, dependency-free reimplementation.
|
|
86
|
+
|
|
87
|
+
## Clean-room policy
|
|
88
|
+
|
|
89
|
+
`desphere` is implemented **only** from:
|
|
90
|
+
|
|
91
|
+
- The **public NIST SPHERE** format description (ASCII header, typed object fields).
|
|
92
|
+
- **ITU-T G.711** for μ-law / a-law.
|
|
93
|
+
- Tony Robinson (1994), *SHORTEN: Simple lossless and near-lossless waveform
|
|
94
|
+
compression* (CUED/F-INFENG/TR.156) for the shorten algorithm.
|
|
95
|
+
- **Black-box testing**: running `sox` / `ffmpeg` / `sph2pipe` as binaries and
|
|
96
|
+
comparing only their **output** — never reading their source.
|
|
97
|
+
|
|
98
|
+
Copyright protects source code, not file formats or algorithms (the
|
|
99
|
+
idea/expression dichotomy), so consulting *prose/tabular* descriptions of a
|
|
100
|
+
format — even third-party ones — is fine. The one rule we never break: **we do
|
|
101
|
+
not read GPL/LGPL source code, and we do not use writeups that embed verbatim
|
|
102
|
+
GPL source.** Authoritative non-source references (the TR, ITU specs, NIST docs)
|
|
103
|
+
are the primary sources; prose writeups are at most a cross-check.
|
|
104
|
+
|
|
105
|
+
## Supported matrix
|
|
106
|
+
|
|
107
|
+
The design principle is **support the obvious lossless path first, and fail
|
|
108
|
+
loudly on anything not yet validated** — never emit a plausible-but-wrong WAV.
|
|
109
|
+
|
|
110
|
+
| Coding | Status |
|
|
111
|
+
|--------|--------|
|
|
112
|
+
| `pcm`, 16-bit, `01`/`10` byte order | ✅ supported |
|
|
113
|
+
| `pcm`, 32-bit, `01`/`10` byte order | ✅ supported |
|
|
114
|
+
| `pcm`, multi-channel (interleaved) | ✅ supported (validate against an oracle for exotic files) |
|
|
115
|
+
| `pcm`, 8-bit / 24-bit | ⛔ rejected (`UnsupportedFormat`) — sign/packing not yet validated |
|
|
116
|
+
| `ulaw` / `alaw` (G.711, 8-bit) | ✅ supported (Phase B) — verified byte-exact vs ffmpeg on all 256 codes |
|
|
117
|
+
| `pcm,embedded-shorten-v2.00` (16-bit) | ✅ supported (Phase C) — byte-exact vs ffmpeg, mono & stereo |
|
|
118
|
+
| `ulaw,embedded-shorten` (shorten type 8), incl. bitshift | ✅ supported (Phase C) — byte-exact vs sph2pipe (real CALLHOME) |
|
|
119
|
+
| shorten QLPC (LPC) blocks | ✅ supported (Phase C) — byte-exact vs shorten encoder + ffmpeg (orders 1–20) |
|
|
120
|
+
|
|
121
|
+
Adding a coding means registering a decoder in `desphere/codecs.py`; until then
|
|
122
|
+
the gate raises a precise error.
|
|
123
|
+
|
|
124
|
+
## Test fixtures (the "zoo")
|
|
125
|
+
|
|
126
|
+
We don't depend on real corpus files to test: `tools/make_fixtures.py`
|
|
127
|
+
*generates* a zoo of SPHERE variants — every byte order, bit depth, and channel
|
|
128
|
+
count — and commits them under `tests/fixtures/` with a manifest recording the
|
|
129
|
+
exact expected output. Harder codings (μ-law/a-law, eventually shorten) are
|
|
130
|
+
produced best-effort by driving `sox`/`ffmpeg`/`shorten` as black-box binaries.
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
python tools/make_fixtures.py # regenerate fixtures + manifest
|
|
134
|
+
pytest # validate everything
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
## Rust, WASM, and consumers
|
|
138
|
+
|
|
139
|
+
The Python in `src/desphere` is the reference **spec**; [`rust/`](rust/) is a
|
|
140
|
+
Rust port that reproduces it **bit-for-bit** (same fixtures), for speed and for
|
|
141
|
+
the web. It builds three ways from one crate:
|
|
142
|
+
|
|
143
|
+
- a **Rust library** — a Rust client (praatfan is a likely consumer) imports it
|
|
144
|
+
via a path/git dependency;
|
|
145
|
+
- **WASM** (`wasm-pack build rust --target web --features wasm`) — a WASM web app
|
|
146
|
+
and the `web/` page consume it; nothing is sent to a server;
|
|
147
|
+
- a **Python extension** `desphere-native` (pyo3/maturin) — the optional
|
|
148
|
+
`desphere[fast]` accelerator above.
|
|
149
|
+
|
|
150
|
+
Because the consumers live in the same `ucpresearch` org, they depend on desphere
|
|
151
|
+
directly (path/git); publishing to **crates.io / npm is not required** (and is
|
|
152
|
+
skipped). **PyPI** is the one registry we target (the pure-Python `desphere`,
|
|
153
|
+
plus the optional `desphere-native` wheels).
|
|
154
|
+
|
|
155
|
+
## Development
|
|
156
|
+
|
|
157
|
+
The virtualenv lives **outside** the (Syncthing-synced) repo and is symlinked in:
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
uv venv ~/local/scr/venvs/desphere
|
|
161
|
+
ln -s ~/local/scr/venvs/desphere .venv
|
|
162
|
+
uv pip install -e ".[dev]"
|
|
163
|
+
pytest
|
|
164
|
+
|
|
165
|
+
# Rust port:
|
|
166
|
+
cd rust && cargo test # byte-exact vs the same fixtures
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
## Roadmap
|
|
170
|
+
|
|
171
|
+
- **Phase A** (done): SPHERE header + 16/32-bit PCM, lossless.
|
|
172
|
+
- **Phase B** (done): μ-law / a-law decode (ITU-T G.711).
|
|
173
|
+
- **Phase C** (done): embedded-shorten of 16-bit PCM, lossless μ-law (type 8,
|
|
174
|
+
including the non-linear BITSHIFT remap), and QLPC (LPC) blocks — validated
|
|
175
|
+
byte-for-byte vs ffmpeg / sph2pipe / the `shorten` encoder on real and
|
|
176
|
+
synthetic streams, mono and stereo. Remaining (low priority): 8/24-bit linear
|
|
177
|
+
PCM. See `docs/STATUS.md` and `docs/SHORTEN.md`.
|
|
178
|
+
- **Eventually → Rust** (for a Rust client / WASM, and for speed), mirroring
|
|
179
|
+
`praatfan-core-clean`'s Python-first-then-Rust path. The Python implementation
|
|
180
|
+
stays as the readable reference. Porting guidance: `docs/RUST_PORT.md`.
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# PROVENANCE — clean-room record for desphere
|
|
2
|
+
|
|
3
|
+
This document is the paper trail establishing that **desphere contains no
|
|
4
|
+
GPL/LGPL-derived code** and may be licensed under the MIT License. It applies to
|
|
5
|
+
both the Python reference (`src/desphere/`) and the Rust port (`rust/`).
|
|
6
|
+
|
|
7
|
+
## Summary
|
|
8
|
+
|
|
9
|
+
desphere is a NIST SPHERE → RIFF/WAV transcoder (PCM, G.711 μ-law/a-law, and
|
|
10
|
+
embedded-shorten v2 including QLPC and lossless μ-law type-8 with bitshift). It
|
|
11
|
+
was implemented **only** from public specifications and **black-box** testing of
|
|
12
|
+
existing tools. **No GPL/LGPL source code was ever read** at any point.
|
|
13
|
+
|
|
14
|
+
Copyright protects source code — not file formats, facts, or algorithms (the
|
|
15
|
+
idea/expression dichotomy). Building a decoder from a published format/algorithm
|
|
16
|
+
description, and checking it by comparing the *output* of existing binaries, does
|
|
17
|
+
not create a derivative work of those binaries.
|
|
18
|
+
|
|
19
|
+
## Permitted sources actually used
|
|
20
|
+
|
|
21
|
+
- **NIST SPHERE format** — public NIST documentation (the ASCII `NIST_1A` header,
|
|
22
|
+
typed object fields). Used for `sphere.py` / `rust/src/sphere.rs`.
|
|
23
|
+
- **ITU-T G.711** — the public telecom standard defining the μ-law / a-law
|
|
24
|
+
companding tables. Used for `g711.py` / `rust/src/g711.rs`.
|
|
25
|
+
- **Tony Robinson (1994), _SHORTEN: Simple lossless and near-lossless waveform
|
|
26
|
+
compression_, CUED/F-INFENG/TR.156** — the published academic report describing
|
|
27
|
+
the shorten algorithm (Rice/uvar coding, polynomial vs LPC prediction). A copy
|
|
28
|
+
was read; it describes the *algorithm*, not anyone's source code. Used for the
|
|
29
|
+
shorten decoder: `shorten.py` / `rust/src/shorten.rs` (block loop, DIFF/QLPC
|
|
30
|
+
prediction, type-8 mu-law) and the bit primitives in `rust/src/bitreader.rs`
|
|
31
|
+
(`uvar`/`ulong`/`var`). The non-obvious details (the `k=energy+1` Rice quirk,
|
|
32
|
+
the type-8 bitshift remap, QLPC fixed-point) were recovered from oracle OUTPUT,
|
|
33
|
+
not from any decoder source — see below.
|
|
34
|
+
- **Microsoft/IBM RIFF/WAVE** — the public container spec. Used for `wav.py` /
|
|
35
|
+
`rust/src/wav.rs`.
|
|
36
|
+
- **Black-box oracle testing** — `ffmpeg`, `sox`, `sph2pipe`, NIST `w_decode`, and
|
|
37
|
+
the `shorten` encoder were run **as binaries**; only their *output bytes* were
|
|
38
|
+
compared against desphere's output. Running a binary is not reading its source.
|
|
39
|
+
|
|
40
|
+
## Never read (hard rule, no exceptions)
|
|
41
|
+
|
|
42
|
+
- FFmpeg's NIST/shorten decoder (LGPL)
|
|
43
|
+
- the original `shorten` C sources and `sph2pipe` sources
|
|
44
|
+
- any GPL/LGPL codebase, **including the author's own praatfan GPL siblings**
|
|
45
|
+
|
|
46
|
+
The `shorten` **encoder** (Tony Robinson's `drtonyr/shorten`, a non-FOSS license)
|
|
47
|
+
was *downloaded and compiled* to generate test fixtures and act as a second
|
|
48
|
+
black-box oracle. Its `.c/.h` source was **never opened**; the build needed only
|
|
49
|
+
a modern-stdarg shim written from scratch. The build scripts
|
|
50
|
+
`oracles/build_shorten.sh` and `oracles/build_w_decode.sh` are committed (text,
|
|
51
|
+
ours) as evidence; the binaries they produce are gitignored (license-restricted).
|
|
52
|
+
Compiling ≠ reading source.
|
|
53
|
+
|
|
54
|
+
## How the hard parts were derived (reverse-engineering, from oracle output only)
|
|
55
|
+
|
|
56
|
+
The non-obvious shorten details were recovered by comparing reconstructed values
|
|
57
|
+
to oracle output, never by reading a decoder:
|
|
58
|
+
|
|
59
|
+
- The Rice parameter quirk (`k = energy + 1`), the running-mean offset, and bit
|
|
60
|
+
order: validated byte-exact against ffmpeg/sph2pipe output.
|
|
61
|
+
- **Type-8 μ-law + BITSHIFT**: derived as a code-space remap by tabulating
|
|
62
|
+
`sph2pipe -u` (true μ-law bytes) against desphere's reconstructed values; the
|
|
63
|
+
closed form was confirmed by mu-law segment geometry (a G.711 fact) and
|
|
64
|
+
validated byte-exact on real CALLHOME audio. See `docs/SHORTEN.md`.
|
|
65
|
+
- **QLPC**: no corpus file uses it, so the `shorten` encoder (compiled, not read)
|
|
66
|
+
was driven on known synthetic input to produce QLPC streams, and the decode
|
|
67
|
+
was reverse-engineered by matching desphere's output to the known input and to
|
|
68
|
+
`ffmpeg`/`shorten -x`. See `docs/SHORTEN.md` → "QLPC blocks".
|
|
69
|
+
|
|
70
|
+
The git history is itself part of the trail: commits record the oracle
|
|
71
|
+
comparisons, the "never reading decoder source" methodology, and that the encoder
|
|
72
|
+
was "compiled, never reading source."
|
|
73
|
+
|
|
74
|
+
## The Rust port
|
|
75
|
+
|
|
76
|
+
`rust/` is a direct translation of desphere's **own MIT Python** (`src/desphere/`,
|
|
77
|
+
which is the spec), guided by `docs/RUST_PORT.md`. It is validated bit-for-bit
|
|
78
|
+
against the same committed fixtures and shown byte-identical to the Python
|
|
79
|
+
`sph2wav`. No GPL/LGPL source — and no third-party Rust decoder — was consulted.
|
|
80
|
+
The `praatfan-core-clean` sibling was used only for high-level project *layout*
|
|
81
|
+
conventions (a `rust/` crate with `examples/`, pyo3, maturin); its source was not
|
|
82
|
+
read.
|
|
83
|
+
|
|
84
|
+
## Licensing
|
|
85
|
+
|
|
86
|
+
desphere is MIT-licensed (`LICENSE`). The permitted sources above impose no
|
|
87
|
+
copyleft obligation on an independent implementation: file formats and algorithms
|
|
88
|
+
are not copyrightable, ITU/NIST/RIFF specs are public standards, and black-box
|
|
89
|
+
output comparison is not derivation. The non-FOSS `shorten` encoder is a build/
|
|
90
|
+
test tool only — it is not linked into, vendored by, or read for desphere.
|
desphere-0.1.0/README.md
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# desphere
|
|
2
|
+
|
|
3
|
+
**Flatten a sphere.** `desphere` is a clean-room, MIT-licensed, zero-dependency
|
|
4
|
+
transcoder from **NIST SPHERE** audio (the `.sph` format used by TIMIT, WSJ,
|
|
5
|
+
Switchboard, and friends) to plain **RIFF/WAV**. The CLI is `sph2wav`.
|
|
6
|
+
|
|
7
|
+
> **No tooling, no upload:** there's a browser page that converts a `.sph` to
|
|
8
|
+
> `.wav` entirely client-side (WASM) — see [`web/`](web/index.html). Prefer
|
|
9
|
+
> `sph2pipe`/`ffmpeg`/`sox` if you have them; the page is for when you don't.
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install desphere # pure Python, zero deps, works anywhere
|
|
13
|
+
pip install "desphere[fast]" # + optional Rust accelerator (big shorten files)
|
|
14
|
+
|
|
15
|
+
sph2wav utterance.sph # -> utterance.wav
|
|
16
|
+
sph2wav utterance.sph out.wav
|
|
17
|
+
sph2wav --info utterance.sph # inspect the SPHERE header
|
|
18
|
+
sph2wav utterance.sph - # WAV to stdout
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
```python
|
|
22
|
+
from desphere import read_sphere, transcode, transcode_bytes
|
|
23
|
+
|
|
24
|
+
# Streaming API (the reference):
|
|
25
|
+
header, data = read_sphere("utterance.sph")
|
|
26
|
+
with open("utterance.wav", "wb") as f:
|
|
27
|
+
transcode(header, data, f)
|
|
28
|
+
|
|
29
|
+
# One-shot bytes API:
|
|
30
|
+
wav_bytes = transcode_bytes(open("utterance.sph", "rb").read())
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
**Both APIs transparently use the Rust accelerator when installed** (and fall
|
|
34
|
+
back to pure Python) — same bytes out either way. With `desphere[fast]`, the
|
|
35
|
+
heavy kernels (shorten decode, G.711 expansion) run in Rust; pure-Python PCM is
|
|
36
|
+
already C-speed. The CLI passes a stray `.wav` through unchanged (with a warning),
|
|
37
|
+
so pointing it at an already-decoded file is harmless.
|
|
38
|
+
|
|
39
|
+
## Why
|
|
40
|
+
|
|
41
|
+
`libsndfile`/`soundfile` cannot read SPHERE (especially shorten-compressed
|
|
42
|
+
SPHERE), and the tools that can — `sph2pipe`, the original `shorten`, FFmpeg's
|
|
43
|
+
decoder — are GPL/LGPL or otherwise awkwardly licensed. `desphere` is a
|
|
44
|
+
permissively licensed, dependency-free reimplementation.
|
|
45
|
+
|
|
46
|
+
## Clean-room policy
|
|
47
|
+
|
|
48
|
+
`desphere` is implemented **only** from:
|
|
49
|
+
|
|
50
|
+
- The **public NIST SPHERE** format description (ASCII header, typed object fields).
|
|
51
|
+
- **ITU-T G.711** for μ-law / a-law.
|
|
52
|
+
- Tony Robinson (1994), *SHORTEN: Simple lossless and near-lossless waveform
|
|
53
|
+
compression* (CUED/F-INFENG/TR.156) for the shorten algorithm.
|
|
54
|
+
- **Black-box testing**: running `sox` / `ffmpeg` / `sph2pipe` as binaries and
|
|
55
|
+
comparing only their **output** — never reading their source.
|
|
56
|
+
|
|
57
|
+
Copyright protects source code, not file formats or algorithms (the
|
|
58
|
+
idea/expression dichotomy), so consulting *prose/tabular* descriptions of a
|
|
59
|
+
format — even third-party ones — is fine. The one rule we never break: **we do
|
|
60
|
+
not read GPL/LGPL source code, and we do not use writeups that embed verbatim
|
|
61
|
+
GPL source.** Authoritative non-source references (the TR, ITU specs, NIST docs)
|
|
62
|
+
are the primary sources; prose writeups are at most a cross-check.
|
|
63
|
+
|
|
64
|
+
## Supported matrix
|
|
65
|
+
|
|
66
|
+
The design principle is **support the obvious lossless path first, and fail
|
|
67
|
+
loudly on anything not yet validated** — never emit a plausible-but-wrong WAV.
|
|
68
|
+
|
|
69
|
+
| Coding | Status |
|
|
70
|
+
|--------|--------|
|
|
71
|
+
| `pcm`, 16-bit, `01`/`10` byte order | ✅ supported |
|
|
72
|
+
| `pcm`, 32-bit, `01`/`10` byte order | ✅ supported |
|
|
73
|
+
| `pcm`, multi-channel (interleaved) | ✅ supported (validate against an oracle for exotic files) |
|
|
74
|
+
| `pcm`, 8-bit / 24-bit | ⛔ rejected (`UnsupportedFormat`) — sign/packing not yet validated |
|
|
75
|
+
| `ulaw` / `alaw` (G.711, 8-bit) | ✅ supported (Phase B) — verified byte-exact vs ffmpeg on all 256 codes |
|
|
76
|
+
| `pcm,embedded-shorten-v2.00` (16-bit) | ✅ supported (Phase C) — byte-exact vs ffmpeg, mono & stereo |
|
|
77
|
+
| `ulaw,embedded-shorten` (shorten type 8), incl. bitshift | ✅ supported (Phase C) — byte-exact vs sph2pipe (real CALLHOME) |
|
|
78
|
+
| shorten QLPC (LPC) blocks | ✅ supported (Phase C) — byte-exact vs shorten encoder + ffmpeg (orders 1–20) |
|
|
79
|
+
|
|
80
|
+
Adding a coding means registering a decoder in `desphere/codecs.py`; until then
|
|
81
|
+
the gate raises a precise error.
|
|
82
|
+
|
|
83
|
+
## Test fixtures (the "zoo")
|
|
84
|
+
|
|
85
|
+
We don't depend on real corpus files to test: `tools/make_fixtures.py`
|
|
86
|
+
*generates* a zoo of SPHERE variants — every byte order, bit depth, and channel
|
|
87
|
+
count — and commits them under `tests/fixtures/` with a manifest recording the
|
|
88
|
+
exact expected output. Harder codings (μ-law/a-law, eventually shorten) are
|
|
89
|
+
produced best-effort by driving `sox`/`ffmpeg`/`shorten` as black-box binaries.
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
python tools/make_fixtures.py # regenerate fixtures + manifest
|
|
93
|
+
pytest # validate everything
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Rust, WASM, and consumers
|
|
97
|
+
|
|
98
|
+
The Python in `src/desphere` is the reference **spec**; [`rust/`](rust/) is a
|
|
99
|
+
Rust port that reproduces it **bit-for-bit** (same fixtures), for speed and for
|
|
100
|
+
the web. It builds three ways from one crate:
|
|
101
|
+
|
|
102
|
+
- a **Rust library** — a Rust client (praatfan is a likely consumer) imports it
|
|
103
|
+
via a path/git dependency;
|
|
104
|
+
- **WASM** (`wasm-pack build rust --target web --features wasm`) — a WASM web app
|
|
105
|
+
and the `web/` page consume it; nothing is sent to a server;
|
|
106
|
+
- a **Python extension** `desphere-native` (pyo3/maturin) — the optional
|
|
107
|
+
`desphere[fast]` accelerator above.
|
|
108
|
+
|
|
109
|
+
Because the consumers live in the same `ucpresearch` org, they depend on desphere
|
|
110
|
+
directly (path/git); publishing to **crates.io / npm is not required** (and is
|
|
111
|
+
skipped). **PyPI** is the one registry we target (the pure-Python `desphere`,
|
|
112
|
+
plus the optional `desphere-native` wheels).
|
|
113
|
+
|
|
114
|
+
## Development
|
|
115
|
+
|
|
116
|
+
The virtualenv lives **outside** the (Syncthing-synced) repo and is symlinked in:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
uv venv ~/local/scr/venvs/desphere
|
|
120
|
+
ln -s ~/local/scr/venvs/desphere .venv
|
|
121
|
+
uv pip install -e ".[dev]"
|
|
122
|
+
pytest
|
|
123
|
+
|
|
124
|
+
# Rust port:
|
|
125
|
+
cd rust && cargo test # byte-exact vs the same fixtures
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
## Roadmap
|
|
129
|
+
|
|
130
|
+
- **Phase A** (done): SPHERE header + 16/32-bit PCM, lossless.
|
|
131
|
+
- **Phase B** (done): μ-law / a-law decode (ITU-T G.711).
|
|
132
|
+
- **Phase C** (done): embedded-shorten of 16-bit PCM, lossless μ-law (type 8,
|
|
133
|
+
including the non-linear BITSHIFT remap), and QLPC (LPC) blocks — validated
|
|
134
|
+
byte-for-byte vs ffmpeg / sph2pipe / the `shorten` encoder on real and
|
|
135
|
+
synthetic streams, mono and stereo. Remaining (low priority): 8/24-bit linear
|
|
136
|
+
PCM. See `docs/STATUS.md` and `docs/SHORTEN.md`.
|
|
137
|
+
- **Eventually → Rust** (for a Rust client / WASM, and for speed), mirroring
|
|
138
|
+
`praatfan-core-clean`'s Python-first-then-Rust path. The Python implementation
|
|
139
|
+
stays as the readable reference. Porting guidance: `docs/RUST_PORT.md`.
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "desphere"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Clean-room NIST SPHERE (and Shorten) to RIFF/WAV transcoder"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { file = "LICENSE" }
|
|
12
|
+
authors = [{ name = "Uriel Cohen Priva" }]
|
|
13
|
+
keywords = ["sphere", "nist", "shorten", "wav", "riff", "audio", "transcode", "sph2pipe"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Science/Research",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Topic :: Multimedia :: Sound/Audio :: Conversion",
|
|
20
|
+
]
|
|
21
|
+
dependencies = []
|
|
22
|
+
|
|
23
|
+
[project.optional-dependencies]
|
|
24
|
+
dev = ["pytest>=7"]
|
|
25
|
+
# Optional Rust accelerator: `pip install desphere[fast]`. desphere transparently
|
|
26
|
+
# uses it for the slow path (shorten on large files) and falls back otherwise.
|
|
27
|
+
fast = ["desphere-native"]
|
|
28
|
+
|
|
29
|
+
[project.scripts]
|
|
30
|
+
sph2wav = "desphere.cli:main"
|
|
31
|
+
|
|
32
|
+
[project.urls]
|
|
33
|
+
Homepage = "https://github.com/ucpresearch/desphere"
|
|
34
|
+
|
|
35
|
+
[tool.hatch.build.targets.wheel]
|
|
36
|
+
packages = ["src/desphere"]
|
|
37
|
+
|
|
38
|
+
# The pure-Python distribution is just the package + tests + docs. A strict
|
|
39
|
+
# allowlist so the sdist never sweeps in rust/, rust/target/, oracles/, web/, etc.
|
|
40
|
+
# (the isolated build doesn't see .gitignore).
|
|
41
|
+
[tool.hatch.build.targets.sdist]
|
|
42
|
+
only-include = [
|
|
43
|
+
"src/desphere",
|
|
44
|
+
"tests",
|
|
45
|
+
"README.md",
|
|
46
|
+
"LICENSE",
|
|
47
|
+
"PROVENANCE.md",
|
|
48
|
+
"CHANGELOG.md",
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
[tool.pytest.ini_options]
|
|
52
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""desphere — a clean-room NIST SPHERE -> RIFF/WAV transcoder.
|
|
2
|
+
|
|
3
|
+
Flatten a "sphere" (NIST SPHERE audio) into a flat WAV. MIT-licensed,
|
|
4
|
+
zero-dependency, built only from public format documentation and black-box
|
|
5
|
+
testing — never from GPL/LGPL source.
|
|
6
|
+
|
|
7
|
+
Public API::
|
|
8
|
+
|
|
9
|
+
from desphere import read_sphere, transcode, sph_to_wav, SphereHeader
|
|
10
|
+
|
|
11
|
+
header, data = read_sphere("utt.sph")
|
|
12
|
+
with open("utt.wav", "wb") as f:
|
|
13
|
+
transcode(header, data, f)
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from .errors import (
|
|
19
|
+
DesphereError,
|
|
20
|
+
SphereHeaderError,
|
|
21
|
+
UnsupportedCoding,
|
|
22
|
+
UnsupportedFormat,
|
|
23
|
+
)
|
|
24
|
+
from .sphere import SphereHeader
|
|
25
|
+
from .transcode import (
|
|
26
|
+
native_available,
|
|
27
|
+
read_sphere,
|
|
28
|
+
sph_to_wav,
|
|
29
|
+
transcode,
|
|
30
|
+
transcode_bytes,
|
|
31
|
+
)
|
|
32
|
+
from .wav import write_wav
|
|
33
|
+
|
|
34
|
+
__version__ = "0.1.0"
|
|
35
|
+
|
|
36
|
+
__all__ = [
|
|
37
|
+
"__version__",
|
|
38
|
+
"SphereHeader",
|
|
39
|
+
"read_sphere",
|
|
40
|
+
"transcode",
|
|
41
|
+
"transcode_bytes",
|
|
42
|
+
"sph_to_wav",
|
|
43
|
+
"native_available",
|
|
44
|
+
"write_wav",
|
|
45
|
+
"DesphereError",
|
|
46
|
+
"SphereHeaderError",
|
|
47
|
+
"UnsupportedCoding",
|
|
48
|
+
"UnsupportedFormat",
|
|
49
|
+
]
|