desphere 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. desphere-0.1.0/.gitignore +42 -0
  2. desphere-0.1.0/CHANGELOG.md +45 -0
  3. desphere-0.1.0/LICENSE +21 -0
  4. desphere-0.1.0/PKG-INFO +180 -0
  5. desphere-0.1.0/PROVENANCE.md +90 -0
  6. desphere-0.1.0/README.md +139 -0
  7. desphere-0.1.0/pyproject.toml +52 -0
  8. desphere-0.1.0/src/desphere/__init__.py +49 -0
  9. desphere-0.1.0/src/desphere/cli.py +155 -0
  10. desphere-0.1.0/src/desphere/codecs.py +217 -0
  11. desphere-0.1.0/src/desphere/errors.py +34 -0
  12. desphere-0.1.0/src/desphere/g711.py +49 -0
  13. desphere-0.1.0/src/desphere/shorten.py +339 -0
  14. desphere-0.1.0/src/desphere/sphere.py +238 -0
  15. desphere-0.1.0/src/desphere/transcode.py +97 -0
  16. desphere-0.1.0/src/desphere/wav.py +73 -0
  17. desphere-0.1.0/tests/conftest.py +60 -0
  18. desphere-0.1.0/tests/fixtures/alaw_allcodes.sph +0 -0
  19. desphere-0.1.0/tests/fixtures/manifest.json +70 -0
  20. desphere-0.1.0/tests/fixtures/pcm16be_mono.sph +0 -0
  21. desphere-0.1.0/tests/fixtures/pcm16be_stereo.sph +0 -0
  22. desphere-0.1.0/tests/fixtures/pcm16le_mono.sph +0 -0
  23. desphere-0.1.0/tests/fixtures/pcm16le_stereo.sph +0 -0
  24. desphere-0.1.0/tests/fixtures/pcm32be_mono.sph +0 -0
  25. desphere-0.1.0/tests/fixtures/pcm32le_mono.sph +0 -0
  26. desphere-0.1.0/tests/fixtures/pcm8_mono.sph +0 -0
  27. desphere-0.1.0/tests/fixtures/qlpc_ar2.shn +0 -0
  28. desphere-0.1.0/tests/fixtures/qlpc_ar2.wav +0 -0
  29. desphere-0.1.0/tests/fixtures/qlpc_ar2_stereo.shn +0 -0
  30. desphere-0.1.0/tests/fixtures/qlpc_ar2_stereo.wav +0 -0
  31. desphere-0.1.0/tests/fixtures/qlpc_hi.shn +0 -0
  32. desphere-0.1.0/tests/fixtures/qlpc_hi.wav +0 -0
  33. desphere-0.1.0/tests/fixtures/shorten_gate.sph +0 -0
  34. desphere-0.1.0/tests/fixtures/ulaw_allcodes.sph +0 -0
  35. desphere-0.1.0/tests/fixtures/ulaw_bitshift.shn +0 -0
  36. desphere-0.1.0/tests/fixtures/ulaw_bitshift.ulaw +0 -0
  37. desphere-0.1.0/tests/test_api.py +81 -0
  38. desphere-0.1.0/tests/test_failures.py +82 -0
  39. desphere-0.1.0/tests/test_fixtures.py +74 -0
  40. desphere-0.1.0/tests/test_g711.py +43 -0
  41. desphere-0.1.0/tests/test_pcm.py +80 -0
  42. desphere-0.1.0/tests/test_qlpc.py +125 -0
  43. desphere-0.1.0/tests/test_robustness.py +126 -0
  44. desphere-0.1.0/tests/test_shorten_local.py +71 -0
  45. desphere-0.1.0/tests/test_sphere.py +81 -0
  46. desphere-0.1.0/tests/test_ulaw_bitshift.py +66 -0
@@ -0,0 +1,42 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *.egg-info/
5
+ build/
6
+ dist/
7
+ .venv
8
+ .venv/
9
+ venv/
10
+ .pytest_cache/
11
+ .ruff_cache/
12
+
13
+ # Oracle / scratch output (regenerated, not committed)
14
+ *.ref.wav
15
+ /scratch/
16
+
17
+ # Real corpus test files — license-restricted (TIMIT/Switchboard/etc).
18
+ # Synced via Syncthing for cross-machine local testing; never committed.
19
+ /local-fixtures/
20
+
21
+ # Black-box oracle BINARIES — used to validate decode output only; we never read
22
+ # their source. License-restricted, synced via Syncthing, not committed. The
23
+ # build SCRIPTS + README ARE committed as clean-room paper-trail evidence
24
+ # (compiling != reading source); see PROVENANCE.md.
25
+ /oracles/sph2pipe
26
+ /oracles/w_decode
27
+ /oracles/shorten
28
+
29
+ # Local working state — Syncthing-synced, not committed (see CLAUDE.md)
30
+ /memories/
31
+ /plans/
32
+
33
+ # Editor
34
+ *.swp
35
+ .DS_Store
36
+
37
+ # Built WASM for the web page (regenerated; deployed by CI)
38
+ /web/pkg/
39
+
40
+ # Rust build dir (the crate is a workspace member under rust/; cargo builds into
41
+ # the workspace-root target/). Cargo.lock at the root IS committed.
42
+ /target/
@@ -0,0 +1,45 @@
1
+ # Changelog
2
+
3
+ All notable changes to desphere (the Python package and the Rust crate) are
4
+ recorded here. Format loosely follows [Keep a Changelog]; versions are shared
5
+ across the Python `desphere`, the Rust crate `desphere`, and the pyo3 module
6
+ `desphere-native`.
7
+
8
+ ## [0.1.0] — 2026-06-28 (first release)
9
+
10
+ First public release. NIST SPHERE → RIFF/WAV transcoder, clean-room and
11
+ MIT-licensed (see `PROVENANCE.md`).
12
+
13
+ ### Decoding (Python reference + Rust port, byte-for-byte identical)
14
+ - PCM 16/32-bit, byte order `01`/`10`, mono & multi-channel.
15
+ - G.711 μ-law / a-law (ITU-T G.711).
16
+ - Embedded-shorten v2:
17
+ - 16-bit PCM (DIFF0–3, ZERO, VERBATIM, BLOCKSIZE).
18
+ - Lossless μ-law (type 8), including the non-linear **BITSHIFT** code-space
19
+ remap (validated on real CALLHOME audio).
20
+ - **QLPC** (LPC-predicted) blocks (validated vs the shorten encoder + ffmpeg,
21
+ orders 1–20, mono & stereo).
22
+ - Fail-loud on everything unvalidated (8/24-bit PCM, QLPC-less unknown codings,
23
+ corrupt/truncated streams) — never emits a plausible-but-wrong WAV, never
24
+ panics (safe for WASM).
25
+
26
+ ### Packaging
27
+ - Python package `desphere` with the `sph2wav` CLI (pure Python, zero deps). The
28
+ whole decode path — the streaming `transcode()`, the one-shot `transcode_bytes()`,
29
+ and the CLI — transparently uses the optional Rust accelerator when installed
30
+ (`pip install desphere[fast]` → `desphere-native`) and falls back to pure Python
31
+ otherwise — same bytes either way. The accelerator delegates the heavy kernels
32
+ (shorten decode, G.711 expansion); typed error checks stay in Python.
33
+ - Rust crate `desphere` (dependency-free core); Rust clients (e.g. praatfan)
34
+ import it via a path/git dependency.
35
+ - WASM bindings via `wasm-bindgen` (opt-in `wasm` feature) + a self-contained
36
+ client-side `sph2wav` web page (`web/`, deployed to GitHub Pages) — converts in
37
+ the browser, nothing uploaded.
38
+ - Python-native module `desphere-native` via pyo3/maturin (opt-in `python`
39
+ feature).
40
+ - The CLI and web page pass a stray RIFF/WAV input through unchanged, with a
41
+ warning (the library API stays strict: SPHERE in, fail loud).
42
+ - Not published to crates.io / npm (consumers depend on the repo directly); PyPI
43
+ is the target registry.
44
+
45
+ [Keep a Changelog]: https://keepachangelog.com/en/1.1.0/
desphere-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Uriel Cohen Priva
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,180 @@
1
+ Metadata-Version: 2.4
2
+ Name: desphere
3
+ Version: 0.1.0
4
+ Summary: Clean-room NIST SPHERE (and Shorten) to RIFF/WAV transcoder
5
+ Project-URL: Homepage, https://github.com/ucpresearch/desphere
6
+ Author: Uriel Cohen Priva
7
+ License: MIT License
8
+
9
+ Copyright (c) 2026 Uriel Cohen Priva
10
+
11
+ Permission is hereby granted, free of charge, to any person obtaining a copy
12
+ of this software and associated documentation files (the "Software"), to deal
13
+ in the Software without restriction, including without limitation the rights
14
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15
+ copies of the Software, and to permit persons to whom the Software is
16
+ furnished to do so, subject to the following conditions:
17
+
18
+ The above copyright notice and this permission notice shall be included in all
19
+ copies or substantial portions of the Software.
20
+
21
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27
+ SOFTWARE.
28
+ License-File: LICENSE
29
+ Keywords: audio,nist,riff,shorten,sph2pipe,sphere,transcode,wav
30
+ Classifier: Development Status :: 3 - Alpha
31
+ Classifier: Intended Audience :: Science/Research
32
+ Classifier: License :: OSI Approved :: MIT License
33
+ Classifier: Programming Language :: Python :: 3
34
+ Classifier: Topic :: Multimedia :: Sound/Audio :: Conversion
35
+ Requires-Python: >=3.9
36
+ Provides-Extra: dev
37
+ Requires-Dist: pytest>=7; extra == 'dev'
38
+ Provides-Extra: fast
39
+ Requires-Dist: desphere-native; extra == 'fast'
40
+ Description-Content-Type: text/markdown
41
+
42
+ # desphere
43
+
44
+ **Flatten a sphere.** `desphere` is a clean-room, MIT-licensed, zero-dependency
45
+ transcoder from **NIST SPHERE** audio (the `.sph` format used by TIMIT, WSJ,
46
+ Switchboard, and friends) to plain **RIFF/WAV**. The CLI is `sph2wav`.
47
+
48
+ > **No tooling, no upload:** there's a browser page that converts a `.sph` to
49
+ > `.wav` entirely client-side (WASM) — see [`web/`](web/index.html). Prefer
50
+ > `sph2pipe`/`ffmpeg`/`sox` if you have them; the page is for when you don't.
51
+
52
+ ```bash
53
+ pip install desphere # pure Python, zero deps, works anywhere
54
+ pip install "desphere[fast]" # + optional Rust accelerator (big shorten files)
55
+
56
+ sph2wav utterance.sph # -> utterance.wav
57
+ sph2wav utterance.sph out.wav
58
+ sph2wav --info utterance.sph # inspect the SPHERE header
59
+ sph2wav utterance.sph - # WAV to stdout
60
+ ```
61
+
62
+ ```python
63
+ from desphere import read_sphere, transcode, transcode_bytes
64
+
65
+ # Streaming API (the reference):
66
+ header, data = read_sphere("utterance.sph")
67
+ with open("utterance.wav", "wb") as f:
68
+ transcode(header, data, f)
69
+
70
+ # One-shot bytes API:
71
+ wav_bytes = transcode_bytes(open("utterance.sph", "rb").read())
72
+ ```
73
+
74
+ **Both APIs transparently use the Rust accelerator when installed** (and fall
75
+ back to pure Python) — same bytes out either way. With `desphere[fast]`, the
76
+ heavy kernels (shorten decode, G.711 expansion) run in Rust; pure-Python PCM is
77
+ already C-speed. The CLI passes a stray `.wav` through unchanged (with a warning),
78
+ so pointing it at an already-decoded file is harmless.
79
+
80
+ ## Why
81
+
82
+ `libsndfile`/`soundfile` cannot read SPHERE (especially shorten-compressed
83
+ SPHERE), and the tools that can — `sph2pipe`, the original `shorten`, FFmpeg's
84
+ decoder — are GPL/LGPL or otherwise awkwardly licensed. `desphere` is a
85
+ permissively licensed, dependency-free reimplementation.
86
+
87
+ ## Clean-room policy
88
+
89
+ `desphere` is implemented **only** from:
90
+
91
+ - The **public NIST SPHERE** format description (ASCII header, typed object fields).
92
+ - **ITU-T G.711** for μ-law / a-law.
93
+ - Tony Robinson (1994), *SHORTEN: Simple lossless and near-lossless waveform
94
+ compression* (CUED/F-INFENG/TR.156) for the shorten algorithm.
95
+ - **Black-box testing**: running `sox` / `ffmpeg` / `sph2pipe` as binaries and
96
+ comparing only their **output** — never reading their source.
97
+
98
+ Copyright protects source code, not file formats or algorithms (the
99
+ idea/expression dichotomy), so consulting *prose/tabular* descriptions of a
100
+ format — even third-party ones — is fine. The one rule we never break: **we do
101
+ not read GPL/LGPL source code, and we do not use writeups that embed verbatim
102
+ GPL source.** Authoritative non-source references (the TR, ITU specs, NIST docs)
103
+ are the primary sources; prose writeups are at most a cross-check.
104
+
105
+ ## Supported matrix
106
+
107
+ The design principle is **support the obvious lossless path first, and fail
108
+ loudly on anything not yet validated** — never emit a plausible-but-wrong WAV.
109
+
110
+ | Coding | Status |
111
+ |--------|--------|
112
+ | `pcm`, 16-bit, `01`/`10` byte order | ✅ supported |
113
+ | `pcm`, 32-bit, `01`/`10` byte order | ✅ supported |
114
+ | `pcm`, multi-channel (interleaved) | ✅ supported (validate against an oracle for exotic files) |
115
+ | `pcm`, 8-bit / 24-bit | ⛔ rejected (`UnsupportedFormat`) — sign/packing not yet validated |
116
+ | `ulaw` / `alaw` (G.711, 8-bit) | ✅ supported (Phase B) — verified byte-exact vs ffmpeg on all 256 codes |
117
+ | `pcm,embedded-shorten-v2.00` (16-bit) | ✅ supported (Phase C) — byte-exact vs ffmpeg, mono & stereo |
118
+ | `ulaw,embedded-shorten` (shorten type 8), incl. bitshift | ✅ supported (Phase C) — byte-exact vs sph2pipe (real CALLHOME) |
119
+ | shorten QLPC (LPC) blocks | ✅ supported (Phase C) — byte-exact vs shorten encoder + ffmpeg (orders 1–20) |
120
+
121
+ Adding a coding means registering a decoder in `desphere/codecs.py`; until then
122
+ the gate raises a precise error.
123
+
124
+ ## Test fixtures (the "zoo")
125
+
126
+ We don't depend on real corpus files to test: `tools/make_fixtures.py`
127
+ *generates* a zoo of SPHERE variants — every byte order, bit depth, and channel
128
+ count — and commits them under `tests/fixtures/` with a manifest recording the
129
+ exact expected output. Harder codings (μ-law/a-law, eventually shorten) are
130
+ produced best-effort by driving `sox`/`ffmpeg`/`shorten` as black-box binaries.
131
+
132
+ ```bash
133
+ python tools/make_fixtures.py # regenerate fixtures + manifest
134
+ pytest # validate everything
135
+ ```
136
+
137
+ ## Rust, WASM, and consumers
138
+
139
+ The Python in `src/desphere` is the reference **spec**; [`rust/`](rust/) is a
140
+ Rust port that reproduces it **bit-for-bit** (same fixtures), for speed and for
141
+ the web. It builds three ways from one crate:
142
+
143
+ - a **Rust library** — a Rust client (praatfan is a likely consumer) imports it
144
+ via a path/git dependency;
145
+ - **WASM** (`wasm-pack build rust --target web --features wasm`) — a WASM web app
146
+ and the `web/` page consume it; nothing is sent to a server;
147
+ - a **Python extension** `desphere-native` (pyo3/maturin) — the optional
148
+ `desphere[fast]` accelerator above.
149
+
150
+ Because the consumers live in the same `ucpresearch` org, they depend on desphere
151
+ directly (path/git); publishing to **crates.io / npm is not required** (and is
152
+ skipped). **PyPI** is the one registry we target (the pure-Python `desphere`,
153
+ plus the optional `desphere-native` wheels).
154
+
155
+ ## Development
156
+
157
+ The virtualenv lives **outside** the (Syncthing-synced) repo and is symlinked in:
158
+
159
+ ```bash
160
+ uv venv ~/local/scr/venvs/desphere
161
+ ln -s ~/local/scr/venvs/desphere .venv
162
+ uv pip install -e ".[dev]"
163
+ pytest
164
+
165
+ # Rust port:
166
+ cd rust && cargo test # byte-exact vs the same fixtures
167
+ ```
168
+
169
+ ## Roadmap
170
+
171
+ - **Phase A** (done): SPHERE header + 16/32-bit PCM, lossless.
172
+ - **Phase B** (done): μ-law / a-law decode (ITU-T G.711).
173
+ - **Phase C** (done): embedded-shorten of 16-bit PCM, lossless μ-law (type 8,
174
+ including the non-linear BITSHIFT remap), and QLPC (LPC) blocks — validated
175
+ byte-for-byte vs ffmpeg / sph2pipe / the `shorten` encoder on real and
176
+ synthetic streams, mono and stereo. Remaining (low priority): 8/24-bit linear
177
+ PCM. See `docs/STATUS.md` and `docs/SHORTEN.md`.
178
+ - **Eventually → Rust** (for a Rust client / WASM, and for speed), mirroring
179
+ `praatfan-core-clean`'s Python-first-then-Rust path. The Python implementation
180
+ stays as the readable reference. Porting guidance: `docs/RUST_PORT.md`.
@@ -0,0 +1,90 @@
1
+ # PROVENANCE — clean-room record for desphere
2
+
3
+ This document is the paper trail establishing that **desphere contains no
4
+ GPL/LGPL-derived code** and may be licensed under the MIT License. It applies to
5
+ both the Python reference (`src/desphere/`) and the Rust port (`rust/`).
6
+
7
+ ## Summary
8
+
9
+ desphere is a NIST SPHERE → RIFF/WAV transcoder (PCM, G.711 μ-law/a-law, and
10
+ embedded-shorten v2 including QLPC and lossless μ-law type-8 with bitshift). It
11
+ was implemented **only** from public specifications and **black-box** testing of
12
+ existing tools. **No GPL/LGPL source code was ever read** at any point.
13
+
14
+ Copyright protects source code — not file formats, facts, or algorithms (the
15
+ idea/expression dichotomy). Building a decoder from a published format/algorithm
16
+ description, and checking it by comparing the *output* of existing binaries, does
17
+ not create a derivative work of those binaries.
18
+
19
+ ## Permitted sources actually used
20
+
21
+ - **NIST SPHERE format** — public NIST documentation (the ASCII `NIST_1A` header,
22
+ typed object fields). Used for `sphere.py` / `rust/src/sphere.rs`.
23
+ - **ITU-T G.711** — the public telecom standard defining the μ-law / a-law
24
+ companding tables. Used for `g711.py` / `rust/src/g711.rs`.
25
+ - **Tony Robinson (1994), _SHORTEN: Simple lossless and near-lossless waveform
26
+ compression_, CUED/F-INFENG/TR.156** — the published academic report describing
27
+ the shorten algorithm (Rice/uvar coding, polynomial vs LPC prediction). A copy
28
+ was read; it describes the *algorithm*, not anyone's source code. Used for the
29
+ shorten decoder: `shorten.py` / `rust/src/shorten.rs` (block loop, DIFF/QLPC
30
+ prediction, type-8 mu-law) and the bit primitives in `rust/src/bitreader.rs`
31
+ (`uvar`/`ulong`/`var`). The non-obvious details (the `k=energy+1` Rice quirk,
32
+ the type-8 bitshift remap, QLPC fixed-point) were recovered from oracle OUTPUT,
33
+ not from any decoder source — see below.
34
+ - **Microsoft/IBM RIFF/WAVE** — the public container spec. Used for `wav.py` /
35
+ `rust/src/wav.rs`.
36
+ - **Black-box oracle testing** — `ffmpeg`, `sox`, `sph2pipe`, NIST `w_decode`, and
37
+ the `shorten` encoder were run **as binaries**; only their *output bytes* were
38
+ compared against desphere's output. Running a binary is not reading its source.
39
+
40
+ ## Never read (hard rule, no exceptions)
41
+
42
+ - FFmpeg's NIST/shorten decoder (LGPL)
43
+ - the original `shorten` C sources and `sph2pipe` sources
44
+ - any GPL/LGPL codebase, **including the author's own praatfan GPL siblings**
45
+
46
+ The `shorten` **encoder** (Tony Robinson's `drtonyr/shorten`, a non-FOSS license)
47
+ was *downloaded and compiled* to generate test fixtures and act as a second
48
+ black-box oracle. Its `.c/.h` source was **never opened**; the build needed only
49
+ a modern-stdarg shim written from scratch. The build scripts
50
+ `oracles/build_shorten.sh` and `oracles/build_w_decode.sh` are committed (text,
51
+ ours) as evidence; the binaries they produce are gitignored (license-restricted).
52
+ Compiling ≠ reading source.
53
+
54
+ ## How the hard parts were derived (reverse-engineering, from oracle output only)
55
+
56
+ The non-obvious shorten details were recovered by comparing reconstructed values
57
+ to oracle output, never by reading a decoder:
58
+
59
+ - The Rice parameter quirk (`k = energy + 1`), the running-mean offset, and bit
60
+ order: validated byte-exact against ffmpeg/sph2pipe output.
61
+ - **Type-8 μ-law + BITSHIFT**: derived as a code-space remap by tabulating
62
+ `sph2pipe -u` (true μ-law bytes) against desphere's reconstructed values; the
63
+ closed form was confirmed by mu-law segment geometry (a G.711 fact) and
64
+ validated byte-exact on real CALLHOME audio. See `docs/SHORTEN.md`.
65
+ - **QLPC**: no corpus file uses it, so the `shorten` encoder (compiled, not read)
66
+ was driven on known synthetic input to produce QLPC streams, and the decode
67
+ was reverse-engineered by matching desphere's output to the known input and to
68
+ `ffmpeg`/`shorten -x`. See `docs/SHORTEN.md` → "QLPC blocks".
69
+
70
+ The git history is itself part of the trail: commits record the oracle
71
+ comparisons, the "never reading decoder source" methodology, and that the encoder
72
+ was "compiled, never reading source."
73
+
74
+ ## The Rust port
75
+
76
+ `rust/` is a direct translation of desphere's **own MIT Python** (`src/desphere/`,
77
+ which is the spec), guided by `docs/RUST_PORT.md`. It is validated bit-for-bit
78
+ against the same committed fixtures and shown byte-identical to the Python
79
+ `sph2wav`. No GPL/LGPL source — and no third-party Rust decoder — was consulted.
80
+ The `praatfan-core-clean` sibling was used only for high-level project *layout*
81
+ conventions (a `rust/` crate with `examples/`, pyo3, maturin); its source was not
82
+ read.
83
+
84
+ ## Licensing
85
+
86
+ desphere is MIT-licensed (`LICENSE`). The permitted sources above impose no
87
+ copyleft obligation on an independent implementation: file formats and algorithms
88
+ are not copyrightable, ITU/NIST/RIFF specs are public standards, and black-box
89
+ output comparison is not derivation. The non-FOSS `shorten` encoder is a build/
90
+ test tool only — it is not linked into, vendored by, or read for desphere.
@@ -0,0 +1,139 @@
1
+ # desphere
2
+
3
+ **Flatten a sphere.** `desphere` is a clean-room, MIT-licensed, zero-dependency
4
+ transcoder from **NIST SPHERE** audio (the `.sph` format used by TIMIT, WSJ,
5
+ Switchboard, and friends) to plain **RIFF/WAV**. The CLI is `sph2wav`.
6
+
7
+ > **No tooling, no upload:** there's a browser page that converts a `.sph` to
8
+ > `.wav` entirely client-side (WASM) — see [`web/`](web/index.html). Prefer
9
+ > `sph2pipe`/`ffmpeg`/`sox` if you have them; the page is for when you don't.
10
+
11
+ ```bash
12
+ pip install desphere # pure Python, zero deps, works anywhere
13
+ pip install "desphere[fast]" # + optional Rust accelerator (big shorten files)
14
+
15
+ sph2wav utterance.sph # -> utterance.wav
16
+ sph2wav utterance.sph out.wav
17
+ sph2wav --info utterance.sph # inspect the SPHERE header
18
+ sph2wav utterance.sph - # WAV to stdout
19
+ ```
20
+
21
+ ```python
22
+ from desphere import read_sphere, transcode, transcode_bytes
23
+
24
+ # Streaming API (the reference):
25
+ header, data = read_sphere("utterance.sph")
26
+ with open("utterance.wav", "wb") as f:
27
+ transcode(header, data, f)
28
+
29
+ # One-shot bytes API:
30
+ wav_bytes = transcode_bytes(open("utterance.sph", "rb").read())
31
+ ```
32
+
33
+ **Both APIs transparently use the Rust accelerator when installed** (and fall
34
+ back to pure Python) — same bytes out either way. With `desphere[fast]`, the
35
+ heavy kernels (shorten decode, G.711 expansion) run in Rust; pure-Python PCM is
36
+ already C-speed. The CLI passes a stray `.wav` through unchanged (with a warning),
37
+ so pointing it at an already-decoded file is harmless.
38
+
39
+ ## Why
40
+
41
+ `libsndfile`/`soundfile` cannot read SPHERE (especially shorten-compressed
42
+ SPHERE), and the tools that can — `sph2pipe`, the original `shorten`, FFmpeg's
43
+ decoder — are GPL/LGPL or otherwise awkwardly licensed. `desphere` is a
44
+ permissively licensed, dependency-free reimplementation.
45
+
46
+ ## Clean-room policy
47
+
48
+ `desphere` is implemented **only** from:
49
+
50
+ - The **public NIST SPHERE** format description (ASCII header, typed object fields).
51
+ - **ITU-T G.711** for μ-law / a-law.
52
+ - Tony Robinson (1994), *SHORTEN: Simple lossless and near-lossless waveform
53
+ compression* (CUED/F-INFENG/TR.156) for the shorten algorithm.
54
+ - **Black-box testing**: running `sox` / `ffmpeg` / `sph2pipe` as binaries and
55
+ comparing only their **output** — never reading their source.
56
+
57
+ Copyright protects source code, not file formats or algorithms (the
58
+ idea/expression dichotomy), so consulting *prose/tabular* descriptions of a
59
+ format — even third-party ones — is fine. The one rule we never break: **we do
60
+ not read GPL/LGPL source code, and we do not use writeups that embed verbatim
61
+ GPL source.** Authoritative non-source references (the TR, ITU specs, NIST docs)
62
+ are the primary sources; prose writeups are at most a cross-check.
63
+
64
+ ## Supported matrix
65
+
66
+ The design principle is **support the obvious lossless path first, and fail
67
+ loudly on anything not yet validated** — never emit a plausible-but-wrong WAV.
68
+
69
+ | Coding | Status |
70
+ |--------|--------|
71
+ | `pcm`, 16-bit, `01`/`10` byte order | ✅ supported |
72
+ | `pcm`, 32-bit, `01`/`10` byte order | ✅ supported |
73
+ | `pcm`, multi-channel (interleaved) | ✅ supported (validate against an oracle for exotic files) |
74
+ | `pcm`, 8-bit / 24-bit | ⛔ rejected (`UnsupportedFormat`) — sign/packing not yet validated |
75
+ | `ulaw` / `alaw` (G.711, 8-bit) | ✅ supported (Phase B) — verified byte-exact vs ffmpeg on all 256 codes |
76
+ | `pcm,embedded-shorten-v2.00` (16-bit) | ✅ supported (Phase C) — byte-exact vs ffmpeg, mono & stereo |
77
+ | `ulaw,embedded-shorten` (shorten type 8), incl. bitshift | ✅ supported (Phase C) — byte-exact vs sph2pipe (real CALLHOME) |
78
+ | shorten QLPC (LPC) blocks | ✅ supported (Phase C) — byte-exact vs shorten encoder + ffmpeg (orders 1–20) |
79
+
80
+ Adding a coding means registering a decoder in `desphere/codecs.py`; until then
81
+ the gate raises a precise error.
82
+
83
+ ## Test fixtures (the "zoo")
84
+
85
+ We don't depend on real corpus files to test: `tools/make_fixtures.py`
86
+ *generates* a zoo of SPHERE variants — every byte order, bit depth, and channel
87
+ count — and commits them under `tests/fixtures/` with a manifest recording the
88
+ exact expected output. Harder codings (μ-law/a-law, eventually shorten) are
89
+ produced best-effort by driving `sox`/`ffmpeg`/`shorten` as black-box binaries.
90
+
91
+ ```bash
92
+ python tools/make_fixtures.py # regenerate fixtures + manifest
93
+ pytest # validate everything
94
+ ```
95
+
96
+ ## Rust, WASM, and consumers
97
+
98
+ The Python in `src/desphere` is the reference **spec**; [`rust/`](rust/) is a
99
+ Rust port that reproduces it **bit-for-bit** (same fixtures), for speed and for
100
+ the web. It builds three ways from one crate:
101
+
102
+ - a **Rust library** — a Rust client (praatfan is a likely consumer) imports it
103
+ via a path/git dependency;
104
+ - **WASM** (`wasm-pack build rust --target web --features wasm`) — a WASM web app
105
+ and the `web/` page consume it; nothing is sent to a server;
106
+ - a **Python extension** `desphere-native` (pyo3/maturin) — the optional
107
+ `desphere[fast]` accelerator above.
108
+
109
+ Because the consumers live in the same `ucpresearch` org, they depend on desphere
110
+ directly (path/git); publishing to **crates.io / npm is not required** (and is
111
+ skipped). **PyPI** is the one registry we target (the pure-Python `desphere`,
112
+ plus the optional `desphere-native` wheels).
113
+
114
+ ## Development
115
+
116
+ The virtualenv lives **outside** the (Syncthing-synced) repo and is symlinked in:
117
+
118
+ ```bash
119
+ uv venv ~/local/scr/venvs/desphere
120
+ ln -s ~/local/scr/venvs/desphere .venv
121
+ uv pip install -e ".[dev]"
122
+ pytest
123
+
124
+ # Rust port:
125
+ cd rust && cargo test # byte-exact vs the same fixtures
126
+ ```
127
+
128
+ ## Roadmap
129
+
130
+ - **Phase A** (done): SPHERE header + 16/32-bit PCM, lossless.
131
+ - **Phase B** (done): μ-law / a-law decode (ITU-T G.711).
132
+ - **Phase C** (done): embedded-shorten of 16-bit PCM, lossless μ-law (type 8,
133
+ including the non-linear BITSHIFT remap), and QLPC (LPC) blocks — validated
134
+ byte-for-byte vs ffmpeg / sph2pipe / the `shorten` encoder on real and
135
+ synthetic streams, mono and stereo. Remaining (low priority): 8/24-bit linear
136
+ PCM. See `docs/STATUS.md` and `docs/SHORTEN.md`.
137
+ - **Eventually → Rust** (for a Rust client / WASM, and for speed), mirroring
138
+ `praatfan-core-clean`'s Python-first-then-Rust path. The Python implementation
139
+ stays as the readable reference. Porting guidance: `docs/RUST_PORT.md`.
@@ -0,0 +1,52 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "desphere"
7
+ version = "0.1.0"
8
+ description = "Clean-room NIST SPHERE (and Shorten) to RIFF/WAV transcoder"
9
+ readme = "README.md"
10
+ requires-python = ">=3.9"
11
+ license = { file = "LICENSE" }
12
+ authors = [{ name = "Uriel Cohen Priva" }]
13
+ keywords = ["sphere", "nist", "shorten", "wav", "riff", "audio", "transcode", "sph2pipe"]
14
+ classifiers = [
15
+ "Development Status :: 3 - Alpha",
16
+ "Intended Audience :: Science/Research",
17
+ "License :: OSI Approved :: MIT License",
18
+ "Programming Language :: Python :: 3",
19
+ "Topic :: Multimedia :: Sound/Audio :: Conversion",
20
+ ]
21
+ dependencies = []
22
+
23
+ [project.optional-dependencies]
24
+ dev = ["pytest>=7"]
25
+ # Optional Rust accelerator: `pip install desphere[fast]`. desphere transparently
26
+ # uses it for the slow path (shorten on large files) and falls back otherwise.
27
+ fast = ["desphere-native"]
28
+
29
+ [project.scripts]
30
+ sph2wav = "desphere.cli:main"
31
+
32
+ [project.urls]
33
+ Homepage = "https://github.com/ucpresearch/desphere"
34
+
35
+ [tool.hatch.build.targets.wheel]
36
+ packages = ["src/desphere"]
37
+
38
+ # The pure-Python distribution is just the package + tests + docs. A strict
39
+ # allowlist so the sdist never sweeps in rust/, rust/target/, oracles/, web/, etc.
40
+ # (the isolated build doesn't see .gitignore).
41
+ [tool.hatch.build.targets.sdist]
42
+ only-include = [
43
+ "src/desphere",
44
+ "tests",
45
+ "README.md",
46
+ "LICENSE",
47
+ "PROVENANCE.md",
48
+ "CHANGELOG.md",
49
+ ]
50
+
51
+ [tool.pytest.ini_options]
52
+ testpaths = ["tests"]
@@ -0,0 +1,49 @@
1
+ """desphere — a clean-room NIST SPHERE -> RIFF/WAV transcoder.
2
+
3
+ Flatten a "sphere" (NIST SPHERE audio) into a flat WAV. MIT-licensed,
4
+ zero-dependency, built only from public format documentation and black-box
5
+ testing — never from GPL/LGPL source.
6
+
7
+ Public API::
8
+
9
+ from desphere import read_sphere, transcode, sph_to_wav, SphereHeader
10
+
11
+ header, data = read_sphere("utt.sph")
12
+ with open("utt.wav", "wb") as f:
13
+ transcode(header, data, f)
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ from .errors import (
19
+ DesphereError,
20
+ SphereHeaderError,
21
+ UnsupportedCoding,
22
+ UnsupportedFormat,
23
+ )
24
+ from .sphere import SphereHeader
25
+ from .transcode import (
26
+ native_available,
27
+ read_sphere,
28
+ sph_to_wav,
29
+ transcode,
30
+ transcode_bytes,
31
+ )
32
+ from .wav import write_wav
33
+
34
+ __version__ = "0.1.0"
35
+
36
+ __all__ = [
37
+ "__version__",
38
+ "SphereHeader",
39
+ "read_sphere",
40
+ "transcode",
41
+ "transcode_bytes",
42
+ "sph_to_wav",
43
+ "native_available",
44
+ "write_wav",
45
+ "DesphereError",
46
+ "SphereHeaderError",
47
+ "UnsupportedCoding",
48
+ "UnsupportedFormat",
49
+ ]