recito 0.7.2__py3-none-any.whl

recito/__init__.py

@@ -0,0 +1,5 @@
+# __init__.py
+
+from recito._version import __version__, __version_tuple__
+
+__all__ = ["__version__", "__version_tuple__"]

recito/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.7.2'
+__version_tuple__ = version_tuple = (0, 7, 2)
+
+__commit_id__ = commit_id = None

recito/assemble.py

@@ -0,0 +1,608 @@
+"""Stage 5 — ASSEMBLE: cached chunk wavs -> final audiobook file.
+
+Per chunk: leading/trailing silence is trimmed (energy threshold relative
+to peak, tts-audiobook-tool pattern, MIT) so chunk joins aren't choppy.
+Per chapter: chunks and boundary silences are concatenated in memory and
+written as FLAC intermediates, one per chapter, built in parallel.
+
+Every ffmpeg audio codec/filter used here is single-threaded, so wall
+time is governed by the longest serial pass over the audio. All passes
+are therefore kept chapter-sized and run in parallel: each chapter gets
+a two-pass EBU R128 loudnorm (I=-14 / TP=-3 / LRA=7, linear mode
+honoring the true-peak ceiling) measured and applied per chapter, then
+is encoded to the target codec. Chapter-level normalization is audibly
+identical to book-wide for TTS narration (chapter loudness varies by
+~1 LU) while keeping chapter-to-chapter levels consistent. The encoded
+chapter segments are concatenated with stream copy (no re-encode) and
+muxed with ffmetadata chapter marks; AAC encoder priming at each join
+(~50 ms) lands inside the boundary silence.
+
+:class:`ChapterPipeline` drives the per-chapter work; it can run after
+synthesis (:func:`assemble`) or overlapped with it, firing each
+chapter's task as soon as that chapter's audio is final in the chunk
+cache (the output is identical either way).
+
+Metadata is written as classic iTunes-style ilst atoms
+(``mdta``-style ``use_metadata_tags`` is deliberately avoided: most
+players — Cozy, VLC, mutagen-based tools — cannot read it), and a cover
+image is embedded as an attached_pic stream (covr/APIC/PICTURE). mp3 and
+flac outputs are also supported (no chapter marks — container
+limitation).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import shutil
+import subprocess
+from concurrent.futures import Future, ThreadPoolExecutor
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Self
+
+import numpy as np
+import soundfile as sf
+
+from recito.config import Pauses
+from recito.models import BookMeta, Boundary, Chunk, Manifest
+
+#: Silence-trim threshold relative to each chunk's peak RMS (dB).
+_TRIM_THRESHOLD_DB = -42.0
+#: Audio retained before/after the detected speech (seconds).
+_TRIM_PAD_S = 0.04
+
+_LOUDNORM = "I=-14:TP=-3:LRA=7"
+
+
+class AssemblyError(Exception):
+    """Missing audio, ffmpeg failure, or unsupported combination."""
+
+
+@dataclass
+class AssemblyResult:
+    output: Path
+    duration_s: float
+    chapters: list[tuple[str, float]]  # (title, duration seconds)
+
+
+# ---------------------------------------------------------------------------
+# ffmpeg/ffprobe helpers
+# ---------------------------------------------------------------------------
+
+
+def require_ffmpeg() -> None:
+    for tool in ("ffmpeg", "ffprobe"):
+        if shutil.which(tool) is None:
+            raise AssemblyError(
+                f"{tool} not found on PATH; install ffmpeg (see README)"
+            )
+
+
+def _run(cmd: list[str], what: str) -> subprocess.CompletedProcess:
+    # stdin must not be the tty: ffmpeg's interactive keyboard mode would
+    # snapshot/restore termios per process, and with concurrent chapter
+    # tasks (or a child orphaned by an abort) the last restore can be a
+    # stale no-echo snapshot — leaving the terminal echo off after the run.
+    proc = subprocess.run(cmd, capture_output=True, text=True, stdin=subprocess.DEVNULL)
+    if proc.returncode != 0:
+        tail = "\n".join(proc.stderr.strip().splitlines()[-12:])
+        raise AssemblyError(f"{what} failed (exit {proc.returncode}):\n{tail}")
+    return proc
+
+
+def _probe_duration(path: Path) -> float:
+    proc = _run(
+        [
+            "ffprobe",
+            "-v",
+            "error",
+            "-show_entries",
+            "format=duration",
+            "-of",
+            "default=noprint_wrappers=1:nokey=1",
+            str(path),
+        ],
+        f"ffprobe {path.name}",
+    )
+    return float(proc.stdout.strip())
+
+
+def _concat_demuxer(
+    inputs: list[Path], output: Path, extra: list[str], what: str
+) -> None:
+    # Absolute paths: the demuxer resolves relative entries against the
+    # list file's directory, not the process cwd.
+    list_file = output.with_suffix(".concat.txt").absolute()
+    # ffmpeg concat-demuxer quoting: wrap in single quotes, escape inner ones.
+    body = "\n".join(
+        f"file '{str(p.absolute()).replace(chr(39), chr(39) + chr(92) + chr(39) + chr(39))}'"
+        for p in inputs
+    )
+    list_file.write_text(body + "\n")
+    try:
+        _run(
+            [
+                "ffmpeg",
+                "-hide_banner",
+                "-nostats",
+                "-y",
+                "-f",
+                "concat",
+                "-safe",
+                "0",
+                "-i",
+                str(list_file),
+                *extra,
+                str(output.absolute()),
+            ],
+            what,
+        )
+    finally:
+        list_file.unlink(missing_ok=True)
+
+
+# ---------------------------------------------------------------------------
+# Silence trimming
+# ---------------------------------------------------------------------------
+
+
+def trim_silence(wav: np.ndarray, sample_rate: int) -> np.ndarray:
+    """Trim leading/trailing silence using peak-relative frame RMS."""
+    if wav.ndim != 1 or len(wav) == 0:
+        return wav
+    frame = max(1, int(0.025 * sample_rate))
+    n_frames = len(wav) // frame
+    if n_frames < 3:
+        return wav
+    frames = wav[: n_frames * frame].reshape(n_frames, frame)
+    rms = np.sqrt(np.mean(frames * frames, axis=1))
+    peak = float(rms.max())
+    if peak <= 0:
+        return wav
+    threshold = peak * (10.0 ** (_TRIM_THRESHOLD_DB / 20.0))
+    active = np.flatnonzero(rms >= threshold)
+    if len(active) == 0:
+        return wav
+    pad = int(_TRIM_PAD_S * sample_rate)
+    start = max(0, active[0] * frame - pad)
+    end = min(len(wav), (active[-1] + 1) * frame + pad)
+    return wav[start:end]
+
+
+# ---------------------------------------------------------------------------
+# ffmetadata
+# ---------------------------------------------------------------------------
+
+
+def _escape_meta(value: str) -> str:
+    return re.sub(r"([=;#\\\n])", lambda m: "\\" + m.group(1), value)
+
+
+def _ffmetadata(meta: BookMeta, chapters: list[tuple[str, float]]) -> str:
+    out = [";FFMETADATA1"]
+    # iTunes/ilst-style keys; ffmpeg maps them to the classic atoms
+    # (composer = author, artist = narrator: Cozy's author/reader fields).
+    tags = {
+        "title": meta.title,
+        "album": meta.title,  # book-grouping key in players (e.g. Cozy)
+        "artist": meta.narrator,
+        "composer": meta.author,
+        "album_artist": meta.author,
+        "date": meta.date,
+        "genre": meta.genre,
+        "description": meta.description,
+    }
+    for key, value in tags.items():
+        value = value.strip()
+        if value:
+            out.append(f"{key}={_escape_meta(value)}")
+    start_ms = 0
+    for chapter_title, duration in chapters:
+        end_ms = start_ms + round(duration * 1000)
+        out.append("[CHAPTER]")
+        out.append("TIMEBASE=1/1000")
+        out.append(f"START={start_ms}")
+        out.append(f"END={end_ms}")
+        out.append(f"title={_escape_meta(chapter_title)}")
+        start_ms = end_ms
+    return "\n".join(out) + "\n"
+
+
+# ---------------------------------------------------------------------------
+# Loudnorm (two-pass, per chapter)
+# ---------------------------------------------------------------------------
+
+
+def _loudnorm_measure(source: Path) -> dict[str, str]:
+    proc = _run(
+        [
+            "ffmpeg",
+            "-hide_banner",
+            "-nostats",
+            "-i",
+            str(source),
+            "-af",
+            f"loudnorm={_LOUDNORM}:print_format=json",
+            "-f",
+            "null",
+            "-",
+        ],
+        "loudnorm measurement pass",
+    )
+    match = re.search(r"\{[^{}]*\"input_i\"[^{}]*\}", proc.stderr, re.DOTALL)
+    if not match:
+        raise AssemblyError("could not parse loudnorm measurement output")
+    measured = json.loads(match.group(0))
+    return {
+        "measured_I": measured["input_i"],
+        "measured_TP": measured["input_tp"],
+        "measured_LRA": measured["input_lra"],
+        "measured_thresh": measured["input_thresh"],
+        "offset": measured["target_offset"],
+    }
+
+
+def _loudnorm_applied(measured: dict[str, str]) -> str:
+    return (
+        f"loudnorm={_LOUDNORM}:"
+        + ":".join(f"{k}={v}" for k, v in measured.items())
+        + ":linear=true:print_format=summary"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Per-chapter pipeline stages (run in parallel across chapters)
+# ---------------------------------------------------------------------------
+
+
+def _build_chapter(
+    chapter_index: int,
+    entries: list[tuple[str, str]],  # (chunk_id, boundary_after value)
+    cache_dir: Path,
+    chapter_dir: Path,
+    pause_seconds: dict[str, float],  # boundary value -> seconds
+    target_rate: int,
+) -> tuple[int, Path]:
+    """Trim one chapter's chunks, interleave boundary silences, write FLAC."""
+    parts: list[np.ndarray] = []
+    for chunk_id, boundary in entries:
+        wav, rate = sf.read(
+            cache_dir / f"{chunk_id}.wav", dtype="float32", always_2d=False
+        )
+        wav = np.asarray(wav, dtype=np.float32).reshape(-1)
+        wav = trim_silence(wav, rate)
+        if rate != target_rate:
+            raw = chapter_dir / f"{chunk_id}.raw.wav"
+            resampled = chapter_dir / f"{chunk_id}.rs.wav"
+            try:
+                sf.write(raw, wav, rate, format="WAV", subtype="FLOAT")
+                _run(
+                    [
+                        "ffmpeg",
+                        "-hide_banner",
+                        "-nostats",
+                        "-y",
+                        "-i",
+                        str(raw),
+                        "-af",
+                        f"aresample={target_rate}",
+                        "-ar",
+                        str(target_rate),
+                        str(resampled),
+                    ],
+                    f"resample chunk {chunk_id}",
+                )
+                wav = sf.read(resampled, dtype="float32", always_2d=False)[0]
+                wav = np.asarray(wav, dtype=np.float32).reshape(-1)
+            finally:
+                raw.unlink(missing_ok=True)
+                resampled.unlink(missing_ok=True)
+        parts.append(wav)
+        silence_s = pause_seconds.get(boundary, 0.0)
+        if silence_s > 0:
+            parts.append(np.zeros(int(silence_s * target_rate), dtype=np.float32))
+    audio = np.concatenate(parts) if parts else np.zeros(target_rate, dtype=np.float32)
+    chapter_path = chapter_dir / f"chapter_{chapter_index:03d}.flac"
+    # 24-bit FLAC: lossless, matching the previous float32 -> s32 pipeline.
+    sf.write(chapter_path, audio, target_rate, format="FLAC", subtype="PCM_24")
+    return chapter_index, chapter_path
+
+
+def _measure_encode_chapter(
+    chapter_index: int,
+    chapter_path: Path,
+    segment_path: Path,
+    codec: list[str],
+) -> tuple[int, Path]:
+    """Two-pass loudnorm measured and applied per chapter, then encode."""
+    measured = _loudnorm_measure(chapter_path)
+    _run(
+        [
+            "ffmpeg",
+            "-hide_banner",
+            "-nostats",
+            "-y",
+            "-i",
+            str(chapter_path.absolute()),
+            "-af",
+            _loudnorm_applied(measured),
+            *codec,
+            str(segment_path.absolute()),
+        ],
+        f"encode chapter {chapter_index + 1}",
+    )
+    return chapter_index, segment_path
+
+
+def _mux_extra(
+    meta_file: Path, cover: Path | None, fmt: str, rate: int | None = None
+) -> list[str]:
+    """Final-mux args placed after the concat input.
+
+    Input 0 is the concatenated encoded audio (stream-copied, no
+    re-encode); the optional cover image is input 1 (mapped as an
+    attached_pic video stream -> covr/APIC/PICTURE); the ffmetadata file
+    is the last input, so ``-map_metadata`` must track the input count.
+    ``rate`` pins the flac re-encode's sample rate (book-modal).
+    """
+    args: list[str] = []
+    if cover is not None:
+        args += ["-i", str(cover.absolute())]
+    args += ["-f", "ffmetadata", "-i", str(meta_file.absolute())]
+    meta_index = 2 if cover is not None else 1
+    args += ["-map", "0:a"]
+    if cover is not None:
+        cover_codec = (
+            "copy" if cover.suffix.lower() in (".png", ".jpg", ".jpeg") else "mjpeg"
+        )
+        args += ["-map", "1:v", "-c:v", cover_codec, "-disposition:v:0", "attached_pic"]
+    args += ["-map_metadata", str(meta_index)]
+    if fmt == "flac":
+        # Stream copy into a raw FLAC container writes a STREAMINFO header
+        # the demuxer trusts (total samples = first segment only), making
+        # everything past the first segment unreachable. Re-encode instead:
+        # lossless -> lossless is bit-exact, and FLAC encode is fast.
+        args += ["-c:a", "flac", "-compression_level", "5"]
+        if rate is not None:
+            args += ["-ar", str(rate)]
+    else:
+        args += ["-c:a", "copy"]
+    if fmt == "m4b":
+        args += ["-movflags", "+faststart"]
+    return args
+
+
+#: (codec args, segment suffix) per output format; same encoders/bitrates
+#: as the previous single-pass pipeline — only the segmentation changed.
+def _codec_args(fmt: str, target_rate: int) -> tuple[list[str], str]:
+    if fmt == "m4b":
+        return ["-c:a", "aac", "-b:a", "128k", "-ar", "44100"], ".m4a"
+    if fmt == "mp3":
+        return ["-c:a", "libmp3lame", "-b:a", "192k", "-ar", "44100"], ".mp3"
+    if fmt == "flac":
+        return [
+            "-c:a",
+            "flac",
+            "-compression_level",
+            "5",
+            "-ar",
+            str(target_rate),
+        ], ".flac"
+    raise AssemblyError(f"unsupported output format {fmt!r} (use .m4b, .mp3 or .flac)")
+
+
+def _worker_count(n_tasks: int) -> int:
+    return max(1, min(n_tasks, os.cpu_count() or 4))
+
+
+# ---------------------------------------------------------------------------
+# Overlappable per-chapter pipeline
+# ---------------------------------------------------------------------------
+
+
+class ChapterPipeline:
+    """Per-chapter assembly that can overlap synthesis.
+
+    ``chunk_done`` is the ``on_chunk_complete`` hook of
+    :func:`recito.synthesize.synthesize_all`: as each chunk's audio
+    becomes final in the cache, the chapter's pending set shrinks, and
+    once a chapter's unique chunk ids are all cached, its build ->
+    loudnorm -> encode task is submitted to a thread pool — chapters
+    assemble while later chapters are still being synthesized.
+    ``finalize`` waits for the stragglers and muxes the final file.
+
+    The output is identical to non-overlapped assembly: the gate is
+    cache completeness (writes are atomic), chapter intermediates are
+    rebuilt unconditionally on every run, and each chapter's output
+    depends only on the cache contents, the manifest, and the pauses.
+
+    Contract: ``chunk_done`` must be called serially, from one thread
+    (true for every ``synthesize_all`` completion path). Chapter task
+    failures surface at ``finalize``, not mid-synthesis: they are
+    deterministic and re-fail on resume, and letting synthesis finish
+    leaves a warm cache for that resume.
+    """
+
+    def __init__(
+        self,
+        manifest: Manifest,
+        cache_dir: Path,
+        workdir: Path,
+        pauses: Pauses,
+        fmt: str,
+    ) -> None:
+        _, self._seg_ext = _codec_args(fmt, 44100)  # validates fmt early
+        self._fmt = fmt
+        self._manifest = manifest
+        self._cache_dir = cache_dir
+        self._workdir = workdir
+        self._chapter_dir = workdir / "chapters"
+        self._segment_dir = workdir / "segments"
+        self._chapter_dir.mkdir(parents=True, exist_ok=True)
+        self._segment_dir.mkdir(parents=True, exist_ok=True)
+        # Every Boundary value maps to a Pauses field of the same name, so
+        # a new boundary is wired automatically; a Boundary value without
+        # a matching Pauses field fails loudly here instead of silently
+        # assembling 0.0 s pauses.
+        self._pause_seconds = {b.value: getattr(pauses, b.value) for b in Boundary}
+        self._entries: dict[int, list[tuple[str, str]]] = {}
+        for chunk in manifest.chunks:
+            self._entries.setdefault(chunk.chapter_index, []).append(
+                (chunk.id, chunk.boundary_after.value)
+            )
+        # Duplicate chunk ids (identical text) share one cache entry and
+        # may span chapters; synthesis queues only the FIRST occurrence
+        # of an uncached id, so its completion callback carries just one
+        # chapter_index. Completion is therefore tracked per id: one
+        # callback clears the id in EVERY chapter that contains it.
+        self._pending: dict[int, set[str]] = {
+            ci: {cid for cid, _ in entries} for ci, entries in self._entries.items()
+        }
+        self._id_chapters: dict[str, set[int]] = {}
+        for ci, ids in self._pending.items():
+            for cid in ids:
+                self._id_chapters.setdefault(cid, set()).add(ci)
+        workers = _worker_count(len(self._entries))
+        self._pool = ThreadPoolExecutor(max_workers=workers)
+        self._futures: dict[int, Future] = {}
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: object, *_: object) -> None:
+        # On abort, cancel not-yet-started tasks but always wait for
+        # running ones: their ffmpeg/soundfile writes into the workdir
+        # must quiesce before the caller's workdir lock
+        # (main._locked_workdir) is released, or a promptly re-run
+        # `speak` could interleave writes with the orphaned tasks.
+        # Running tasks are chapter-sized ffmpeg passes (seconds each),
+        # so the abort delay stays bounded; chapter intermediates are
+        # rebuilt unconditionally on the resumed run.
+        self._pool.shutdown(wait=True, cancel_futures=exc_type is not None)
+
+    def chunk_done(self, chunk: Chunk) -> None:
+        """Record one cached chunk; fire each chapter whose ids are all in."""
+        for ci in self._id_chapters.get(chunk.id, ()):
+            pending = self._pending[ci]
+            if chunk.id not in pending:
+                continue  # already cleared by an earlier occurrence
+            pending.discard(chunk.id)
+            if not pending:
+                self._futures[ci] = self._pool.submit(
+                    self._assemble_chapter, ci, self._entries[ci]
+                )
+
+    def _assemble_chapter(
+        self, chapter_index: int, entries: list[tuple[str, str]]
+    ) -> int:
+        """Build, loudnorm and encode one chapter. Returns its sample rate."""
+        try:
+            # Chapter-modal rate from headers (chunks are all cached by
+            # construction). Equals the book-modal rate whenever chapters
+            # are internally uniform — the only case seen in practice.
+            rates: dict[int, int] = {}
+            for chunk_id, _ in entries:
+                rate = sf.info(self._cache_dir / f"{chunk_id}.wav").samplerate
+                rates[rate] = rates.get(rate, 0) + 1
+            rate = max(rates, key=rates.get)
+            codec, _ = _codec_args(self._fmt, rate)
+            _, chapter_path = _build_chapter(
+                chapter_index,
+                entries,
+                self._cache_dir,
+                self._chapter_dir,
+                self._pause_seconds,
+                rate,
+            )
+            _measure_encode_chapter(
+                chapter_index,
+                chapter_path,
+                self._segment_dir / f"chapter_{chapter_index:03d}{self._seg_ext}",
+                codec,
+            )
+            return rate
+        except Exception as exc:
+            raise AssemblyError(
+                f"chapter {chapter_index + 1} assembly failed: {exc}"
+            ) from exc
+
+    def finalize(self, output: Path, meta: BookMeta) -> AssemblyResult:
+        """Wait for all chapter tasks, then mux the final audiobook."""
+        unfinished = sorted(set(self._pending) - set(self._futures))
+        if unfinished:
+            raise AssemblyError(
+                f"{len(unfinished)} chapter(s) lack completed audio; "
+                "run the synthesis stage first (re-run `speak` to resume)"
+            )
+        chapter_indexes = sorted(self._futures)
+        # In chapter order: the first task failure propagates.
+        chapter_rates = {ci: self._futures[ci].result() for ci in chapter_indexes}
+        segment_paths = [
+            self._segment_dir / f"chapter_{i:03d}{self._seg_ext}"
+            for i in chapter_indexes
+        ]
+
+        # Chapter marks follow the encoded segments' real durations (which
+        # include per-segment encoder priming, so marks stay on the joins).
+        with ThreadPoolExecutor(
+            max_workers=_worker_count(len(chapter_indexes))
+        ) as pool:
+            segment_durations = list(pool.map(_probe_duration, segment_paths))
+        chapters_meta = list(
+            zip(
+                (self._manifest.chapters[i] for i in chapter_indexes), segment_durations
+            )
+        )
+
+        meta_file = self._workdir / "ffmetadata.txt"
+        meta_file.write_text(_ffmetadata(meta, chapters_meta))
+
+        # Book-modal rate for the flac mux (chunk-weighted, matching the
+        # historical whole-manifest modal rate). Computed now — on an
+        # overlapped run the cache was incomplete at __init__ time.
+        tally: dict[int, int] = {}
+        for ci, rate in chapter_rates.items():
+            tally[rate] = tally.get(rate, 0) + len(self._entries[ci])
+        mux_rate = max(tally, key=tally.get)
+
+        # Final mux: stream copy, metadata + chapter marks + cover, faststart.
+        _concat_demuxer(
+            segment_paths,
+            output,
+            _mux_extra(meta_file, meta.cover, self._fmt, mux_rate),
+            f"mux {output.name}",
+        )
+        duration = _probe_duration(output)
+        return AssemblyResult(
+            output=output, duration_s=duration, chapters=chapters_meta
+        )
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+
+def assemble(
+    manifest: Manifest,
+    cache_dir: Path,
+    output: Path,
+    workdir: Path,
+    pauses: Pauses,
+    meta: BookMeta,
+) -> AssemblyResult:
+    """Assemble the final audiobook from cached chunk audio."""
+    require_ffmpeg()
+    missing = [c for c in manifest.chunks if not (cache_dir / f"{c.id}.wav").exists()]
+    if missing:
+        raise AssemblyError(
+            f"{len(missing)} of {len(manifest.chunks)} chunks lack audio; "
+            "run the synthesis stage first (re-run `speak` to resume)"
+        )
+    fmt = output.suffix.lstrip(".").lower()
+    with ChapterPipeline(manifest, cache_dir, workdir, pauses, fmt) as pipe:
+        for chunk in manifest.chunks:
+            pipe.chunk_done(chunk)
+        return pipe.finalize(output, meta)