pdftts 0.2.0__py3-none-any.whl

pdftts/__init__.py

@@ -0,0 +1,8 @@
+"""pdftts — turn PDFs and pasted text into spoken audio, entirely offline."""
+
+from . import engines
+from .core import Document, from_pdf, from_text, render
+from .device import probe
+
+__version__ = "0.2.0"
+__all__ = ["Document", "from_pdf", "from_text", "render", "engines", "probe"]

pdftts/audio.py

@@ -0,0 +1,139 @@
+"""Post-processing: compressed delivery formats and playback."""
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+
+
+def have_ffmpeg() -> bool:
+    return shutil.which("ffmpeg") is not None
+
+
+#: ISO 639-1 -> 639-2/B. EPUB declares two-letter codes; MP4 stores three, and
+#: ffmpeg drops a two-letter value without a word. Covers every language pdftts
+#: can narrate plus the common European ones a library is likely to contain;
+#: anything unmapped is left off rather than written wrong.
+_ISO_639_2 = {
+    "en": "eng", "es": "spa", "fr": "fre", "hi": "hin", "it": "ita", "pt": "por",
+    "ja": "jpn", "zh": "chi", "de": "ger", "nl": "dut", "ru": "rus", "pl": "pol",
+    "sv": "swe", "da": "dan", "no": "nor", "fi": "fin", "tr": "tur", "ar": "ara",
+    "ko": "kor", "cs": "cze", "el": "gre", "he": "heb", "hu": "hun", "ro": "ron",
+    "uk": "ukr", "vi": "vie", "id": "ind", "th": "tha", "ca": "cat", "la": "lat",
+}
+
+
+def iso_639_2(code: str) -> str:
+    """Three-letter form of a language code, or "" when it cannot be mapped."""
+    code = (code or "").strip().lower().replace("_", "-").split("-")[0]
+    if len(code) == 3:
+        return code
+    return _ISO_639_2.get(code, "")
+
+
+def _tag_args(tags: dict) -> list[str]:
+    """ffmpeg -metadata flags. Language belongs to the audio stream in MP4:
+    passed as a container tag it is silently dropped."""
+    out: list[str] = []
+    for key, value in tags.items():
+        if key == "language":
+            value = iso_639_2(value)
+        if not value:
+            continue
+        out += ["-metadata:s:a:0" if key == "language" else "-metadata", f"{key}={value}"]
+    return out
+
+
+def to_m4a(wav: Path, out: Path | None = None, bitrate: str = "64k", **tags: str) -> Path:
+    """Convert to a tagged m4a — a 47-minute WAV is 130 MB, the m4a is 24 MB."""
+    if not have_ffmpeg():
+        raise RuntimeError("ffmpeg is required for m4a output (brew install ffmpeg)")
+    out = out or wav.with_suffix(".m4a")
+    cmd = ["ffmpeg", "-nostdin", "-loglevel", "error", "-y", "-i", str(wav),
+           "-c:a", "aac", "-b:a", bitrate, "-ac", "1"]
+    cmd += _tag_args(tags)
+    cmd.append(str(out))
+    subprocess.run(cmd, check=True)
+    return out
+
+
+def _cover_file(cover: bytes, near: Path) -> Path | None:
+    """Write jacket bytes beside the output so ffmpeg can read them as an input."""
+    if not cover:
+        return None
+    kind = "png" if cover[:8] == b"\x89PNG\r\n\x1a\n" else "jpg"
+    path = near.with_name(f".{near.stem}-cover.{kind}")
+    path.write_bytes(cover)
+    return path
+
+
+def to_m4b(wav: Path, out: Path | None = None, chapters: str = "",
+           bitrate: str = "64k", cover: bytes = b"", **tags: str) -> Path:
+    """Convert to a chaptered m4b — the format audiobook players expect.
+
+    An m4b remembers your position and exposes chapter navigation; an m4a of the
+    same audio does neither. A jacket image is attached as a still video stream
+    flagged `attached_pic`, which is how players find cover art in an MP4
+    container; without the flag they treat it as a video track and some refuse
+    to play the file at all.
+    """
+    if not have_ffmpeg():
+        raise RuntimeError("ffmpeg is required for m4b output (brew install ffmpeg)")
+    out = out or wav.with_suffix(".m4b")
+    cmd = ["ffmpeg", "-nostdin", "-loglevel", "error", "-y", "-i", str(wav)]
+    meta: Path | None = None
+    art = _cover_file(cover, out)
+    next_input = 1
+    meta_input = art_input = None
+    if chapters:
+        meta = out.with_suffix(".ffmeta")
+        meta.write_text(chapters)
+        cmd += ["-i", str(meta)]
+        meta_input = next_input
+        next_input += 1
+    if art:
+        cmd += ["-i", str(art)]
+        art_input = next_input
+    cmd += ["-map", "0:a"]
+    if art_input is not None:
+        cmd += ["-map", f"{art_input}:v", "-c:v", "mjpeg", "-disposition:v:0", "attached_pic"]
+    if meta_input is not None:
+        cmd += ["-map_metadata", str(meta_input), "-map_chapters", str(meta_input)]
+    cmd += ["-c:a", "aac", "-b:a", bitrate, "-ac", "1"]
+    cmd += _tag_args(tags)
+    cmd.append(str(out))
+    try:
+        subprocess.run(cmd, check=True)
+    finally:
+        for scratch in (meta, art):
+            if scratch and scratch.exists():
+                scratch.unlink()
+    return out
+
+
+def convert(wav: Path, fmt: str, out: Path | None = None, bitrate: str = "64k",
+            **tags: str) -> Path:
+    """Encode to mp3/flac/opus/m4a. Lossless formats ignore the bitrate."""
+    codecs = {"mp3": "libmp3lame", "flac": "flac", "opus": "libopus", "m4a": "aac"}
+    if fmt not in codecs:
+        raise ValueError(f"unsupported format {fmt!r}; choose from {', '.join(codecs)}")
+    if not have_ffmpeg():
+        raise RuntimeError("ffmpeg is required for compressed output")
+    out = out or wav.with_suffix(f".{fmt}")
+    cmd = ["ffmpeg", "-nostdin", "-loglevel", "error", "-y", "-i", str(wav),
+           "-c:a", codecs[fmt], "-ac", "1"]
+    if fmt != "flac":
+        cmd += ["-b:a", bitrate]
+    cmd += _tag_args(tags)
+    cmd.append(str(out))
+    subprocess.run(cmd, check=True)
+    return out
+
+
+def play(path: Path, rate: float = 1.0) -> None:
+    """Play through whatever the platform provides; never fatal."""
+    for cmd in (["afplay", "-r", str(rate), str(path)],
+                ["ffplay", "-nodisp", "-autoexit", "-loglevel", "quiet", str(path)]):
+        if shutil.which(cmd[0]):
+            subprocess.run(cmd)
+            return

pdftts/batch.py

@@ -0,0 +1,81 @@
+"""Convert a whole folder in one run.
+
+A shelf of books is the normal case for anyone who wants an audiobook library,
+and doing it one command at a time means babysitting a machine for hours. The
+rule here is that one bad file must not cost you the queue: every source is
+tried, failures are recorded and reported at the end, and the run keeps going.
+"""
+from __future__ import annotations
+
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+from pathlib import Path
+
+from . import documents
+
+
+@dataclass
+class Result:
+    source: Path
+    out: Path | None = None
+    minutes: float = 0.0
+    reused: int = 0
+    error: str = ""
+
+    @property
+    def ok(self) -> bool:
+        return self.error == ""
+
+
+def sources(target: Path, recursive: bool = False) -> list[Path]:
+    """Every readable document under `target`, in a stable order.
+
+    Hidden files and the artefacts of an earlier run (audio, subtitles) are
+    skipped, so pointing this at the same folder twice does not try to narrate
+    the narrations.
+    """
+    if target.is_file():
+        return [target]
+    walk: Iterable[Path] = target.rglob("*") if recursive else target.iterdir()
+    found = [
+        p for p in walk
+        if p.is_file()
+        and p.suffix.lower() in documents.SUFFIXES
+        and not p.name.startswith(".")
+    ]
+    return sorted(found)
+
+
+def plan(targets: Iterable[Path], recursive: bool = False) -> list[Path]:
+    """Flatten several files and folders into one de-duplicated work list."""
+    seen: dict[Path, None] = {}
+    for target in targets:
+        for path in sources(target, recursive=recursive):
+            seen.setdefault(path.resolve(), None)
+    return list(seen)
+
+
+def summarise(results: list[Result]) -> str:
+    done = [r for r in results if r.ok]
+    failed = [r for r in results if not r.ok]
+    minutes = sum(r.minutes for r in done)
+    reused = sum(r.reused for r in done)
+    line = f"{len(done)}/{len(results)} converted, {minutes:.0f} min of audio"
+    if reused:
+        line += f" ({reused} chunks reused from cache)"
+    if failed:
+        line += "\nfailed:\n" + "\n".join(f"  {r.source.name}: {r.error}" for r in failed)
+    return line
+
+
+def run(paths: list[Path], convert, on_start=None) -> Iterator[Result]:
+    """Apply `convert` to each path, turning any failure into a Result."""
+    for path in paths:
+        if on_start:
+            on_start(path)
+        try:
+            yield convert(path)
+        except KeyboardInterrupt:
+            raise
+        except Exception as exc:
+            yield Result(source=path, error=f"{type(exc).__name__}: {exc}")

pdftts/cache.py

@@ -0,0 +1,130 @@
+"""Per-chunk render cache, so an interrupted book resumes instead of restarting.
+
+A long render is a sequence of independent chunks, each costing real seconds of
+CPU. If the process dies at chunk 900 of 1000 — a laptop lid, a dropped SSH
+session, a Ctrl-C — there is no reason to pay for the first 899 again. Every
+finished chunk is written here keyed by exactly the inputs that determine its
+audio, so a re-run replays what is already on disk and synthesizes only the rest.
+
+The key deliberately includes voice and speed: changing either changes the audio,
+and silently reusing a stale chunk would be worse than re-rendering it.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import shutil
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+
+def _root() -> Path:
+    if sys.platform == "darwin":
+        base = Path.home() / "Library" / "Caches" / "pdftts"
+    elif sys.platform.startswith("win"):
+        base = Path.home() / "AppData" / "Local" / "pdftts" / "cache"
+    else:
+        import os
+
+        base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "pdftts"
+    return base / "chunks"
+
+
+ROOT = _root()
+
+
+def key(text: str, engine: str, voice: str, speed: float, sample_rate: int) -> str:
+    """Everything that changes the audio, and nothing that does not."""
+    seed = "\x1f".join([engine, voice, f"{speed:.4f}", str(sample_rate), text])
+    return hashlib.sha256(seed.encode("utf-8")).hexdigest()[:32]
+
+
+@dataclass
+class Store:
+    """A cache rooted at `root`. Disabled stores are no-ops, so callers need no branches."""
+    root: Path = ROOT
+    enabled: bool = True
+    hits: int = 0
+    misses: int = 0
+
+    def _paths(self, digest: str) -> tuple[Path, Path]:
+        # One subdirectory per two hex chars keeps any single directory small.
+        shard = self.root / digest[:2]
+        return shard / f"{digest}.npy", shard / f"{digest}.json"
+
+    def get(self, digest: str):
+        """Return (samples, words) if this chunk was rendered before, else None."""
+        if not self.enabled:
+            return None
+        wav_path, meta_path = self._paths(digest)
+        if not (wav_path.exists() and meta_path.exists()):
+            self.misses += 1
+            return None
+        try:
+            import numpy as np
+
+            samples = np.load(wav_path)
+            words = [tuple(w) for w in json.loads(meta_path.read_text())["words"]]
+        except Exception:
+            # A half-written or corrupt entry is a miss, never a crash.
+            wav_path.unlink(missing_ok=True)
+            meta_path.unlink(missing_ok=True)
+            self.misses += 1
+            return None
+        self.hits += 1
+        return samples, words
+
+    def put(self, digest: str, samples, words: list) -> None:
+        if not self.enabled:
+            return
+        wav_path, meta_path = self._paths(digest)
+        wav_path.parent.mkdir(parents=True, exist_ok=True)
+        try:
+            import numpy as np
+
+            # Write to a temp name and rename, so a kill mid-write cannot leave
+            # a truncated .npy that later reads back as valid-looking audio.
+            # np.save() appends ".npy" to a *path* that lacks it, which would
+            # defeat the rename — hand it an open file object instead.
+            tmp = wav_path.with_name(wav_path.name + ".part")
+            with tmp.open("wb") as fh:
+                np.save(fh, samples)
+            tmp.replace(wav_path)
+            meta_path.write_text(json.dumps(
+                {"words": [list(w) for w in words], "saved": time.time()}))
+        except Exception:
+            for scratch in (wav_path, meta_path, wav_path.with_name(wav_path.name + ".part")):
+                scratch.unlink(missing_ok=True)
+
+    def usage(self) -> int:
+        if not self.root.exists():
+            return 0
+        return sum(f.stat().st_size for f in self.root.rglob("*") if f.is_file())
+
+    def clear(self) -> int:
+        freed = self.usage()
+        shutil.rmtree(self.root, ignore_errors=True)
+        return freed
+
+
+def prune(older_than_days: float = 30.0, root: Path = ROOT) -> int:
+    """Drop entries untouched for a while. Returns bytes freed."""
+    if not root.exists():
+        return 0
+    cutoff = time.time() - older_than_days * 86_400
+    freed = 0
+    for meta_path in root.rglob("*.json"):
+        try:
+            saved = json.loads(meta_path.read_text()).get("saved", 0)
+        except Exception:
+            saved = 0
+        if saved >= cutoff:
+            continue
+        wav_path = meta_path.with_suffix(".npy")
+        for f in (wav_path, meta_path):
+            if f.exists():
+                freed += f.stat().st_size
+                f.unlink()
+    return freed

pdftts/chapters.py

@@ -0,0 +1,37 @@
+"""Map document chapters onto the finished audio timeline."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class Marker:
+    title: str
+    start: float
+    end: float
+
+    def as_dict(self) -> dict:
+        return {"title": self.title, "start": round(self.start, 3),
+                "end": round(self.end, 3)}
+
+
+def markers(titles: list[str], part_chapter: list[int],
+            part_spans: list[tuple[float, float]]) -> list[Marker]:
+    """Turn per-chunk chapter ownership into one time span per chapter."""
+    out: list[Marker] = []
+    for idx, title in enumerate(titles):
+        mine = [span for span, owner in zip(part_spans, part_chapter) if owner == idx]
+        if not mine:
+            continue
+        out.append(Marker(title, mine[0][0], mine[-1][1]))
+    return out
+
+
+def ffmetadata(marks: list[Marker]) -> str:
+    """ffmpeg chapter metadata — what makes an m4b navigable in a player."""
+    lines = [";FFMETADATA1"]
+    for m in marks:
+        lines += ["[CHAPTER]", "TIMEBASE=1/1000",
+                  f"START={int(m.start * 1000)}", f"END={int(m.end * 1000)}",
+                  f"title={m.title}"]
+    return "\n".join(lines) + "\n"

pdftts/chunk.py

@@ -0,0 +1,99 @@
+"""Split text into synthesizer-sized pieces at real sentence boundaries.
+
+Naive splitting on `[.!?]` breaks inside initials and list markers, which is
+inaudible as text and very audible as speech: the synthesizer treats "…by T. S."
+as a finished sentence, drops its pitch, pauses, and then restarts on "Eliot".
+Across a chapter that reads as constant glitching.
+"""
+from __future__ import annotations
+
+import re
+
+# Kokoro degrades on very long inputs; ~380 characters keeps prosody natural
+# while staying long enough that sentences are not clipped into fragments.
+DEFAULT_LIMIT = 380
+MIN_CHUNK = 60          # anything shorter gets folded into a neighbour
+
+_SENTINEL = "\x00"
+
+# Periods that do NOT end a sentence.
+_ABBREV = re.compile(
+    r"""(?:
+        \b[A-Z]\.                                  # initials: T. S. Eliot
+      | \b(?:Mr|Mrs|Ms|Dr|Prof|Rev|St|Jr|Sr|Hon|Gen|Col|Capt)\.
+      | \b(?:vs|etc|cf|ibid|al|Inc|Ltd|Co)\.
+      | \b(?:i\.e|e\.g|a\.m|p\.m|A\.D|B\.C)\.
+      | \b(?:ch|chap|pp|p|vol|no|fig|ed|trans)\.    # citation shorthand
+    )""",
+    re.X,
+)
+
+# List markers ("1.", "II.") at the start of a line or straight after a sentence.
+# Matched with the preceding context rather than a lookbehind, because the
+# context is variable width; only the marker's own period is neutralised.
+_MARKER = re.compile(r"(^|[.!?][\"'\u201d]?\s)(\s*)((?:\d{1,3}|[IVXivx]{1,5}))\.", re.M)
+
+_SPLIT = re.compile(r"(?<=[.!?])[\"'\u201d\u2019)]?\s+|\n\n+")
+
+
+def _protect(text: str) -> str:
+    text = _ABBREV.sub(lambda m: m.group(0).replace(".", _SENTINEL), text)
+    return _MARKER.sub(lambda m: f"{m.group(1)}{m.group(2)}{m.group(3)}{_SENTINEL}", text)
+
+
+def _restore(text: str) -> str:
+    return text.replace(_SENTINEL, ".")
+
+
+def _merge_runts(parts: list[str], limit: int) -> list[str]:
+    """Fold stubs into a neighbour — a two-word chunk gets read as its own breath."""
+    out: list[str] = []
+    for part in parts:
+        if out and len(part) < MIN_CHUNK and len(out[-1]) + len(part) + 1 <= limit:
+            out[-1] = f"{out[-1]} {part}"
+        else:
+            out.append(part)
+    # A short final chunk has no successor to absorb it, so pull it backwards.
+    if len(out) > 1 and len(out[-1]) < MIN_CHUNK:
+        tail = out.pop()
+        out[-1] = f"{out[-1]} {tail}"
+    return out
+
+
+def chunks(text: str, limit: int = DEFAULT_LIMIT) -> list[str]:
+    protected = _protect(text)
+    out: list[str] = []
+    buf = ""
+
+    for piece in _SPLIT.split(protected):
+        piece = (piece or "").strip()
+        if not piece:
+            continue
+        if len(piece) > limit:                    # a sentence longer than the cap
+            for part in re.split(r"(?<=[,;:])\s+", piece):
+                if len(buf) + len(part) + 1 > limit and buf:
+                    out.append(buf)
+                    buf = part
+                else:
+                    buf = f"{buf} {part}".strip()
+            continue
+        if len(buf) + len(piece) + 1 > limit and buf:
+            out.append(buf)
+            buf = piece
+        else:
+            buf = f"{buf} {piece}".strip()
+    if buf:
+        out.append(buf)
+
+    return [_restore(p) for p in _merge_runts(out, limit)]
+
+
+def sentences(text: str) -> list[str]:
+    """Split into sentences without breaking initials or list markers."""
+    parts = [_restore(p).strip() for p in _SPLIT.split(_protect(text))]
+    return [p for p in parts if p]
+
+
+def estimate_minutes(text: str, speed: float = 1.0) -> float:
+    """Rough runtime, calibrated against measured Kokoro output (~925 chars/min)."""
+    return len(text) / 925.0 / max(speed, 0.1)

pdftts/clean.py

@@ -0,0 +1,100 @@
+"""Turn extracted page text into something worth listening to.
+
+Everything here exists because it was audible: a folio read out mid-sentence,
+a drop cap pronounced as a letter, a vocabulary list run into the next
+paragraph. Print artifacts are invisible on the page and glaring in audio.
+"""
+from __future__ import annotations
+
+import collections
+import re
+
+
+def _folio_key(line: str) -> str:
+    """Normalise a running head so 'Title 175' and 'Title 177' compare equal."""
+    return re.sub(r"^\d{1,4}\s*|\s*\d{1,4}$", "", line).strip()
+
+
+def strip_running_heads(text: str, min_repeats: int = 3) -> str:
+    """Drop page numbers and headers/footers that repeat across pages."""
+    lines = text.split("\n")
+    counts = collections.Counter(_folio_key(l.strip()) for l in lines if l.strip())
+
+    def keep(line: str) -> bool:
+        t = line.strip()
+        if not t:
+            return True
+        if re.fullmatch(r"[ivxlcdm\d]{1,6}[.,]?", t, re.I):     # bare folio
+            return False
+        if len(t) == 1 and not t.isalpha():                     # stray * or dagger
+            return False
+        key = _folio_key(t)
+        return not (key and len(t) < 60 and counts[key] >= min_repeats)
+
+    return "\n".join(l for l in lines if keep(l))
+
+
+def fix_drop_caps(text: str) -> str:
+    """Rejoin a decorative initial split off from its word.
+
+    Scanners read a drop cap as its own line and usually mangle the letter
+    that follows it, so 'W\\nTe are' is really 'We are'.
+    """
+    text = re.sub(r"^([A-Z])\n([A-Z])(?=[a-z])", r"\1", text, flags=re.M)
+    return re.sub(r"^([A-Z])\n(?=[a-z])", r"\1", text, flags=re.M)
+
+
+def punctuate_word_lists(text: str) -> str:
+    """Give vocabulary lists sentence breaks so they are not read as prose."""
+    def repl(m: re.Match) -> str:
+        words = [w.strip() for w in m.group(0).strip().split("\n") if w.strip()]
+        return "\n\n" + ".\n".join(words) + ".\n\n"
+
+    return re.sub(r"(?:^[A-Za-z][\w'-]*\.?$\n){3,}", repl, text, flags=re.M)
+
+
+def strip_footnote_markers(text: str) -> str:
+    """Remove reference numerals that would otherwise be read as digits."""
+    text = re.sub(r"\[\d{1,3}\]", "", text)                 # [12]
+    text = re.sub(r"(?<=[.,;:!?\"'])\s*\d{1,3}(?=\s|$)", "", text)   # trailing 108
+    return re.sub(r"\s+\*(?=\s|$)", "", text)               # lone asterisk
+
+
+def normalize_for_speech(text: str) -> str:
+    """Remove or spell out characters the synthesizer mangles.
+
+    Stray typographic marks left by extraction (bullets, tildes, brackets around
+    editorial insertions) are silent on the page but come out as clicks, pauses
+    or literal words in the audio.
+    """
+    text = text.replace("\u00b7", " ").replace("~", " ").replace("\u2022", " ")
+    text = re.sub(r"[\[\]{}<>|_*^]", " ", text)
+    text = re.sub(r"(?<=\w)/(?=\w)", " or ", text)        # and/or -> and or
+    text = re.sub(r"(?<!\w)/(?!\w)", " ", text)           # stray slashes from bad OCR
+    # Punctuation debris left where extraction mangled a heading, e.g. ( /" —
+    # silent on the page, audible as clicks and false pauses.
+    text = re.sub(r"(?<=\s)[^\w\s]{2,}(?=\s)", " ", text)
+    text = re.sub(r"\(\s*(?=[^\w(]*\s)", " ", text)
+    text = text.replace("&", " and ")
+    text = re.sub(r"\.{3,}", ", ", text)                   # ellipses read as pauses
+    text = re.sub(r"(?<=[a-zA-Z])-{2,}(?=[a-zA-Z])", ", ", text)
+    return re.sub(r"[ \t]{2,}", " ", text)
+
+
+def unwrap(text: str) -> str:
+    """Undo print line-wrapping so sentences reach the synthesizer whole."""
+    text = text.replace("­", "")                       # soft hyphen
+    text = re.sub(r"(\w)-\n(?=\w)", r"\1", text)            # de-hyphenate
+    text = re.sub(r"(?<![.!?:;\n])\n(?!\n)", " ", text)     # join soft wraps
+    text = re.sub(r"[ \t]+", " ", text)
+    return re.sub(r"\n{3,}", "\n\n", text).strip()
+
+
+def clean(text: str) -> str:
+    """Full pipeline, in the order the fixes depend on each other."""
+    text = strip_running_heads(text)
+    text = fix_drop_caps(text)
+    text = punctuate_word_lists(text)
+    text = strip_footnote_markers(text)
+    text = normalize_for_speech(text)
+    return unwrap(text)