hushscript 1.0.0__py3-none-any.whl

hushscript/transcribe/probe.py

@@ -0,0 +1,185 @@
+from __future__ import annotations
+
+import math
+import os
+import struct
+from typing import TYPE_CHECKING, BinaryIO, Final, Protocol, cast
+
+from mutagen import MutagenError
+from mutagen.flac import FLAC
+from mutagen.mp3 import MP3, BitrateMode
+from mutagen.mp4 import MP4
+from mutagen.oggopus import OggOpus
+from mutagen.oggvorbis import OggVorbis
+from mutagen.wave import WAVE
+
+from hushscript._errors import UnsupportedMediaError, describe_failure
+
+from .sniff import MediaFormat, sniff
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from .source import Source
+
+_OPUS_RATE: Final = 48000
+_OPUS_PRE_SKIP_OFFSET: Final = 10
+_MPEG1: Final = 3
+_MPEG2: Final = 2
+_MPEG25: Final = 0
+_LAYER3: Final = 1
+_BITRATES: Final = {
+    _MPEG1: (0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320),
+    _MPEG2: (0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160),
+}
+_SAMPLE_RATES: Final = {
+    _MPEG1: (44100, 48000, 32000),
+    _MPEG2: (22050, 24000, 16000),
+    _MPEG25: (11025, 12000, 8000),
+}
+_ID3_HEADER: Final = 10
+_RESYNC_LIMIT: Final = 64 * 1024
+_FRAME_HEADER: Final = 4
+_SYNC_BYTE: Final = 0xFF
+_SYNC_BITS: Final = 0xE0
+_FIELD_MASK: Final = 3
+_RESERVED_RATE: Final = 3
+
+
+def _unreadable(reason: str) -> UnsupportedMediaError:
+    return UnsupportedMediaError(
+        f"Could not read the duration ({reason}); pass duration_seconds.",
+        code="duration_unreadable",
+    )
+
+
+def _id3_size(handle: BinaryIO) -> int:
+    handle.seek(0)
+    head = handle.read(_ID3_HEADER)
+    if len(head) < _ID3_HEADER or not head.startswith(b"ID3"):
+        return 0
+    size = sum(
+        (byte & 0x7F) << shift for byte, shift in zip(head[6:10], (21, 14, 7, 0), strict=True)
+    )
+    footer = _ID3_HEADER if head[5] & 0x10 else 0
+    return _ID3_HEADER + size + footer
+
+
+def _frame(header: bytes) -> tuple[int, int, int] | None:
+    """(frame length, samples, sample rate) of a Layer III frame header, or None."""
+    if (
+        len(header) < _FRAME_HEADER
+        or header[0] != _SYNC_BYTE
+        or header[1] & _SYNC_BITS != _SYNC_BITS
+    ):
+        return None
+    version = (header[1] >> 3) & _FIELD_MASK
+    layer = (header[1] >> 1) & _FIELD_MASK
+    bitrate_index = header[2] >> 4
+    rate_index = (header[2] >> 2) & _FIELD_MASK
+    if version == 1 or layer != _LAYER3 or bitrate_index in {0, 15} or rate_index == _RESERVED_RATE:
+        return None
+    table = _BITRATES[_MPEG1 if version == _MPEG1 else _MPEG2]
+    rate = _SAMPLE_RATES[version][rate_index]
+    samples = 1152 if version == _MPEG1 else 576
+    length = samples // 8 * table[bitrate_index] * 1000 // rate + ((header[2] >> 1) & 1)
+    return length, samples, rate
+
+
+def scan_mp3_duration(handle: BinaryIO) -> float:
+    """The exact duration of an MP3 from its frames, for a file with no Xing, VBRI or Info header.
+
+    mutagen estimates such a file from the first frame's bitrate, which under-reads a VBR file by
+    up to ten times; declaring that would leave the hold short of the billed duration.
+    """
+    handle.seek(_id3_size(handle))
+    total = 0.0
+    frames = 0
+    skipped = 0
+    while True:
+        header = handle.read(_FRAME_HEADER)
+        if len(header) < _FRAME_HEADER:
+            break
+        frame = _frame(header)
+        if frame is None:
+            skipped += 1
+            if skipped > _RESYNC_LIMIT and frames == 0:
+                break
+            handle.seek(-(_FRAME_HEADER - 1), os.SEEK_CUR)
+            continue
+        length, samples, rate = frame
+        total += samples / rate
+        frames += 1
+        handle.seek(length - _FRAME_HEADER, os.SEEK_CUR)
+    if frames == 0:
+        raise _unreadable("no MPEG frames")
+    return total
+
+
+class _Info(Protocol):
+    length: float
+    bitrate_mode: object
+
+
+class _Parsed(Protocol):
+    info: _Info
+
+
+def _info(parser: Callable[[BinaryIO], object], handle: BinaryIO) -> _Info:
+    """The stream info of a mutagen file; mutagen's own annotations leave it untyped."""
+    return cast("_Parsed", parser(handle)).info
+
+
+def _mp3(handle: BinaryIO) -> float:
+    handle.seek(0)
+    info = _info(MP3, handle)
+    if info.bitrate_mode == BitrateMode.UNKNOWN:
+        return scan_mp3_duration(handle)
+    return float(info.length)
+
+
+def _ogg(handle: BinaryIO) -> float:
+    handle.seek(0)
+    head = handle.read(512)
+    handle.seek(0)
+    marker = head.find(b"OpusHead")
+    if marker < 0:
+        return float(_info(OggVorbis, handle).length)
+    # mutagen takes the pre-skip off the end; counting it keeps the declared value at or above
+    # the container's own duration.
+    (pre_skip,) = struct.unpack_from("<H", head, marker + _OPUS_PRE_SKIP_OFFSET)
+    return float(_info(OggOpus, handle).length) + float(pre_skip) / _OPUS_RATE
+
+
+def _read(kind: MediaFormat | None, handle: BinaryIO) -> float:
+    handle.seek(0)
+    if kind == "mp3":
+        return _mp3(handle)
+    if kind == "isobmff":
+        return float(_info(MP4, handle).length)
+    if kind == "wav":
+        return float(_info(WAVE, handle).length)
+    if kind == "flac":
+        return float(_info(FLAC, handle).length)
+    if kind == "ogg":
+        return _ogg(handle)
+    raise _unreadable("not a container the SDK reads")
+
+
+def probe_duration(source: Source) -> float:
+    """The file's duration in seconds (spec 8.2), read with mutagen.
+
+    Raises UnsupportedMediaError (`duration_unreadable`) when it cannot be read; the caller then
+    needs to pass the duration. The declared value is `ceil` of this, so the probe errs high.
+    """
+    kind = sniff(source.head())
+    with source.probe_file() as handle:
+        if handle is None:
+            raise _unreadable("the input is a stream")
+        try:
+            seconds = _read(kind, handle)
+        except (MutagenError, ValueError, struct.error, EOFError, OSError) as failure:
+            raise _unreadable(describe_failure(failure)) from failure
+    if not math.isfinite(seconds) or seconds <= 0:
+        raise _unreadable("the file reports no length")
+    return seconds

hushscript/transcribe/resume.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import base64
+import binascii
+import json
+import os
+from dataclasses import dataclass
+from typing import Final
+
+from hushscript._errors import ConfigurationError, ResumeMismatchError, describe_failure
+from hushscript._json import as_dict, loads
+
+from .segment import usable_segment
+from .source import FileInput, Source, make_source, path_size
+
+TOKEN_VERSION: Final = 1
+
+
+@dataclass(frozen=True, slots=True)
+class ResumeState:
+    job_id: str
+    idempotency_key: str
+    size: int
+    duration_seconds: int
+
+
+def encode_token(state: ResumeState) -> str:
+    """The opaque resume token: base64url of `{v, jobId, idempotencyKey, size, durationSeconds}`
+    (spec 6.1), the same bytes in every SDK."""
+    payload = {
+        "v": TOKEN_VERSION,
+        "jobId": state.job_id,
+        "idempotencyKey": state.idempotency_key,
+        "size": state.size,
+        "durationSeconds": state.duration_seconds,
+    }
+    raw = json.dumps(payload, separators=(",", ":")).encode()
+    return base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
+
+
+def _whole(value: object) -> int | None:
+    return value if isinstance(value, int) and not isinstance(value, bool) else None
+
+
+def decode_token(token: object) -> ResumeState:
+    invalid = ConfigurationError("This is not a resume token from this SDK.")
+    if not isinstance(token, str):
+        raise invalid
+    try:
+        raw = base64.urlsafe_b64decode(token + "=" * (-len(token) % 4))
+        fields = as_dict(loads(raw))
+    except (ValueError, binascii.Error) as failure:
+        raise ConfigurationError(
+            f"This is not a resume token from this SDK ({describe_failure(failure)})."
+        ) from failure
+    if fields is None or fields.get("v") != TOKEN_VERSION:
+        raise invalid
+    job_id = fields.get("jobId")
+    key = fields.get("idempotencyKey")
+    size = _whole(fields.get("size"))
+    duration = _whole(fields.get("durationSeconds"))
+    if not isinstance(job_id, str) or not isinstance(key, str) or size is None or duration is None:
+        raise invalid
+    if not usable_segment(job_id):
+        raise invalid
+    return ResumeState(job_id=job_id, idempotency_key=key, size=size, duration_seconds=duration)
+
+
+def check_same_file(state: ResumeState, size: int) -> None:
+    """A replay with a different file silently returns the old job (spec 7.3), so it is refused
+    before any request."""
+    if size != state.size:
+        raise ResumeMismatchError(
+            f"The file is {size} bytes; the upload started with {state.size}.",
+            code="resume_file_mismatch",
+        )
+
+
+def early_source(state: ResumeState, file: FileInput | None) -> Source | None:
+    """The source of a resume's file, when it can be had before any request. A path is only sized
+    (a different size is refused now) and opened later, once the job is known to be `uploading`
+    (spec 7.3)."""
+    if file is None:
+        return None
+    if isinstance(file, (str, os.PathLike)):
+        size = path_size(file)
+        if size is not None:
+            check_same_file(state, size)
+        return None
+    source = make_source(file, default_size=state.size)
+    check_same_file(state, source.size)
+    return source
+
+
+def late_source(state: ResumeState, file: FileInput | None, source: Source | None) -> Source:
+    """The source for the upload a resume has to finish."""
+    if source is not None:
+        return source
+    if file is None:
+        raise file_required()
+    opened = make_source(file, default_size=state.size)
+    check_same_file(state, opened.size)
+    return opened
+
+
+def file_required() -> ResumeMismatchError:
+    return ResumeMismatchError(
+        "The upload was not finished and no file was given.", code="resume_file_required"
+    )
+
+
+def job_missing() -> ResumeMismatchError:
+    return ResumeMismatchError(
+        "The job behind this resume token no longer exists.", code="resume_job_missing"
+    )

hushscript/transcribe/segment.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from urllib.parse import quote
+
+_DOT_SEGMENTS = frozenset({"", ".", ".."})
+
+
+def usable_segment(value: str) -> bool:
+    """Whether an id the server or a resume token gave can stand as one path segment: not empty,
+    not a dot segment (`GET /v1/jobs/..` is `GET /v1`), no slash or backslash, valid UTF-8."""
+    if value in _DOT_SEGMENTS or "/" in value or "\\" in value:
+        return False
+    try:
+        value.encode()
+    except UnicodeEncodeError:
+        return False
+    return True
+
+
+def segment(value: str) -> str:
+    """`value` percent-encoded as one path segment; callers check `usable_segment` first where the
+    id comes from outside."""
+    return quote(value, safe="")

hushscript/transcribe/session.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import httpx
+
+    from hushscript._clock import Clock
+    from hushscript._log import RequestLog
+    from hushscript._redact import Credentials
+
+
+@dataclass(frozen=True, slots=True)
+class Session:
+    """What the pipeline needs from a client. `http` talks to the API (the retrying transport adds
+    the token); `r2` is a second client with no token and no base URL, for the signed direct part
+    URLs, which are never API requests."""
+
+    clock: Clock
+    credentials: Credentials
+    log: RequestLog
+
+
+@dataclass(frozen=True, slots=True)
+class SyncSession(Session):
+    http: httpx.Client
+    r2: httpx.Client
+
+
+@dataclass(frozen=True, slots=True)
+class AsyncSession(Session):
+    http: httpx.AsyncClient
+    r2: httpx.AsyncClient

hushscript/transcribe/sniff.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from typing import Final, Literal
+
+MediaFormat = Literal["mp3", "isobmff", "wav", "flac", "ogg"]
+
+HEAD_BYTES: Final = 4096
+_RIFF: Final = (b"RIFF", b"RF64", b"BW64")
+_FRAME_SYNC_MASK: Final = 0xE0
+
+
+def sniff(head: bytes) -> MediaFormat | None:
+    """The container of `head` by its magic bytes (spec 8.1), never by the file name.
+
+    Matches `backend/src/audio_probe.rs`: MP3 (an ID3 tag or a frame sync), ISO-BMFF (`ftyp` at
+    offset 4: m4a, mp4, mov), WAV, FLAC and Ogg. Anything else is None.
+    """
+    if head.startswith(b"ID3") or (
+        len(head) >= 2 and head[0] == 0xFF and head[1] & _FRAME_SYNC_MASK == _FRAME_SYNC_MASK  # noqa: PLR2004
+    ):
+        return "mp3"
+    if len(head) >= 12 and head[4:8] == b"ftyp":  # noqa: PLR2004
+        return "isobmff"
+    if len(head) >= 12 and head[:4] in _RIFF and head[8:12] == b"WAVE":  # noqa: PLR2004
+        return "wav"
+    if head.startswith(b"fLaC"):
+        return "flac"
+    if head.startswith(b"OggS"):
+        return "ogg"
+    return None

hushscript/transcribe/source.py

@@ -0,0 +1,218 @@
+from __future__ import annotations
+
+import io
+import os
+import stat
+import threading
+from contextlib import contextmanager, nullcontext
+from typing import TYPE_CHECKING, BinaryIO, Final
+
+from hushscript._errors import PreflightError, UploadError, describe_failure
+
+from .sniff import HEAD_BYTES
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+    from contextlib import AbstractContextManager
+
+FileInput = str | os.PathLike[str] | bytes | bytearray | memoryview | BinaryIO
+
+
+def unreadable(path: str, reason: str) -> PreflightError:
+    return PreflightError(f"The file {path} cannot be read: {reason}.", code="file_unreadable")
+
+
+def path_size(file: FileInput) -> int | None:
+    """The size of a path input, without opening it; None for any other input or when the path
+    cannot be sized (the error is raised later, when the file is opened)."""
+    if not isinstance(file, (str, os.PathLike)):
+        return None
+    try:
+        status = os.stat(file)  # noqa: PTH116
+    except OSError:
+        return None
+    return status.st_size if stat.S_ISREG(status.st_mode) else None
+
+
+def read_full(stream: BinaryIO, length: int) -> bytes:
+    """Up to `length` bytes, reading on until they are all there or the stream ends. A raw
+    unbuffered stream (a pipe, `io.FileIO`) may return fewer bytes than asked without being at
+    its end, so one short read is not EOF."""
+    data = stream.read(length)
+    if len(data) >= length or not data:
+        return data
+    chunks = [data]
+    remaining = length - len(data)
+    while remaining > 0:
+        chunk = stream.read(remaining)
+        if not chunk:
+            break
+        chunks.append(chunk)
+        remaining -= len(chunk)
+    return b"".join(chunks)
+
+
+class Source:
+    """A file the pipeline reads in byte ranges: from a path, from memory or from a stream."""
+
+    size: int
+    sequential: bool = False
+    """True for a stream that cannot seek: its parts must be read one after the other."""
+    is_stream: bool = False
+
+    def read_range(self, offset: int, length: int) -> bytes:
+        raise NotImplementedError
+
+    def head(self) -> bytes:
+        return self.read_range(0, min(HEAD_BYTES, self.size))
+
+    def probe_file(self) -> AbstractContextManager[BinaryIO | None]:
+        """A seekable binary file for the duration probe, or None when there is none."""
+        return nullcontext(None)
+
+    def at_end(self) -> bool:
+        """True when nothing is left past `size`; a stream that holds more than it declared is
+        not (spec 7.3)."""
+        return True
+
+
+class PathSource(Source):
+    def __init__(self, path: str | os.PathLike[str]) -> None:
+        self._path = os.fspath(path)
+        try:
+            status = os.stat(self._path)  # noqa: PTH116  # the plain os API keeps str paths
+            if not stat.S_ISREG(status.st_mode):
+                raise unreadable(self._path, "it is not a regular file")
+            with open(self._path, "rb"):  # noqa: PTH123  # proves the file can be opened
+                pass
+        except OSError as failure:
+            raise PreflightError(
+                f"The file {self._path} cannot be read ({describe_failure(failure)}).",
+                code="file_unreadable",
+            ) from failure
+        self.size = status.st_size
+
+    def read_range(self, offset: int, length: int) -> bytes:
+        with open(self._path, "rb") as handle:  # noqa: PTH123
+            handle.seek(offset)
+            return handle.read(length)
+
+    def probe_file(self) -> AbstractContextManager[BinaryIO | None]:
+        return open(self._path, "rb")  # noqa: PTH123  # the caller enters the context
+
+
+class BytesSource(Source):
+    def __init__(self, data: bytes) -> None:
+        self._data = data
+        self.size = len(data)
+
+    def read_range(self, offset: int, length: int) -> bytes:
+        return self._data[offset : offset + length]
+
+    def probe_file(self) -> AbstractContextManager[BinaryIO | None]:
+        return nullcontext(io.BytesIO(self._data))
+
+
+class SeekableSource(Source):
+    """A seekable stream. Reads take a lock, so parts can be read from several threads."""
+
+    is_stream = True
+
+    def __init__(self, stream: BinaryIO, size: int | None) -> None:
+        self._stream = stream
+        self._lock = threading.Lock()
+        if size is None:
+            with self._lock:
+                size = stream.seek(0, os.SEEK_END)
+        self.size = size
+
+    def read_range(self, offset: int, length: int) -> bytes:
+        with self._lock:
+            self._stream.seek(offset)
+            return read_full(self._stream, length)
+
+    def at_end(self) -> bool:
+        with self._lock:
+            self._stream.seek(self.size)
+            return self._stream.read(1) == b""
+
+    def probe_file(self) -> AbstractContextManager[BinaryIO | None]:
+        @contextmanager
+        def locked() -> Generator[BinaryIO]:
+            with self._lock:
+                self._stream.seek(0)
+                yield self._stream
+
+        return locked()
+
+
+class SequentialSource(Source):
+    """A stream that cannot seek. It is read once, in order, with at most one part buffered."""
+
+    sequential = True
+    is_stream = True
+
+    def __init__(self, stream: BinaryIO, size: int) -> None:
+        self._stream = stream
+        self.size = size
+        self._position = 0
+        self._head = b""
+        self._head_taken = False
+
+    def head(self) -> bytes:
+        if not self._head_taken:
+            self._head = read_full(self._stream, min(HEAD_BYTES, self.size))
+            self._head_taken = True
+        return self._head
+
+    def read_range(self, offset: int, length: int) -> bytes:
+        if offset != self._position:
+            raise UploadError(
+                "A stream can only be uploaded in order, one part at a time.",
+                code="stream_not_seekable",
+            )
+        head = self.head()
+        if offset >= len(head):
+            data = read_full(self._stream, length)
+        else:
+            data = head[offset : offset + length]
+            if len(data) < length:
+                data += read_full(self._stream, length - len(data))
+        self._position += len(data)
+        return data
+
+    def at_end(self) -> bool:
+        return self._stream.read(1) == b""
+
+
+NEEDS_SIZE: Final = (
+    "A stream that cannot seek needs `size` (its length in bytes) and `duration_seconds`."
+)
+
+
+def check_size(size: object) -> None:
+    """A stream `size` is a positive whole number of bytes (spec 7.3)."""
+    if size is not None and (not isinstance(size, int) or isinstance(size, bool) or size < 1):
+        raise PreflightError(
+            "size must be a positive whole number of bytes.", code="invalid_option"
+        )
+
+
+def make_source(
+    file: FileInput, size: int | None = None, *, default_size: int | None = None
+) -> Source:
+    """Wraps a path, bytes or a binary stream. `size` is required only for a non-seekable stream;
+    `default_size` stands in for it there (a resume knows the size from its token) and is ignored
+    for a stream that can seek, which is measured."""
+    if isinstance(file, (str, os.PathLike)):
+        return PathSource(file)
+    if isinstance(file, (bytes, bytearray, memoryview)):
+        return BytesSource(bytes(file))
+    check_size(size)
+    stream = file
+    if stream.seekable():
+        return SeekableSource(stream, size)
+    size = default_size if size is None else size
+    if size is None:
+        raise PreflightError(NEEDS_SIZE, code="invalid_option")
+    return SequentialSource(stream, size)