hushscript 1.0.0__py3-none-any.whl

hushscript/transcribe/heartbeat.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import threading
+from typing import TYPE_CHECKING, Final
+
+from hushscript._errors import AuthenticationError, ConflictError, HushscriptError
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+
+    from hushscript._clock import Clock
+    from hushscript._log import RequestLog
+
+HEARTBEAT_INTERVAL: Final = 120.0
+WRONG_STATE: Final = "wrong_state"
+
+
+class Heartbeat:
+    """Keeps an upload alive: a ping every 120 s from create until complete (spec 8.5).
+
+    A failed ping was already retried by the transport and is otherwise ignored, whatever it
+    raised: only a clean `wrong_state` or `unauthorized` ends the loop. `wrong_state`
+    ends the loop (the job has moved on); `unauthorized` ends it and is kept in `error` for the
+    upload to raise.
+    """
+
+    def __init__(self, clock: Clock, log: RequestLog | None = None) -> None:
+        self._clock = clock
+        self._log = log
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+        self.error: AuthenticationError | None = None
+
+    def _keep_going(self, failure: HushscriptError) -> bool:
+        if isinstance(failure, ConflictError) and failure.code == WRONG_STATE:
+            return False
+        if isinstance(failure, AuthenticationError):
+            self.error = failure
+            return False
+        return True
+
+    def _unexpected(self, failure: Exception) -> None:
+        if self._log is not None:
+            self._log.note(f"heartbeat ping failed ({type(failure).__name__}: {failure})")
+
+    def run(self, ping: Callable[[], None]) -> None:
+        while not self._clock.wait(self._stop, HEARTBEAT_INTERVAL):
+            try:
+                ping()
+            except HushscriptError as failure:  # noqa: PERF203  # recover per ping
+                if not self._keep_going(failure):
+                    return
+            except Exception as failure:  # noqa: BLE001  # a ping that breaks must not end the beat
+                self._unexpected(failure)
+
+    async def arun(self, ping: Callable[[], Awaitable[None]]) -> None:
+        while True:
+            await self._clock.asleep(HEARTBEAT_INTERVAL)
+            try:
+                await ping()
+            except HushscriptError as failure:
+                if not self._keep_going(failure):
+                    return
+            except Exception as failure:  # noqa: BLE001  # a ping that breaks must not end the beat
+                self._unexpected(failure)
+
+    def start(self, ping: Callable[[], None]) -> None:
+        self._thread = threading.Thread(target=self.run, args=(ping,), daemon=True)
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._stop.set()
+        if self._thread is not None:
+            self._thread.join(timeout=5)
+
+    def raise_if_failed(self) -> None:
+        if self.error is not None:
+            raise self.error

hushscript/transcribe/pipeline.py

@@ -0,0 +1,312 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Final, TypeVar
+
+from hushscript._errors import (
+    HushscriptError,
+    NotFoundError,
+    add_note,
+    raise_for_response,
+    unexpected_error,
+)
+from hushscript._models import Job, UploadingProgress
+from hushscript._response import json_body, parsed
+
+from .heartbeat import Heartbeat
+from .preflight import Media, Options, check_account, check_media, create_body
+from .resume import (
+    ResumeState,
+    decode_token,
+    early_source,
+    encode_token,
+    file_required,
+    job_missing,
+    late_source,
+)
+from .segment import segment
+from .source import FileInput, Source, make_source
+from .upload import SyncUploader, parse_created, stream_size_mismatch
+from .wait import finish, job_from, transcript_of, wait_for_job
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Mapping
+
+    from hushscript._models import Progress
+    from hushscript._redact import Credentials
+    from hushscript.transcript import Transcript
+
+    from .session import SyncSession
+    from .upload import Created
+
+UPLOADING: Final = "uploading"
+RESUME_NOTE: Final = "The job exists on the server. Resume it with this token: "
+SENT_NOTE: Final = "The create request was sent, so the job may exist. Idempotency key: "
+T = TypeVar("T")
+
+
+@dataclass(slots=True)
+class Tracker:
+    """What an error raised after the create was sent has to carry (spec 7.3)."""
+
+    key: str
+    token: str | None = None
+    sent: bool = False
+    """True from the moment the create goes out: the server may have committed it."""
+    callers: list[BaseException] = field(default_factory=list)
+    """What the caller's own callbacks raised: those pass through unchanged (R63)."""
+    credentials: Credentials | None = None
+    """What an unexpected failure's cause is redacted with."""
+
+    def note(self) -> str:
+        if self.token is None:
+            return f"{SENT_NOTE}{self.key}"
+        return f"{RESUME_NOTE}{self.token} (idempotency key {self.key})"
+
+    def from_caller(self, failure: BaseException) -> bool:
+        return any(failure is raised for raised in self.callers)
+
+
+def note_interrupt(tracker: Tracker, interrupted: BaseException) -> None:
+    """Spec 7.4 and R26: an interrupt is never converted (asyncio.timeout and task groups depend on
+    it); once the create is out, the way back to the job rides along as a note."""
+    if tracker.sent:
+        add_note(interrupted, tracker.note())
+
+
+def annotate(tracker: Tracker, failure: Exception) -> HushscriptError:
+    """What an error raised after the create went out turns into. Called from the handler of
+    `failure`: what passes through unchanged is re-raised from here, and the error returned is for
+    the caller to raise once its handler has ended, so the unredacted original is not left on the
+    error's `__context__`."""
+    # R63 first: what the caller's own callbacks raised, an SDK error among it (a nested `buy()`
+    # failing with its own key), reaches the caller as it is.
+    if tracker.from_caller(failure):
+        raise failure
+    if isinstance(failure, HushscriptError):
+        failure.idempotency_key = tracker.key
+        if tracker.token is not None and failure.resume_token is None:
+            failure.resume_token = tracker.token
+        raise failure
+    # R61: once the create is out nothing but an SDK error leaves.
+    if not tracker.sent:
+        raise failure
+    failed = unexpected_error(failure, tracker.credentials, sent=True)
+    failed.idempotency_key = tracker.key
+    failed.resume_token = tracker.token
+    return failed
+
+
+def annotated(tracker: Tracker, run: Callable[[], T]) -> T:
+    """`run()`, where every failure after the create went out is an SDK error that carries the key
+    and the token (spec 7.3)."""
+    try:
+        return run()
+    except (asyncio.CancelledError, KeyboardInterrupt) as interrupted:
+        note_interrupt(tracker, interrupted)
+        raise
+    except Exception as failure:  # noqa: BLE001  # R61: nothing but an SDK error leaves
+        failed = annotate(tracker, failure)
+    raise failed
+
+
+def replay_body(state: ResumeState) -> Mapping[str, object]:
+    return {
+        "size_bytes": state.size,
+        "duration_seconds": state.duration_seconds,
+        "idempotency_key": state.idempotency_key,
+    }
+
+
+@dataclass(frozen=True, slots=True)
+class Callbacks:
+    on_progress: Callable[[Progress], None] | None = None
+    on_resume_token: Callable[[str], None] | None = None
+    raised: list[BaseException] = field(default_factory=list)
+    """Whatever the callbacks raised, so the pipeline can tell it from a failure of its own."""
+
+    def progress(self, event: Progress) -> None:
+        if self.on_progress is not None:
+            try:
+                self.on_progress(event)
+            except Exception as failure:
+                self.raised.append(failure)
+                raise
+
+    def resume_token(self, token: str) -> None:
+        if self.on_resume_token is not None:
+            try:
+                self.on_resume_token(token)
+            except Exception as failure:
+                self.raised.append(failure)
+                raise
+
+
+def percent(sent: int, total: int) -> int:
+    return min(100, sent * 100 // total) if total else 100
+
+
+def create(session: SyncSession, body: Mapping[str, object]) -> Created:
+    """`POST /v1/uploads`; the transport paces it to the 4 per minute the API allows (spec 7.5)."""
+    response = session.http.post("/v1/uploads", json=body)
+    return parsed(response, session.credentials, parse_created)
+
+
+def get_job(session: SyncSession, job_segment: str) -> Job:
+    return job_from(session.http.get(f"/v1/jobs/{job_segment}"), session.credentials)
+
+
+def wait_job(
+    session: SyncSession,
+    job_segment: str,
+    wait_timeout: float,
+    callbacks: Callbacks,
+    token: str | None,
+    first: Job | None = None,
+) -> Job:
+    return wait_for_job(
+        lambda: get_job(session, job_segment),
+        session.clock,
+        wait_timeout,
+        callbacks.progress,
+        token,
+        first,
+    )
+
+
+def fetch_transcript(session: SyncSession, job: Job) -> Transcript:
+    transcript_id = finish(job)
+    return transcript_of(
+        session.http.get(f"/v1/transcripts/{segment(transcript_id)}"), session.credentials
+    )
+
+
+def _upload(
+    session: SyncSession,
+    created: Created,
+    source: Source,
+    concurrency: int,
+    callbacks: Callbacks,
+) -> None:
+    job_path = f"/v1/uploads/{segment(created.job_id)}"
+    uploader = SyncUploader(session, created.job_id, created.plan, source, concurrency)
+    heartbeat = Heartbeat(session.clock, session.log)
+
+    def ping() -> None:
+        raise_for_response(
+            session.http.post(f"{job_path}/heartbeat"), credentials=session.credentials
+        )
+
+    heartbeat.start(ping)
+    try:
+        total = source.size
+        uploader.upload(
+            lambda sent: callbacks.progress(UploadingProgress(percent(sent, total), sent, total)),
+            heartbeat,
+        )
+        heartbeat.raise_if_failed()
+        if not source.at_end():
+            raise stream_size_mismatch()
+        body = uploader.state.parts_body()
+        response = (
+            session.http.post(f"{job_path}/complete")
+            if body is None
+            else session.http.post(f"{job_path}/complete", json=body)
+        )
+        raise_for_response(response, credentials=session.credentials)
+    finally:
+        heartbeat.stop()
+
+
+def _after_create(  # the upload, wait and fetch that follow a create, for a new call and a resume
+    session: SyncSession,
+    created: Created,
+    tracker: Tracker,
+    state: ResumeState,
+    source: Source | None,
+    options: Options,
+    callbacks: Callbacks,
+) -> Transcript:
+    token = encode_token(
+        ResumeState(created.job_id, state.idempotency_key, state.size, state.duration_seconds)
+    )
+    tracker.token = token
+    callbacks.resume_token(token)
+    if created.state == UPLOADING:
+        if source is None:
+            raise file_required()
+        _upload(session, created, source, options.concurrency, callbacks)
+    job = wait_job(session, segment(created.job_id), options.wait_timeout, callbacks, token)
+    return fetch_transcript(session, job)
+
+
+def run_transcribe(
+    session: SyncSession,
+    file: FileInput,
+    options: Options,
+    size: int | None,
+    callbacks: Callbacks,
+) -> Transcript:
+    options.validate()
+    source = make_source(file, size)
+    media: Media = check_media(source, options)
+    account = json_body(session.http.get("/v1/account"), session.credentials)
+    check_account(account, media)
+    key = options.new_key()
+    state = ResumeState("", key, media.size, media.duration_seconds)
+    body = create_body(media, options, key, title=options.title)
+    tracker = Tracker(key, callers=callbacks.raised, credentials=session.credentials)
+
+    def run() -> Transcript:
+        callbacks.progress(UploadingProgress(0, 0, state.size))
+        tracker.sent = True
+        created = create(session, body)
+        return _after_create(session, created, tracker, state, source, options, callbacks)
+
+    return annotated(tracker, run)
+
+
+def read_job_to_resume(session: SyncSession, job_id: str) -> Job:
+    """The job a token names; a 404 means it was deleted or pruned (spec 7.3)."""
+    try:
+        return get_job(session, segment(job_id))
+    except NotFoundError as missing:
+        raise job_missing() from missing
+
+
+def run_resume(
+    session: SyncSession,
+    token: str,
+    file: FileInput | None,
+    options: Options,
+    callbacks: Callbacks,
+) -> Transcript:
+    options.validate()
+    state = decode_token(token)
+    source = early_source(state, file)
+    tracker = Tracker(
+        state.idempotency_key,
+        token,
+        sent=True,
+        callers=callbacks.raised,
+        credentials=session.credentials,
+    )
+
+    def run() -> Transcript:
+        job = read_job_to_resume(session, state.job_id)
+        if job.state != UPLOADING:
+            return fetch_transcript(
+                session,
+                wait_job(
+                    session, segment(state.job_id), options.wait_timeout, callbacks, token, job
+                ),
+            )
+        late = late_source(state, file, source)
+        callbacks.progress(UploadingProgress(0, 0, state.size))
+        created = create(session, replay_body(state))
+        if created.job_id != state.job_id:
+            raise job_missing()
+        return _after_create(session, created, tracker, state, late, options, callbacks)
+
+    return annotated(tracker, run)

hushscript/transcribe/preflight.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+
+import math
+import re
+import uuid
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Final, cast
+
+from hushscript._errors import (
+    DailyLimitError,
+    FileLimitError,
+    PreflightError,
+    UnsupportedMediaError,
+    describe_failure,
+)
+from hushscript._json import as_dict
+from hushscript._shape import INT_LIMIT
+
+from .probe import probe_duration
+from .sniff import sniff
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from .source import Source
+
+MAX_BYTES: Final = 5_000_000_000
+MIN_SECONDS: Final = 15
+MAX_SECONDS: Final = 36_000
+MAX_CONCURRENCY: Final = 8
+MAX_KEYTERMS: Final = 1000
+MAX_KEYTERM_WORDS: Final = 6
+MAX_KEYTERM_CHARS: Final = 128
+MAX_PROMPT_CHARS: Final = 2000
+MAX_MATTER_CODE: Final = 64
+MEDICAL_LANGUAGES: Final = frozenset({"en", "es", "fr", "de"})
+KEY_MIN: Final = 16
+KEY_MAX: Final = 128
+_PRINTABLE_FIRST: Final = 0x21
+_PRINTABLE_LAST: Final = 0x7E
+# Unicode White_Space, the set of Rust's `char::is_whitespace`, which the API trims and splits by.
+_WHITESPACE: Final = (
+    "\t\n\v\f\r \x85\xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008"
+    "\u2009\u200a\u2028\u2029\u202f\u205f\u3000"
+)
+_WHITESPACE_RUN: Final = re.compile(f"[{_WHITESPACE}]+")
+# The fixed text the API wraps around a prompt and key terms when both are set.
+_COMPILED_FIXED_CHARS: Final = len(
+    "Transcription context:\n\nBehavior:\n"
+    "\n\nVocabulary:\n"
+    "\n\nInstructions:\n"
+    "- Prefer exact spellings when the audio supports them.\n"
+    "- Do not insert terms that are not audible.\n"
+    "- Preserve speaker turns and timestamps."
+)
+
+
+def invalid_option(message: str) -> PreflightError:
+    return PreflightError(message, code="invalid_option")
+
+
+def _clean_keyterms(terms: tuple[str, ...]) -> list[tuple[int, str]]:
+    """Trimmed, non-blank, unique terms in order, each with its index in the given list."""
+    seen: set[str] = set()
+    kept: list[tuple[int, str]] = []
+    for index, term in enumerate(terms):
+        trimmed = term.strip(_WHITESPACE)
+        if trimmed and trimmed not in seen:
+            seen.add(trimmed)
+            kept.append((index, trimmed))
+    return kept
+
+
+def _clean_prompt(prompt: str | None) -> str | None:
+    trimmed = None if prompt is None else prompt.strip(_WHITESPACE)
+    return trimmed or None
+
+
+def _check_context(terms: list[tuple[int, str]], prompt: str | None) -> None:
+    if prompt is not None and len(prompt) > MAX_PROMPT_CHARS:
+        raise invalid_option(f"prompt is longer than {MAX_PROMPT_CHARS} characters.")
+    if len(terms) > MAX_KEYTERMS:
+        raise invalid_option(f"keyterms has more than {MAX_KEYTERMS} unique terms.")
+    for index, term in terms:
+        if len(term) > MAX_KEYTERM_CHARS:
+            raise invalid_option(
+                f"keyterms[{index}] is longer than {MAX_KEYTERM_CHARS} characters."
+            )
+        if len(_WHITESPACE_RUN.split(term)) > MAX_KEYTERM_WORDS:
+            raise invalid_option(f"keyterms[{index}] has more than {MAX_KEYTERM_WORDS} words.")
+    if prompt is not None and terms:
+        compiled = (
+            _COMPILED_FIXED_CHARS
+            + len(prompt)
+            + sum(len(term) + 2 for _, term in terms)
+            + len(terms)
+            - 1
+        )
+        if compiled > MAX_PROMPT_CHARS:
+            raise invalid_option(
+                f"prompt and keyterms together exceed {MAX_PROMPT_CHARS} characters ({compiled})."
+            )
+
+
+def _is_number(value: object) -> bool:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return False
+    return isinstance(value, int) or math.isfinite(value)
+
+
+def check_wait_timeout(wait_timeout: object) -> None:
+    if not _is_number(wait_timeout) or cast("float", wait_timeout) <= 0:
+        raise invalid_option("wait_timeout must be a positive number of seconds.")
+
+
+@dataclass(frozen=True, slots=True)
+class Options:
+    """The options of `transcribe()` (spec 6.1), checked before any request."""
+
+    title: str | None = None
+    language: str | None = None
+    speakers: bool = True
+    translate_to: tuple[str, ...] = ()
+    keyterms: tuple[str, ...] = ()
+    prompt: str | None = None
+    medical: bool = False
+    multichannel: bool = False
+    auto_insights: bool | None = None
+    matter_code: str | None = None
+    duration_seconds: float | None = None
+    idempotency_key: str | None = None
+    concurrency: int = 4
+    wait_timeout: float = 3600
+
+    def validate(self) -> Options:
+        concurrency = cast("object", self.concurrency)
+        if (
+            not isinstance(concurrency, int)
+            or isinstance(concurrency, bool)
+            or not 1 <= concurrency <= MAX_CONCURRENCY
+        ):
+            raise invalid_option(f"concurrency must be a whole number from 1 to {MAX_CONCURRENCY}.")
+        check_wait_timeout(self.wait_timeout)
+        if self.duration_seconds is not None and (
+            not _is_number(self.duration_seconds) or self.duration_seconds <= 0
+        ):
+            raise invalid_option("duration_seconds must be a positive number of seconds.")
+        key = cast("object", self.idempotency_key)
+        if key is not None and not (
+            isinstance(key, str)
+            and KEY_MIN <= len(key) <= KEY_MAX
+            and all(_PRINTABLE_FIRST <= ord(char) <= _PRINTABLE_LAST for char in key)
+        ):
+            raise invalid_option(
+                f"idempotency_key must be {KEY_MIN} to {KEY_MAX} visible ASCII characters,"
+                " no spaces."
+            )
+        if self.medical and self.language not in MEDICAL_LANGUAGES:
+            raise invalid_option("medical needs language to be one of en, es, fr, de.")
+        _check_context(_clean_keyterms(self.keyterms), _clean_prompt(self.prompt))
+        if self.matter_code is not None and len(self.matter_code) > MAX_MATTER_CODE:
+            raise invalid_option(f"matter_code must be at most {MAX_MATTER_CODE} characters.")
+        return self
+
+    def new_key(self) -> str:
+        return self.idempotency_key or str(uuid.uuid4())
+
+    def transcription_options(self) -> dict[str, object]:
+        wire: dict[str, object] = {}
+        if self.language is not None:
+            wire["language_code"] = self.language
+        if not self.speakers and not self.translate_to:
+            wire["speaker_detection"] = False
+        if self.translate_to:
+            wire["translation_targets"] = list(self.translate_to)
+        terms = [term for _, term in _clean_keyterms(self.keyterms)]
+        if terms:
+            wire["keyterms_prompt"] = terms
+        prompt = _clean_prompt(self.prompt)
+        if prompt is not None:
+            wire["prompt"] = prompt
+        if self.medical:
+            wire["medical_mode"] = True
+        if self.multichannel:
+            wire["multichannel"] = True
+        if self.auto_insights is not None:
+            wire["auto_generate_insights"] = self.auto_insights
+        return wire
+
+
+@dataclass(frozen=True, slots=True)
+class Media:
+    size: int
+    duration_seconds: int
+    """The declared duration: `ceil` of the probe, so the hold never falls short of the file."""
+
+
+def check_media(source: Source, options: Options) -> Media:
+    """Spec 8.1 steps 1 to 3: format by magic bytes, size, duration."""
+    if source.sequential and options.duration_seconds is None:
+        raise invalid_option("A stream that cannot seek needs duration_seconds and size.")
+    try:
+        head = source.head()
+    except OSError as failure:
+        raise PreflightError(
+            f"The file cannot be read ({describe_failure(failure)}).", code="file_unreadable"
+        ) from failure
+    if sniff(head) is None:
+        raise UnsupportedMediaError("The file is not a supported audio or video format.")
+    if not 1 <= source.size <= MAX_BYTES:
+        raise FileLimitError(
+            f"The file is {source.size} bytes; it must be between 1 byte and 5 GB.",
+            code="size_out_of_range",
+        )
+    seconds = options.duration_seconds
+    if seconds is None:
+        try:
+            seconds = probe_duration(source)
+        except OSError as failure:
+            raise PreflightError(
+                f"The file cannot be read ({describe_failure(failure)}).", code="file_unreadable"
+            ) from failure
+    declared = math.ceil(seconds)
+    if seconds < MIN_SECONDS or declared > MAX_SECONDS:
+        raise FileLimitError(
+            f"The file is {seconds:g} seconds long; it must be between 15 seconds and 10 hours.",
+            code="duration_out_of_range",
+        )
+    return Media(size=source.size, duration_seconds=declared)
+
+
+def _whole(value: object) -> int | None:
+    return (
+        value
+        if isinstance(value, int) and not isinstance(value, bool) and abs(value) < INT_LIMIT
+        else None
+    )
+
+
+def check_account(account: object, media: Media) -> None:
+    """Spec 8.1 step 5, from one `GET /v1/account`: the daily limit only.
+
+    The balance is the server's to judge at create (R77).
+    """
+    fields = as_dict(account) or {}
+    limits = as_dict(fields.get("limits")) or {}
+    daily = _whole(limits.get("max_daily_seconds"))
+    used = _whole(limits.get("used_today_seconds"))
+    if daily is not None and used is not None and used + media.duration_seconds > daily:
+        raise DailyLimitError(
+            "This upload would pass the daily transcription limit.",
+            code="daily_limit",
+            details={"used_today_seconds": used, "max_daily_seconds": daily},
+        )
+
+
+def create_body(
+    media: Media, options: Options, key: str, *, title: str | None
+) -> Mapping[str, object]:
+    body: dict[str, object] = {
+        "size_bytes": media.size,
+        "duration_seconds": media.duration_seconds,
+        "idempotency_key": key,
+    }
+    if title is not None:
+        body["title"] = title
+    if options.matter_code is not None:
+        body["matter_code"] = options.matter_code
+    wire = options.transcription_options()
+    if wire:
+        body["transcription_options"] = wire
+    return body