hushscript 1.0.0__py3-none-any.whl

hushscript/resources/transcripts.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from hushscript._json import ExtraFields, as_dict, as_list, extra_fields
+from hushscript._response import parsed
+from hushscript._shape import (
+    as_text,
+    as_whole,
+    check_shape,
+    maybe_text,
+    to_datetime,
+)
+from hushscript._shapes import DELETED_LIST, TRANSCRIPT_LIST
+from hushscript.paginate import Page, apaginate, check_cursor_advanced, paginate
+from hushscript.resources._params import check_page_size, list_params, path_id
+from hushscript.transcript import Transcript
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator, Iterator
+    from datetime import datetime
+
+    from hushscript.transcribe.session import AsyncSession, SyncSession
+
+
+@dataclass(frozen=True, slots=True)
+class TranscriptSummary(ExtraFields):
+    """One row of the transcript list."""
+
+    id: str
+    title: str
+    language: str | None
+    duration_seconds: int
+    created_at: datetime
+    auto_delete_at: datetime | None
+    tags: list[str]
+
+    __hash__ = None  # type: ignore[assignment]  # holds a list, so it is unhashable by design
+
+
+@dataclass(frozen=True, slots=True)
+class DeletedTranscript(ExtraFields):
+    """The record kept after a transcript was removed; its content is gone."""
+
+    id: str
+    title: str
+    duration_seconds: int
+    created_at: datetime
+    """When the transcript was uploaded."""
+    deleted_at: datetime
+    cause: str
+    """`manual`, `scheduled`, `dormant` or `expired`; a cause the SDK does not know yet passes
+    through as the API's own string."""
+    receipt: dict[str, object] | None
+    """The stored deletion receipt, opaque and as the API sends it (snake_case keys, string
+    timestamps); None when it could not be read back."""
+
+    __hash__ = None  # type: ignore[assignment]  # holds a dict, so it is unhashable by design
+
+
+def summary_page(body: object) -> Page[TranscriptSummary]:
+    check_shape(body, TRANSCRIPT_LIST, "transcript list response")
+    fields = as_dict(body) or {}
+    items: list[TranscriptSummary] = []
+    for row in as_list(fields["transcripts"]) or []:
+        item = as_dict(row) or {}
+        auto_delete = item["auto_delete_at"]
+        items.append(
+            TranscriptSummary(
+                _extra=extra_fields(
+                    item,
+                    "id",
+                    "title",
+                    "language_detected",
+                    "duration_seconds",
+                    "created_at",
+                    "auto_delete_at",
+                    "tags",
+                ),
+                id=as_text(item["id"]),
+                title=as_text(item["title"]),
+                language=maybe_text(item["language_detected"]),
+                duration_seconds=as_whole(item["duration_seconds"]),
+                created_at=to_datetime(item["created_at"]),
+                auto_delete_at=None if auto_delete is None else to_datetime(auto_delete),
+                tags=[as_text(tag) for tag in as_list(item["tags"]) or []],
+            )
+        )
+    return Page(
+        _extra=extra_fields(fields, "transcripts", "next_cursor"),
+        items=items,
+        next_cursor=maybe_text(fields["next_cursor"]),
+    )
+
+
+def deleted_page(body: object) -> Page[DeletedTranscript]:
+    check_shape(body, DELETED_LIST, "deleted transcript list response")
+    fields = as_dict(body) or {}
+    items: list[DeletedTranscript] = []
+    for row in as_list(fields["deleted"]) or []:
+        item = as_dict(row) or {}
+        items.append(
+            DeletedTranscript(
+                _extra=extra_fields(
+                    item,
+                    "id",
+                    "title",
+                    "duration_seconds",
+                    "created_at",
+                    "deleted_at",
+                    "cause",
+                    "receipt",
+                ),
+                id=as_text(item["id"]),
+                title=as_text(item["title"]),
+                duration_seconds=as_whole(item["duration_seconds"]),
+                created_at=to_datetime(item["created_at"]),
+                deleted_at=to_datetime(item["deleted_at"]),
+                cause=as_text(item["cause"]),
+                receipt=as_dict(item["receipt"]),
+            )
+        )
+    return Page(
+        _extra=extra_fields(fields, "deleted", "next_cursor"),
+        items=items,
+        next_cursor=maybe_text(fields["next_cursor"]),
+    )
+
+
+class Transcripts:
+    """`hs.transcripts`: the account's finished transcripts. Ones still processing are jobs: see
+    `hs.jobs.list(active=True)`."""
+
+    def __init__(self, session: SyncSession) -> None:
+        self._session = session
+
+    def list(self, *, page_size: int | None = None) -> Iterator[TranscriptSummary]:
+        """Every finished transcript, newest first, fetching pages as it goes. `page_size` is 1 to
+        100 (the API clamps it there) and defaults to 50."""
+        check_page_size(page_size)
+        return paginate(lambda cursor: self._page(cursor, page_size))
+
+    def page(
+        self, *, cursor: str | None = None, page_size: int | None = None
+    ) -> Page[TranscriptSummary]:
+        """One page of `list()`; `cursor` is a previous page's `next_cursor`. A page that returns
+        the cursor it was sent raises `pagination_loop`."""
+        check_page_size(page_size)
+        page = self._page(cursor, page_size)
+        check_cursor_advanced(cursor, page)
+        return page
+
+    def get(self, transcript_id: str) -> Transcript:
+        path = f"/v1/transcripts/{path_id(transcript_id)}"
+        return parsed(self._session.http.get(path), self._session.credentials, Transcript.from_api)
+
+    def deleted(self) -> Iterator[DeletedTranscript]:
+        """The deletion records, by upload time, newest first."""
+        return paginate(self._deleted_page)
+
+    def _page(self, cursor: str | None, page_size: int | None) -> Page[TranscriptSummary]:
+        response = self._session.http.get("/v1/transcripts", params=list_params(cursor, page_size))
+        return parsed(response, self._session.credentials, summary_page)
+
+    def _deleted_page(self, cursor: str | None) -> Page[DeletedTranscript]:
+        response = self._session.http.get(
+            "/v1/transcripts/deleted", params=list_params(cursor, None)
+        )
+        return parsed(response, self._session.credentials, deleted_page)
+
+
+class AsyncTranscripts:
+    """`hs.transcripts` on the asyncio client; the lists are async iterators."""
+
+    def __init__(self, session: AsyncSession) -> None:
+        self._session = session
+
+    def list(self, *, page_size: int | None = None) -> AsyncIterator[TranscriptSummary]:
+        check_page_size(page_size)
+        return apaginate(lambda cursor: self._page(cursor, page_size))
+
+    async def page(
+        self, *, cursor: str | None = None, page_size: int | None = None
+    ) -> Page[TranscriptSummary]:
+        check_page_size(page_size)
+        page = await self._page(cursor, page_size)
+        check_cursor_advanced(cursor, page)
+        return page
+
+    async def get(self, transcript_id: str) -> Transcript:
+        path = f"/v1/transcripts/{path_id(transcript_id)}"
+        response = await self._session.http.get(path)
+        return parsed(response, self._session.credentials, Transcript.from_api)
+
+    def deleted(self) -> AsyncIterator[DeletedTranscript]:
+        return apaginate(self._deleted_page)
+
+    async def _page(self, cursor: str | None, page_size: int | None) -> Page[TranscriptSummary]:
+        response = await self._session.http.get(
+            "/v1/transcripts", params=list_params(cursor, page_size)
+        )
+        return parsed(response, self._session.credentials, summary_page)
+
+    async def _deleted_page(self, cursor: str | None) -> Page[DeletedTranscript]:
+        response = await self._session.http.get(
+            "/v1/transcripts/deleted", params=list_params(cursor, None)
+        )
+        return parsed(response, self._session.credentials, deleted_page)

hushscript/transcribe/__init__.py

@@ -0,0 +1 @@
+"""The transcribe pipeline (spec 8): preflight, probe, create, parts, heartbeat, complete, wait."""

hushscript/transcribe/apipeline.py

@@ -0,0 +1,234 @@
+from __future__ import annotations
+
+import asyncio
+from typing import TYPE_CHECKING, Any, TypeVar
+
+from hushscript._errors import NotFoundError, raise_for_response
+from hushscript._models import Job, UploadingProgress
+from hushscript._response import json_body, parsed
+
+from .aupload import AsyncUploader
+from .heartbeat import Heartbeat
+from .pipeline import (
+    UPLOADING,
+    Callbacks,
+    Tracker,
+    annotate,
+    note_interrupt,
+    percent,
+    replay_body,
+)
+from .preflight import Options, check_account, check_media, create_body
+from .resume import (
+    ResumeState,
+    decode_token,
+    early_source,
+    encode_token,
+    file_required,
+    job_missing,
+    late_source,
+)
+from .segment import segment
+from .source import FileInput, Source, make_source
+from .upload import parse_created, stream_size_mismatch
+from .wait import await_job, finish, job_from, transcript_of
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable, Mapping
+
+    from hushscript.transcript import Transcript
+
+    from .session import AsyncSession
+    from .upload import Created
+
+T = TypeVar("T")
+
+
+async def create(session: AsyncSession, body: Mapping[str, object]) -> Created:
+    response = await session.http.post("/v1/uploads", json=body)
+    return parsed(response, session.credentials, parse_created)
+
+
+async def get_job(session: AsyncSession, job_segment: str) -> Job:
+    return job_from(await session.http.get(f"/v1/jobs/{job_segment}"), session.credentials)
+
+
+async def wait_job(
+    session: AsyncSession,
+    job_segment: str,
+    wait_timeout: float,
+    callbacks: Callbacks,
+    token: str | None,
+    first: Job | None = None,
+) -> Job:
+    return await await_job(
+        lambda: get_job(session, job_segment),
+        session.clock,
+        wait_timeout,
+        callbacks.progress,
+        token,
+        first,
+    )
+
+
+async def fetch_transcript(session: AsyncSession, job: Job) -> Transcript:
+    transcript_id = finish(job)
+    return transcript_of(
+        await session.http.get(f"/v1/transcripts/{segment(transcript_id)}"), session.credentials
+    )
+
+
+async def _supervised(
+    heartbeat: Heartbeat, ping: Callable[[], Awaitable[None]], work: Awaitable[T]
+) -> T:
+    """Runs `work` beside the heartbeat. A heartbeat that ends in `unauthorized` cancels the work
+    and raises its error; one that ends in `wrong_state` just stops."""
+    work_task = asyncio.ensure_future(work)
+    beat_task = asyncio.ensure_future(heartbeat.arun(ping))
+    try:
+        waiting: set[asyncio.Future[Any]] = {work_task, beat_task}
+        while True:
+            done, _ = await asyncio.wait(waiting, return_when=asyncio.FIRST_COMPLETED)
+            if work_task in done:
+                return work_task.result()
+            heartbeat.raise_if_failed()
+            waiting = {work_task}
+    finally:
+        for task in (work_task, beat_task):
+            task.cancel()
+        await asyncio.gather(work_task, beat_task, return_exceptions=True)
+
+
+async def _upload(
+    session: AsyncSession,
+    created: Created,
+    source: Source,
+    concurrency: int,
+    callbacks: Callbacks,
+) -> None:
+    job_path = f"/v1/uploads/{segment(created.job_id)}"
+    uploader = AsyncUploader(session, created.job_id, created.plan, source, concurrency)
+    total = source.size
+
+    async def ping() -> None:
+        raise_for_response(
+            await session.http.post(f"{job_path}/heartbeat"), credentials=session.credentials
+        )
+
+    async def parts_and_complete() -> None:
+        await uploader.upload(
+            lambda sent: callbacks.progress(UploadingProgress(percent(sent, total), sent, total))
+        )
+        if not await asyncio.to_thread(source.at_end):
+            raise stream_size_mismatch()
+        body = uploader.state.parts_body()
+        response = (
+            await session.http.post(f"{job_path}/complete")
+            if body is None
+            else await session.http.post(f"{job_path}/complete", json=body)
+        )
+        raise_for_response(response, credentials=session.credentials)
+
+    await _supervised(Heartbeat(session.clock, session.log), ping, parts_and_complete())
+
+
+async def _after_create(  # the upload, wait and fetch that follow a create
+    session: AsyncSession,
+    created: Created,
+    tracker: Tracker,
+    state: ResumeState,
+    source: Source | None,
+    options: Options,
+    callbacks: Callbacks,
+) -> Transcript:
+    token = encode_token(
+        ResumeState(created.job_id, state.idempotency_key, state.size, state.duration_seconds)
+    )
+    tracker.token = token
+    callbacks.resume_token(token)
+    if created.state == UPLOADING:
+        if source is None:
+            raise file_required()
+        await _upload(session, created, source, options.concurrency, callbacks)
+    job = await wait_job(session, segment(created.job_id), options.wait_timeout, callbacks, token)
+    return await fetch_transcript(session, job)
+
+
+async def annotated(tracker: Tracker, run: Callable[[], Awaitable[T]]) -> T:
+    """`pipeline.annotated` for the asyncio client."""
+    try:
+        return await run()
+    except (asyncio.CancelledError, KeyboardInterrupt) as interrupted:
+        note_interrupt(tracker, interrupted)
+        raise
+    except Exception as failure:  # noqa: BLE001  # R61: nothing but an SDK error leaves
+        failed = annotate(tracker, failure)
+    raise failed
+
+
+async def run_transcribe(
+    session: AsyncSession,
+    file: FileInput,
+    options: Options,
+    size: int | None,
+    callbacks: Callbacks,
+) -> Transcript:
+    options.validate()
+    source = make_source(file, size)
+    media = await asyncio.to_thread(check_media, source, options)
+    account = json_body(await session.http.get("/v1/account"), session.credentials)
+    check_account(account, media)
+    key = options.new_key()
+    state = ResumeState("", key, media.size, media.duration_seconds)
+    body = create_body(media, options, key, title=options.title)
+    tracker = Tracker(key, callers=callbacks.raised, credentials=session.credentials)
+
+    async def run() -> Transcript:
+        callbacks.progress(UploadingProgress(0, 0, state.size))
+        tracker.sent = True
+        created = await create(session, body)
+        return await _after_create(session, created, tracker, state, source, options, callbacks)
+
+    return await annotated(tracker, run)
+
+
+async def read_job_to_resume(session: AsyncSession, job_id: str) -> Job:
+    try:
+        return await get_job(session, segment(job_id))
+    except NotFoundError as missing:
+        raise job_missing() from missing
+
+
+async def run_resume(
+    session: AsyncSession,
+    token: str,
+    file: FileInput | None,
+    options: Options,
+    callbacks: Callbacks,
+) -> Transcript:
+    options.validate()
+    state = decode_token(token)
+    source = early_source(state, file)
+    tracker = Tracker(
+        state.idempotency_key,
+        token,
+        sent=True,
+        callers=callbacks.raised,
+        credentials=session.credentials,
+    )
+
+    async def run() -> Transcript:
+        job = await read_job_to_resume(session, state.job_id)
+        if job.state != UPLOADING:
+            waited = await wait_job(
+                session, segment(state.job_id), options.wait_timeout, callbacks, token, job
+            )
+            return await fetch_transcript(session, waited)
+        late = late_source(state, file, source)
+        callbacks.progress(UploadingProgress(0, 0, state.size))
+        created = await create(session, replay_body(state))
+        if created.job_id != state.job_id:
+            raise job_missing()
+        return await _after_create(session, created, tracker, state, late, options, callbacks)
+
+    return await annotated(tracker, run)

hushscript/transcribe/aupload.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+import asyncio
+import sys
+from typing import TYPE_CHECKING
+
+import anyio
+
+if sys.version_info < (3, 11):
+    from exceptiongroup import ExceptionGroup
+
+from hushscript._errors import NetworkError, raise_for_response
+from hushscript._http import STREAM_EXTENSION, release
+from hushscript._log import hiding_signed_url
+
+from .segment import segment
+from .upload import (
+    OCTET_STREAM,
+    PART_TIMEOUTS,
+    Action,
+    PartAttempts,
+    Plan,
+    UploadState,
+    UrlBook,
+    etag_missing,
+    etag_of,
+    exhausted,
+    first_mode,
+    parse_page,
+    part_url_missing,
+    plan_for,
+    r2_extensions,
+    read_part,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from hushscript._redact import Secret
+
+    from .session import AsyncSession
+    from .source import Source
+
+
+class AsyncUploader:
+    """The asyncio twin of `SyncUploader`: `concurrency` worker tasks in a task group, so one
+    failed part cancels the others."""
+
+    def __init__(
+        self,
+        session: AsyncSession,
+        job_id: str,
+        plan: Plan | None,
+        source: Source,
+        concurrency: int,
+    ) -> None:
+        self._session = session
+        self._job_id = job_id
+        self._source = source
+        self.plan = plan_for(plan, source.size)
+        self.state = UploadState(first_mode(plan))
+        self._book = UrlBook(session.clock)
+        self._book.store(self.plan.urls, self.plan.next_part)
+        self._pages = asyncio.Lock()
+        self._width = 1 if source.sequential else concurrency
+        self._sent = 0
+
+    async def upload(self, on_bytes: Callable[[int], None]) -> None:
+        numbers = iter(range(1, self.plan.total_parts + 1))
+
+        async def worker() -> None:
+            for n in numbers:
+                size = await self._upload_part(n)
+                self._sent += size
+                on_bytes(self._sent)
+
+        try:
+            async with anyio.create_task_group() as group:
+                for _ in range(self._width):
+                    group.start_soon(worker)
+        except ExceptionGroup as failed:
+            raise failed.exceptions[0] from None
+
+    async def _read(self, n: int) -> bytes:
+        return await asyncio.to_thread(read_part, self._source, self.plan, n)
+
+    async def _upload_part(self, n: int) -> int:
+        data = await self._read(n)
+        attempts = PartAttempts()
+        while True:
+            if self.state.proxied:
+                await self._put_proxied(n, data)
+                return len(data)
+            url = await self._url_for(n)
+            if url is None:
+                continue
+            status, etag = await self._put_direct(n, url, data)
+            if etag is not None:
+                if not self.state.record(n, etag):
+                    await self._put_proxied(n, data)
+                return len(data)
+            action = self.state.decide(attempts, status)
+            if action is Action.EXHAUSTED:
+                raise exhausted(n)
+            if action is Action.REFRESH:
+                await self._refresh(n, url)
+
+    async def _put_direct(self, n: int, url: Secret, data: bytes) -> tuple[int | None, str | None]:
+        client = self._session.r2
+        try:
+            with hiding_signed_url(f"r2:part/{n}"):
+                response = await client.send(
+                    client.build_request(
+                        "PUT",
+                        url.reveal(),
+                        content=data,
+                        timeout=PART_TIMEOUTS,
+                        extensions=r2_extensions(n),
+                    ),
+                    stream=True,
+                )
+            try:
+                etag = etag_of(response)
+                if response.is_success and etag is None:
+                    raise etag_missing(n)
+                return response.status_code, etag
+            finally:
+                try:
+                    await response.aclose()
+                finally:
+                    release(response.request)
+        except NetworkError:
+            return None, None
+
+    async def _put_proxied(self, n: int, data: bytes) -> None:
+        client = self._session.http
+        response = await client.send(
+            client.build_request(
+                "PUT",
+                f"/v1/uploads/{segment(self._job_id)}/parts/{n}",
+                content=data,
+                headers=OCTET_STREAM,
+                timeout=PART_TIMEOUTS,
+                extensions={STREAM_EXTENSION: True},
+            ),
+            stream=True,
+        )
+        try:
+            if not response.is_success:
+                await response.aread()
+                raise_for_response(response, credentials=self._session.credentials)
+        finally:
+            try:
+                await response.aclose()
+            finally:
+                release(response.request)
+
+    async def _url_for(self, n: int) -> Secret | None:
+        async with self._pages:
+            url = self._book.get(n)
+            if url is None:
+                await self._fetch_for(n)
+                if self.state.proxied:
+                    return None
+                url = self._book.get(n)
+            if url is None:
+                raise part_url_missing(n)
+            return url
+
+    async def _fetch_for(self, n: int) -> None:
+        start = self._book.next_start
+        if start is None or n < start:
+            await self._fetch(n, advance=False)
+            return
+        while (
+            self._book.get(n) is None
+            and self._book.next_start is not None
+            and not self.state.proxied
+        ):
+            await self._fetch(self._book.next_start, advance=True)
+
+    async def _refresh(self, n: int, failed_url: Secret) -> None:
+        async with self._pages:
+            if self._book.is_current(n, failed_url):
+                await self._fetch(n, advance=False)
+
+    async def _fetch(self, start: int, *, advance: bool) -> None:
+        response = await self._session.http.get(
+            f"/v1/uploads/{segment(self._job_id)}/part-urls", params={"start": start}
+        )
+        plan = parse_page(response, self._session.credentials)
+        if not plan.direct and self.state.fall_back():
+            return
+        cursor = plan.next_part if advance else self._book.next_start
+        if advance and cursor is not None and cursor <= start:
+            cursor = None
+        self._book.store(plan.urls, cursor)