textflowkit 0.1.3__py3-none-any.whl

textflowkit/core/translate.py

@@ -0,0 +1,244 @@
+"""Transcript translation.
+
+Translation is a separate stage from transcription: it runs after the transcript
+exists, writes into `Segment.translated_text` (a field the model already had), and
+is opt-in.
+
+The backend is pluggable. The shipped implementation talks to an Ollama host,
+local by default. The selected model may itself be cloud-hosted, so an explicit
+model is required and the route is reported rather than assuming locality.
+
+Two engineering choices worth stating:
+
+- **Batching with a fallback.** One request per segment is correct but slow (a
+  214-segment transcript would be 214 round trips). Requests are batched, and if
+  a batch comes back unparseable or the wrong length, that batch is retried one
+  segment at a time. Correctness does not depend on the model obeying a format.
+- **A cache.** Whisper repeats phrases; identical source text is translated once.
+
+A backend that is unreachable or misconfigured raises. It never returns the
+source text as though it were a translation.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import urllib.error
+import urllib.request
+from collections.abc import Sequence
+from typing import Protocol
+from urllib.parse import urlparse
+
+from textflowkit.core.model import Segment
+
+ENV_OLLAMA_HOST = "TEXTFLOWKIT_OLLAMA_HOST"
+ENV_OLLAMA_MODEL = "TEXTFLOWKIT_TRANSLATE_MODEL"
+DEFAULT_OLLAMA_HOST = "http://127.0.0.1:11434"
+BATCH_SIZE = 20
+
+_NUMBERED = re.compile(r"^\s*(\d+)\s*[.):\-]\s*(.*)$")
+
+
+class TranslationError(RuntimeError):
+    """Raised when translation was requested but could not be performed."""
+
+
+class Translator(Protocol):
+    """Translates text into a target language."""
+
+    name: str
+
+    def translate(self, texts: Sequence[str], target: str) -> list[str]: ...
+
+
+class OllamaTranslator:
+    """Translation through a local Ollama instance.
+
+    Uses /api/generate with a strict numbered-list prompt. The model is not
+    trusted to be well behaved: a response that cannot be parsed back into
+    exactly the requested number of items is rejected, and the caller falls back
+    to per-segment requests.
+    """
+
+    name = "ollama"
+
+    def __init__(
+        self,
+        model: str | None = None,
+        host: str | None = None,
+        *,
+        timeout: float = 300.0,
+    ) -> None:
+        self.model = model or os.environ.get(ENV_OLLAMA_MODEL)
+        if not self.model:
+            raise TranslationError(
+                f"translation requires an explicit model; set {ENV_OLLAMA_MODEL}. "
+                "A cloud model is never selected automatically."
+            )
+        self.host = (host or os.environ.get(ENV_OLLAMA_HOST, DEFAULT_OLLAMA_HOST)).rstrip("/")
+        hostname = urlparse(self.host).hostname or ""
+        self.route = (
+            "cloud model" if self.model.endswith(":cloud")
+            else "local Ollama" if hostname in {"localhost", "127.0.0.1", "::1"}
+            else "remote Ollama host"
+        )
+        self.timeout = timeout
+        self._cache: dict[tuple[str, str], str] = {}
+
+    @staticmethod
+    def _read_error_body(exc: urllib.error.HTTPError) -> str:
+        """Best-effort read of an error body for the message. Never raises."""
+        reader = getattr(exc, "read", None)
+        if reader is None:
+            return ""
+        try:
+            return reader().decode("utf-8", errors="replace")[:200]
+        except (OSError, ValueError, AttributeError):
+            return ""
+
+    # -- transport ---------------------------------------------------------
+
+    def _generate(self, prompt: str) -> str:
+        payload = json.dumps(
+            {
+                "model": self.model,
+                "prompt": prompt,
+                "stream": False,
+                "options": {"temperature": 0},
+            }
+        ).encode("utf-8")
+        request = urllib.request.Request(
+            f"{self.host}/api/generate",
+            data=payload,
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=self.timeout) as response:
+                body = json.loads(response.read().decode("utf-8"))
+        except urllib.error.HTTPError as exc:
+            detail = self._read_error_body(exc)
+            raise TranslationError(
+                f"Ollama rejected the request ({exc.code}) for model '{self.model}'. "
+                f"Is the model pulled? {detail}"
+            ) from exc
+        except (urllib.error.URLError, TimeoutError, OSError) as exc:
+            raise TranslationError(
+                f"could not reach Ollama at {self.host}: {exc}. "
+                f"Start Ollama, or set {ENV_OLLAMA_HOST}."
+            ) from exc
+        except json.JSONDecodeError as exc:
+            raise TranslationError(f"Ollama returned a non-JSON response: {exc}") from exc
+
+        if "response" not in body:
+            raise TranslationError(f"Ollama response had no 'response' field: {body}")
+        return str(body["response"])
+
+    # -- translation -------------------------------------------------------
+
+    def _translate_one(self, text: str, target: str) -> str:
+        stripped = text.strip()
+        if not stripped:
+            return ""
+        key = (stripped, target)
+        if key in self._cache:
+            return self._cache[key]
+
+        prompt = (
+            f"Translate the following text into {target}. "
+            "Reply with ONLY the translation - no preamble, no quotes, no notes.\n\n"
+            f"{stripped}"
+        )
+        out = self._generate(prompt).strip()
+        if not out:
+            raise TranslationError("model returned an empty translation")
+        self._cache[key] = out
+        return out
+
+    def _translate_batch(self, texts: Sequence[str], target: str) -> list[str]:
+        numbered = "\n".join(f"{i + 1}. {t.strip()}" for i, t in enumerate(texts))
+        prompt = (
+            f"Translate each numbered line into {target}. "
+            "Keep the numbering exactly as given and reply with ONLY the "
+            "translated numbered lines, one per line, no preamble and no notes.\n\n"
+            f"{numbered}"
+        )
+        raw = self._generate(prompt)
+
+        parsed: dict[int, str] = {}
+        for line in raw.splitlines():
+            match = _NUMBERED.match(line)
+            if match:
+                parsed[int(match.group(1))] = match.group(2).strip()
+
+        expected = list(range(1, len(texts) + 1))
+        if any(i not in parsed for i in expected):
+            raise TranslationError("batch response did not match the requested items")
+        return [parsed[i] for i in expected]
+
+    def translate(self, texts: Sequence[str], target: str) -> list[str]:
+        """Translate each text. Length of the result always matches the input."""
+        if not target.strip():
+            raise ValueError("target language must not be empty")
+
+        results: list[str] = []
+        for start in range(0, len(texts), BATCH_SIZE):
+            chunk = list(texts[start: start + BATCH_SIZE])
+            # Cached and empty entries never need a round trip.
+            if all(not t.strip() or (t.strip(), target) in self._cache for t in chunk):
+                results.extend(self._translate_one(t, target) for t in chunk)
+                continue
+            try:
+                batch = self._translate_batch(chunk, target)
+            except TranslationError:
+                # Correctness beats speed: redo this chunk one at a time rather
+                # than trust a malformed batch.
+                batch = [self._translate_one(t, target) for t in chunk]
+            else:
+                # Populate the cache from the batch path too. Only caching on the
+                # single-item path meant a phrase repeated in a later batch was
+                # retranslated every time.
+                for source_text, translated_text in zip(chunk, batch, strict=True):
+                    stripped = source_text.strip()
+                    if stripped and translated_text.strip():
+                        self._cache[(stripped, target)] = translated_text.strip()
+            results.extend(batch)
+        return results
+
+
+def translate_segments(
+    segments: list[Segment],
+    target: str,
+    *,
+    translator: Translator,
+) -> int:
+    """Fill `translated_text` on each segment. Returns how many were translated.
+
+    Blank segments are left alone. The translator's result length is checked
+    against the input so a misbehaving backend cannot silently shift text from
+    one segment onto another.
+    """
+    texts = [s.text for s in segments]
+    if len(texts) != len(segments):  # pragma: no cover - defensive
+        raise TranslationError("segment/text count mismatch")
+
+    translated = translator.translate(texts, target)
+    if len(translated) != len(texts):
+        raise TranslationError(
+            f"translator returned {len(translated)} results for {len(texts)} segments"
+        )
+
+    count = 0
+    for segment, text in zip(segments, translated, strict=True):
+        if segment.text.strip() and text.strip():
+            segment.translated_text = text
+            count += 1
+    return count
+
+
+def get_translator(backend: str = "ollama", **kwargs) -> Translator:
+    if backend in ("ollama", "default"):
+        return OllamaTranslator(**kwargs)
+    raise TranslationError(f"unknown translation backend: {backend}")

textflowkit/render/__init__.py

@@ -0,0 +1,191 @@
+"""Renderer layer: canonical transcript -> output formats.
+
+Two paths, because two kinds of output:
+
+- `render()` returns **text** (txt, srt, vtt, md, json) and is what the MCP
+  `get_transcript` tool returns inline.
+- `render_bytes()` returns **bytes** and additionally handles the binary formats
+  (docx, pdf), which exist for file export rather than for reading in a model's
+  context.
+
+`SUPPORTED_FORMATS` is the union and is what callers should validate against.
+`TEXT_FORMATS` is what can be returned as a string.
+"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from pathlib import Path
+
+from textflowkit.core.model import Transcript
+from textflowkit.core.service import enforce_output_limit
+from textflowkit.render.markdown import render_markdown
+from textflowkit.render.srt import render_srt
+from textflowkit.render.txt import render_txt
+from textflowkit.render.vtt import render_vtt
+
+RENDERERS = {
+    "txt": render_txt,
+    "srt": render_srt,
+    "vtt": render_vtt,
+    "md": render_markdown,
+}
+
+TEXT_FORMATS = tuple(RENDERERS) + ("json",)
+BINARY_FORMATS = ("docx", "pdf")
+SUPPORTED_FORMATS = TEXT_FORMATS + BINARY_FORMATS
+
+
+def _render_requested(
+    transcript: Transcript, formats: list[str], title: str | None
+) -> list[tuple[str, bytes]]:
+    """Validate and render every format before touching any destination file."""
+    rendered: list[tuple[str, bytes]] = []
+    seen: set[str] = set()
+    for fmt in formats:
+        norm = fmt.lower().lstrip(".")
+        if norm not in SUPPORTED_FORMATS:
+            raise ValueError(f"unsupported format: {fmt}")
+        if norm in seen:
+            raise ValueError(f"duplicate output format: {fmt}")
+        seen.add(norm)
+        rendered.append((norm, render_bytes(transcript, norm, title=title)))
+    enforce_output_limit(sum(len(data) for _, data in rendered))
+    return rendered
+
+
+def atomic_write_bytes(path: Path, data: bytes, *, replace: bool = False) -> None:
+    """Publish a complete file; optionally replace an explicitly chosen path."""
+    from textflowkit.core.paths import verify_output_file_target
+
+    enforce_output_limit(len(data))
+    verify_output_file_target(path)
+    temp: Path | None = None
+    try:
+        with tempfile.NamedTemporaryFile(dir=path.parent, prefix=f".{path.name}.",
+                                         suffix=".tmp", delete=False) as handle:
+            temp = Path(handle.name)
+            handle.write(data)
+            handle.flush()
+            os.fsync(handle.fileno())
+        # A hard link commits the fully-written temp file atomically and fails
+        # if the destination already exists, unlike os.replace().
+        verify_output_file_target(path)
+        if replace:
+            os.replace(temp, path)
+            temp = None
+        else:
+            os.link(temp, path)
+    finally:
+        if temp is not None:
+            temp.unlink(missing_ok=True)
+
+
+def render(transcript: Transcript, fmt: str, *, title: str | None = None) -> str:
+    """Render to text. Raises for a binary format - use `render_bytes`."""
+    fmt = fmt.lower().lstrip(".")
+    if fmt == "json":
+        return transcript.to_json()
+    if fmt in BINARY_FORMATS:
+        raise ValueError(
+            f"'{fmt}' is a binary format; use render_bytes() (and export to a file)"
+        )
+    if fmt not in RENDERERS:
+        raise ValueError(f"unsupported format: {fmt} (choose from {', '.join(SUPPORTED_FORMATS)})")
+    if fmt == "md":
+        return render_markdown(transcript, title=title)
+    return RENDERERS[fmt](transcript)
+
+
+def render_bytes(transcript: Transcript, fmt: str, *, title: str | None = None) -> bytes:
+    """Render to bytes, for any supported format including binary ones."""
+    fmt = fmt.lower().lstrip(".")
+
+    if fmt == "docx":
+        from textflowkit.render.docx import render_docx
+
+        return render_docx(transcript, title=title or "Transcript")
+    if fmt == "pdf":
+        from textflowkit.render.pdf import render_pdf
+
+        return render_pdf(transcript, title=title or "Transcript")
+    if fmt not in TEXT_FORMATS:
+        raise ValueError(f"unsupported format: {fmt} (choose from {', '.join(SUPPORTED_FORMATS)})")
+    return render(transcript, fmt, title=title).encode("utf-8")
+
+
+def ensure_outputs(
+    transcript: Transcript,
+    *,
+    formats: list[str],
+    output_dir: str | Path | None,
+    stem: str,
+    existing: list[str | Path] | None = None,
+    title: str | None = None,
+) -> list[Path]:
+    """Write requested formats, reusing already-present outputs when possible.
+
+    Resume must not redo transcription. Rendering missing files is cheap and
+    keeps the checkpoint contract true even when the original output directory
+    was removed between runs.
+    """
+    if output_dir is None:
+        return []
+    from textflowkit.core.paths import ensure_output_dir
+
+    out_dir = ensure_output_dir(str(output_dir))
+    rendered = _render_requested(transcript, formats, title)
+    by_suffix: dict[str, Path] = {}
+    for raw in existing or []:
+        path = Path(raw)
+        if path.parent.resolve() == out_dir.resolve():
+            by_suffix[path.suffix.lower().lstrip(".")] = path
+    written: list[Path] = []
+    for norm, data in rendered:
+        prior = by_suffix.get(norm)
+        if prior is not None and prior.exists():
+            written.append(prior)
+            continue
+        path = out_dir / f"{stem}.{norm}"
+        atomic_write_bytes(path, data)
+        written.append(path)
+    return written
+
+
+def write_all(
+    transcript: Transcript,
+    *,
+    formats: list[str],
+    output_dir: str | Path,
+    stem: str,
+    title: str | None = None,
+) -> list[Path]:
+    """Write each requested format to `output_dir`."""
+    from textflowkit.core.paths import ensure_output_dir
+
+    out_dir = ensure_output_dir(str(output_dir))
+    rendered = _render_requested(transcript, formats, title)
+    written: list[Path] = []
+    for norm, data in rendered:
+        path = out_dir / f"{stem}.{norm}"
+        atomic_write_bytes(path, data)
+        written.append(path)
+    return written
+
+
+__all__ = [
+    "BINARY_FORMATS",
+    "RENDERERS",
+    "SUPPORTED_FORMATS",
+    "TEXT_FORMATS",
+    "atomic_write_bytes",
+    "ensure_outputs",
+    "render",
+    "render_bytes",
+    "render_markdown",
+    "render_srt",
+    "render_txt",
+    "render_vtt",
+    "write_all",
+]

textflowkit/render/docx.py

@@ -0,0 +1,71 @@
+"""DOCX renderer.
+
+Binary output, so it does not go through `render()` - see `render_bytes()`.
+
+Renders the **finished** data model: speaker labels and translated text are both
+included, which is why this format is built after those stages exist rather than
+before.
+"""
+
+from __future__ import annotations
+
+import io
+
+from textflowkit.core.model import Transcript
+from textflowkit.core.timeutil import srt_timestamp
+
+try:
+    from docx import Document
+except ImportError as exc:  # pragma: no cover - optional extra
+    raise ImportError(
+        "DOCX export requires the 'export' extra. Install with: pip install 'textflowkit[export]'"
+    ) from exc
+
+
+def _hms(seconds: float) -> str:
+    return srt_timestamp(seconds).split(",")[0]
+
+
+def render_docx(transcript: Transcript, *, title: str = "Transcript") -> bytes:
+    """Render a transcript as a .docx document."""
+    doc = Document()
+    doc.add_heading(title, level=1)
+
+    meta = []
+    if transcript.source:
+        meta.append(f"Source: {transcript.source}")
+    if transcript.language:
+        meta.append(f"Language: {transcript.language}")
+    if transcript.duration:
+        meta.append(f"Duration: {_hms(transcript.duration)}")
+    diar = (transcript.metadata or {}).get("diarization")
+    if diar:
+        speakers = ", ".join(diar.get("speakers") or []) or "none"
+        meta.append(f"Speakers: {speakers}")
+    trans = (transcript.metadata or {}).get("translation")
+    if trans:
+        meta.append(f"Translated to: {trans.get('target')}")
+
+    for line in meta:
+        doc.add_paragraph(line)
+    doc.add_paragraph("")
+
+    current_speaker: str | None = None
+    for segment in transcript.segments:
+        if segment.hidden:
+            continue
+        if segment.speaker and segment.speaker != current_speaker:
+            current_speaker = segment.speaker
+            doc.add_heading(current_speaker, level=2)
+
+        para = doc.add_paragraph()
+        para.add_run(f"[{_hms(segment.start)}] ").bold = True
+        if segment.translated_text:
+            para.add_run(segment.translated_text.strip())
+            para.add_run(f"\n(original: {segment.text.strip()})").italic = True
+        else:
+            para.add_run(segment.text.strip())
+
+    buffer = io.BytesIO()
+    doc.save(buffer)
+    return buffer.getvalue()

textflowkit/render/fonts/OFL-NotoSans.txt

@@ -0,0 +1,94 @@
+Copyright 2018 The Noto Project Authors (github.com/googlei18n/noto-fonts)
+
+This Font Software is licensed under the SIL Open Font License,
+Version 1.1.
+
+This license is copied below, and is also available with a FAQ at:
+http://scripts.sil.org/OFL
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font
+creation efforts of academic and linguistic communities, and to
+provide a free and open framework in which fonts may be shared and
+improved in partnership with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded,
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply to
+any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software
+components as distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to,
+deleting, or substituting -- in part or in whole -- any of the
+components of the Original Version, by changing formats or by porting
+the Font Software to a new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed,
+modify, redistribute, and sell modified and unmodified copies of the
+Font Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components, in
+Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the
+corresponding Copyright Holder. This restriction only applies to the
+primary font name as presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created using
+the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.