salin 0.1.0__py3-none-any.whl

salin/__init__.py

@@ -0,0 +1,29 @@
+from importlib.metadata import PackageNotFoundError, version
+
+from .cleaner import clean_cues, clean_transcript
+from .client import Salin
+from .core import extract_info
+from .error import RateLimitedError, SubtitleDownloadError, TranscriptUnavailableError
+from .shared.models import LanguageMatch, SubtitleFormat, SubtitleSourcePolicy
+from .transcript import FetchedTranscript, TranscriptSnippet
+
+try:
+    __version__ = version("salin")
+except PackageNotFoundError:  # pragma: no cover - running from a source checkout
+    __version__ = "0.0.0"
+
+__all__ = [
+    "Salin",
+    "__version__",
+    "clean_cues",
+    "clean_transcript",
+    "extract_info",
+    "FetchedTranscript",
+    "TranscriptSnippet",
+    "RateLimitedError",
+    "SubtitleDownloadError",
+    "TranscriptUnavailableError",
+    "LanguageMatch",
+    "SubtitleSourcePolicy",
+    "SubtitleFormat",
+]

salin/cleaner/__init__.py

@@ -0,0 +1,19 @@
+"""
+cleaner/__init__.py
+
+Public surface of the cleaner package. Import from here rather than from
+sub-modules so internal reorganisation doesn't break callers.
+"""
+
+from salin.cleaner.cleaner import clean_cues, clean_transcript
+from salin.cleaner.dedup import deduplicate_blocks, sanitize_word
+from salin.cleaner.normalize import normalize_caption_text, strip_inline_markup
+
+__all__ = [
+    "clean_cues",
+    "clean_transcript",
+    "deduplicate_blocks",
+    "normalize_caption_text",
+    "sanitize_word",
+    "strip_inline_markup",
+]

salin/cleaner/cleaner.py

@@ -0,0 +1,88 @@
+"""Turn downloaded caption files into readable, deduplicated plain text."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterable
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+from salin.cleaner.dedup import deduplicate_blocks
+from salin.cleaner.normalize import normalize_caption_text
+from salin.cleaner.parsers import read_cues
+from salin.shared.models import SubtitleFile, SubtitleFormat, SubtitleResult
+
+log = logging.getLogger(__name__)
+
+
+def clean_cues(path: Path, subtitle_format: SubtitleFormat) -> str:
+    """
+    Parse a subtitle file and return the cleaned transcript as a single string.
+
+    Normalization runs before deduplication so stage directions like
+    ``[Laughter]`` don't break the overlap detection matching.
+    """
+    if not path.exists():
+        raise FileNotFoundError(path)
+
+    cues = read_cues(path, subtitle_format)
+    blocks = [normalize_caption_text(cue.text) for cue in cues]
+    return deduplicate_blocks(blocks)
+
+
+def _clean_one(subtitle: SubtitleFile, out_path: Path | None) -> Path:
+    if subtitle.sub_path is None:
+        raise ValueError("subtitle path is required")
+    if subtitle.format is None:
+        raise ValueError("subtitle format is required")
+
+    source = subtitle.sub_path
+    txt_path = (
+        out_path / source.with_suffix(".txt").name
+        if out_path is not None
+        else source.with_suffix(".txt")
+    )
+
+    transcript = clean_cues(source, subtitle.format)
+    txt_path.write_text(transcript, encoding="utf-8")
+    log.info("     ↳ clean transcript: %s", txt_path.name)
+    return txt_path
+
+
+def clean_transcript(
+    transcript_result: SubtitleResult | Iterable[SubtitleFile],
+    *,
+    output_dir: Path | str | None = None,
+    max_workers: int = 1,
+) -> tuple[Path, ...]:
+    """
+    Clean every successfully downloaded subtitle in ``transcript_result``.
+
+    Writes one ``.txt`` per subtitle. With ``output_dir`` set, the transcripts
+    land there; otherwise they are written next to the source subtitle.
+
+    Returns the paths written, in input order.
+    """
+    if max_workers < 1:
+        raise ValueError("max_workers must be at least 1")
+
+    out_path = Path(output_dir) if output_dir is not None else None
+    if out_path is not None:
+        out_path.mkdir(parents=True, exist_ok=True)
+
+    subtitles = (
+        transcript_result.ok_subs
+        if isinstance(transcript_result, SubtitleResult)
+        else transcript_result
+    )
+    subtitle_list = [s for s in subtitles if s.sub_path is not None]
+    if not subtitle_list:
+        return ()
+
+    if max_workers == 1 or len(subtitle_list) == 1:
+        return tuple(_clean_one(subtitle, out_path) for subtitle in subtitle_list)
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        return tuple(
+            executor.map(_clean_one, subtitle_list, [out_path] * len(subtitle_list))
+        )

salin/cleaner/dedup.py

@@ -0,0 +1,53 @@
+"""Collapse the rolling word overlap that auto-generated captions repeat."""
+
+from __future__ import annotations
+
+
+def sanitize_word(word: str) -> str:
+    """Fast alnum-only lowercase compare key (no regex)."""
+    return "".join(c.lower() for c in word if c.isalnum())
+
+
+def _matching_overlap(tail: list[str], head: list[str]) -> int:
+    """
+    Length of the longest overlap between the end of ``tail`` and the start of
+    ``head``, where tokens are compared by :func:`sanitize_word` so punctuation
+    and casing differences don't break a match.
+
+    Tokens that sanitize to an empty string (punctuation-only, e.g. ``"..."``)
+    never take part in a match. Comparing them literally would let two
+    unrelated punctuation tokens compare equal and merge unrelated blocks.
+    """
+    for size in range(min(len(tail), len(head)), 0, -1):
+        pairs = zip(tail[-size:], head[:size], strict=True)
+        if all(
+            tail_key and head_key and tail_key == head_key
+            for tail_key, head_key in pairs
+        ):
+            return size
+    return 0
+
+
+def deduplicate_blocks(blocks: list[str]) -> str:
+    """Merge parsed caption blocks into one deduplicated transcript."""
+    final_words: list[str] = []
+
+    for block in blocks:
+        block_words = block.split()
+        if not block_words:
+            continue
+
+        if not final_words:
+            final_words = block_words
+            continue
+
+        # Bounding strictly by the shorter list size.
+        # Capping this by a small constant prevented any duplicate longer
+        # than that constant from being matched.
+        window = min(len(final_words), len(block_words))
+        clean_tail = [sanitize_word(w) for w in final_words[-window:]]
+        clean_head = [sanitize_word(w) for w in block_words[:window]]
+
+        final_words.extend(block_words[_matching_overlap(clean_tail, clean_head) :])
+
+    return " ".join(final_words)

salin/cleaner/normalize.py

@@ -0,0 +1,54 @@
+"""Caption text normalization shared by the parsers and the deduplicator."""
+
+from __future__ import annotations
+
+import html
+import re
+
+_INLINE_TAG_RE = re.compile(r"<[^>]+>")
+_BREAK_TAG_RE = re.compile(r"<br\s*/?>", re.IGNORECASE)
+_BROKEN_TAG_RE = re.compile(r"<[^>]*\s*$")
+_MUSIC_CHAR_RE = re.compile(r"[\u266a\u266b\U0001f3b5\U0001f3b6]")
+
+# Matches speaker chevrons (>> or >) preceded by start-of-string or whitespace
+_SPEAKER_PREFIX_RE = re.compile(r"(?:^|\s)>{1,2}\s*")
+
+_BRACKET_STAGE_RE = re.compile(
+    r"\[(?:Music|Applause|Laughter|Silence|Inaudible|Crowd)[^\]]*\]",
+    re.IGNORECASE,
+)
+_PAREN_STAGE_RE = re.compile(
+    r"\((?:music|applause|laughter|inaudible|crowd noise)[^)]*\)",
+    re.IGNORECASE,
+)
+
+# A cue that is only music symbols, e.g. "[♪♪♪]", matches neither stage pattern
+# above because neither spells a keyword. The brackets survive the symbol
+# removal and reach the transcript as a stray "[]".
+_EMPTY_GROUP_RE = re.compile(r"\[\s*\]|\(\s*\)")
+
+
+def strip_inline_markup(text: str) -> str:
+    # <br> separates words, so it has to become a space before the generic tag
+    # stripper removes it -- otherwise "rules<br />and" collapses to
+    # "rulesand". TTML uses it; VTT and SRT use newlines instead.
+    text = _BREAK_TAG_RE.sub(" ", text)
+    text = _INLINE_TAG_RE.sub("", text)
+    text = _BROKEN_TAG_RE.sub("", text)
+    text = html.unescape(text)
+    return " ".join(text.split())
+
+
+def normalize_caption_text(text: str) -> str:
+    """Remove stage directions and caption artifacts while keeping spoken words."""
+    text = strip_inline_markup(text)
+    text = _MUSIC_CHAR_RE.sub("", text)
+    text = _EMPTY_GROUP_RE.sub("", text)
+
+    # Replace chevrons with a space to prevent merging adjacent words,
+    # then let the final split().join() clean up double spaces.
+    text = _SPEAKER_PREFIX_RE.sub(" ", text)
+
+    text = _BRACKET_STAGE_RE.sub("", text)
+    text = _PAREN_STAGE_RE.sub("", text)
+    return " ".join(text.split())

salin/cleaner/parsers/__init__.py

@@ -0,0 +1,97 @@
+"""
+parsers/__init__.py
+
+Registry mapping SubtitleFormat -> parser. Adding a format means adding one
+entry here plus a module that returns ParsedCues.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from salin.error import UnsupportedFormatError
+from salin.shared.models import SubtitleFormat
+
+from .common import CueReader, ParsedCue, SourceParser, timestamp_to_ms
+from .json3 import parse_json3_cues, read_json3_cues
+from .srt import parse_srt_cues, read_srt_cues
+from .srv import (
+    parse_srv1_cues,
+    parse_srv2_cues,
+    parse_srv3_cues,
+    read_srv1_cues,
+    read_srv2_cues,
+    read_srv3_cues,
+)
+from .ttml import parse_ttml_cues, read_ttml_cues
+from .vtt import parse_vtt_cues, read_vtt_cues
+
+CUE_READERS: dict[SubtitleFormat, CueReader] = {
+    SubtitleFormat.SRT: read_srt_cues,
+    SubtitleFormat.VTT: read_vtt_cues,
+    SubtitleFormat.JSON3: read_json3_cues,
+    SubtitleFormat.SRV1: read_srv1_cues,
+    SubtitleFormat.SRV2: read_srv2_cues,
+    SubtitleFormat.SRV3: read_srv3_cues,
+    SubtitleFormat.TTML: read_ttml_cues,
+}
+
+SOURCE_PARSERS: dict[SubtitleFormat, SourceParser] = {
+    SubtitleFormat.SRT: parse_srt_cues,
+    SubtitleFormat.VTT: parse_vtt_cues,
+    SubtitleFormat.JSON3: parse_json3_cues,
+    SubtitleFormat.SRV1: parse_srv1_cues,
+    SubtitleFormat.SRV2: parse_srv2_cues,
+    SubtitleFormat.SRV3: parse_srv3_cues,
+    SubtitleFormat.TTML: parse_ttml_cues,
+}
+
+SUPPORTED_FORMATS = tuple(CUE_READERS)
+
+
+def reader_for(subtitle_format: SubtitleFormat) -> CueReader:
+    reader = CUE_READERS.get(subtitle_format)
+    if reader is None:
+        supported = ", ".join(f"'{f}'" for f in SUPPORTED_FORMATS)
+        raise UnsupportedFormatError(
+            f"No subtitle parser for format '{subtitle_format}'. Supported: {supported}"
+        )
+    return reader
+
+
+def read_cues(path: Path, subtitle_format: SubtitleFormat) -> list[ParsedCue]:
+    """Read a subtitle file into fully-resolved cues."""
+    return reader_for(subtitle_format)(path)
+
+
+def read_blocks(path: Path, subtitle_format: SubtitleFormat) -> list[str]:
+    """Read a subtitle file into text blocks, one per cue."""
+    return [cue.text for cue in read_cues(path, subtitle_format)]
+
+
+__all__ = [
+    "CUE_READERS",
+    "SOURCE_PARSERS",
+    "SUPPORTED_FORMATS",
+    "CueReader",
+    "SourceParser",
+    "ParsedCue",
+    "parse_json3_cues",
+    "parse_srt_cues",
+    "parse_srv1_cues",
+    "parse_srv2_cues",
+    "parse_srv3_cues",
+    "parse_ttml_cues",
+    "parse_vtt_cues",
+    "read_blocks",
+    "read_cues",
+    "read_json3_cues",
+    "read_srt_cues",
+    "read_srv1_cues",
+    "read_srv2_cues",
+    "read_srv3_cues",
+    "read_ttml_cues",
+    "read_vtt_cues",
+    "reader_for",
+    "timestamp_to_ms",
+]

salin/cleaner/parsers/common.py

@@ -0,0 +1,64 @@
+"""Shared types for subtitle cue parsers."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass
+from pathlib import Path
+from xml.etree import ElementTree
+
+
+@dataclass(frozen=True, slots=True)
+class ParsedCue:
+    index: int
+    start_ms: int
+    end_ms: int
+    text: str
+
+
+type CueReader = Callable[[Path], list[ParsedCue]]
+
+# Every parse_*_cues function takes the file's full text, so callers can pick
+# whichever form is convenient and the registry stays uniform.
+type SourceParser = Callable[[str], list[ParsedCue]]
+
+
+def timestamp_to_ms(value: str) -> int:
+    """Converts a standard caption timestamp string (with , or .) into milliseconds."""
+    normalized = value.strip().replace(",", ".")
+    hours, minutes, seconds = normalized.split(":")
+    total_seconds = int(hours) * 3600 + int(minutes) * 60 + float(seconds)
+    return int(total_seconds * 1000)
+
+
+def iter_local(root: ElementTree.Element, name: str) -> Iterator[ElementTree.Element]:
+    """
+    Yield descendants whose tag is ``name``, ignoring any XML namespace.
+
+    A namespaced document qualifies every tag as ``{uri}name``, so a plain
+    ``root.iter("p")`` silently finds nothing in TTML, which is always served
+    with ``xmlns="http://www.w3.org/ns/ttml"``.
+    """
+    for node in root.iter():
+        if isinstance(node.tag, str) and node.tag.rpartition("}")[2] == name:
+            yield node
+
+
+def element_text(node: ElementTree.Element) -> str:
+    """
+    Flatten an element's full text content, turning ``<br>`` into a space.
+
+    ``node.text`` alone stops at the first child, so for TTML
+    ``<p>rules<br />and</p>`` would yield just "rules" and silently drop the
+    rest of the cue.
+    """
+    parts: list[str] = [node.text or ""]
+
+    for child in node:
+        if isinstance(child.tag, str) and child.tag.rpartition("}")[2] == "br":
+            parts.append(" ")
+        else:
+            parts.append(element_text(child))
+        parts.append(child.tail or "")
+
+    return "".join(parts)

salin/cleaner/parsers/json3.py

@@ -0,0 +1,72 @@
+"""
+json3.py
+
+YouTube's JSON caption format. Structured rather than rolling, so the cues come
+out already separated -- no cue-boundary heuristics needed::
+
+    {"events": [{"tStartMs": 1360, "dDurationMs": 1680,
+                 "segs": [{"utf8": "..."}]}]}
+
+Events with no ``segs`` are gaps between captions and carry no text; ``segs``
+entries hold contiguous chunks of one line, so they join without a separator.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from salin.cleaner.normalize import normalize_caption_text
+
+from .common import ParsedCue
+
+
+def parse_json3_cues(source: str) -> list[ParsedCue]:
+    payload: Any = json.loads(source)
+    if not isinstance(payload, dict):
+        raise ValueError("json3 payload must be a JSON object")
+
+    events = payload.get("events")
+    if not isinstance(events, list):
+        raise ValueError("json3 payload has no 'events' list")
+
+    cues: list[ParsedCue] = []
+
+    for event in events:
+        if not isinstance(event, dict):
+            continue
+
+        start = event.get("tStartMs")
+        if not isinstance(start, int):
+            continue
+
+        segs = event.get("segs")
+        if not isinstance(segs, list):
+            continue
+
+        text = normalize_caption_text(
+            "".join(
+                seg["utf8"] for seg in segs if isinstance(seg, dict) and seg.get("utf8")
+            )
+        )
+        if not text:
+            continue
+
+        duration = event.get("dDurationMs")
+        cues.append(
+            ParsedCue(
+                index=len(cues),
+                start_ms=start,
+                end_ms=start + (duration if isinstance(duration, int) else 0),
+                text=text,
+            )
+        )
+
+    return cues
+
+
+def read_json3_cues(path: Path) -> list[ParsedCue]:
+    if not path.exists():
+        raise FileNotFoundError(path)
+    return parse_json3_cues(path.read_text(encoding="utf-8-sig"))

salin/cleaner/parsers/srt.py

@@ -0,0 +1,71 @@
+import re
+from pathlib import Path
+
+from salin.cleaner.normalize import normalize_caption_text
+
+from .common import ParsedCue, timestamp_to_ms
+
+_SRT_TIMESTAMP_RE = re.compile(
+    r"(\d{2}:\d{2}:\d{2},\d{3})\s+-->\s+(\d{2}:\d{2}:\d{2},\d{3})"
+)
+_SRT_SEQUENCE_RE = re.compile(r"^\d+$")
+
+
+def parse_srt_cues(source: str) -> list[ParsedCue]:
+    """Parse SRT source text into structured SRT ParsedCues."""
+    cues: list[ParsedCue] = []
+    current_lines: list[str] = []
+    start_ms: int | None = None
+    end_ms: int | None = None
+
+    def flush() -> None:
+        nonlocal start_ms, end_ms, current_lines
+        if start_ms is None or end_ms is None:
+            current_lines = []
+            return
+
+        # Merge lines and normalize them using the shared normalization logic
+        text = normalize_caption_text(" ".join(current_lines)) if current_lines else ""
+        if text:
+            cues.append(
+                ParsedCue(
+                    index=len(cues),
+                    start_ms=start_ms,
+                    end_ms=end_ms,
+                    text=text,
+                )
+            )
+        current_lines = []
+        start_ms = None
+        end_ms = None
+
+    for raw in source.splitlines():
+        line = raw.strip()
+        if not line:
+            flush()
+            continue
+
+        match = _SRT_TIMESTAMP_RE.search(line)
+        if match:
+            start_ms = timestamp_to_ms(match.group(1))
+            end_ms = timestamp_to_ms(match.group(2))
+            continue
+
+        # A bare index only ever precedes a timestamp, so a digit-only line
+        # before one is the sequence number. After a timestamp the same line is
+        # real text -- a caption can legitimately be just "2024".
+        if start_ms is None and _SRT_SEQUENCE_RE.match(line):
+            continue
+
+        current_lines.append(line)
+
+    flush()
+    return cues
+
+
+def read_srt_cues(path: Path) -> list[ParsedCue]:
+    """Read an SRT file and return fully-resolved ParsedCues (including times)."""
+    if not path.exists():
+        raise FileNotFoundError(path)
+    with open(path, encoding="utf-8-sig") as f:
+        return parse_srt_cues(f.read())