clipscribe 0.1.0__py3-none-any.whl

clipscribe/__init__.py

@@ -0,0 +1,52 @@
+from importlib.metadata import PackageNotFoundError, version
+
+from .extractor import ExtractionResponse, TranscriptExtractor
+from .utils.extras import missing_extra
+from .utils.transcript_utils import (
+    TranscriptResult,
+    TranscriptSaver,
+    TranscriptSegment,
+    TranscriptSource,
+)
+
+try:
+    __version__ = version("clipscribe")
+except PackageNotFoundError:
+    __version__ = "0.1.0"
+
+__all__ = [
+    "ExtractionResponse",
+    "GenericProxyConfig",
+    "InstagramTranscriptExtractor",
+    "TikTokTranscriptExtractor",
+    "TranscriptExtractor",
+    "TranscriptResult",
+    "TranscriptSaver",
+    "TranscriptSegment",
+    "TranscriptSource",
+    "YouTubeTranscriptExtractor",
+    "__version__",
+]
+
+
+def __dir__():
+    return sorted(set(globals()) | set(__all__))
+
+
+def __getattr__(name: str):
+    if name == "GenericProxyConfig":
+        try:
+            from youtube_transcript_api.proxies import GenericProxyConfig
+        except ImportError as exc:
+            raise missing_extra("youtube", "YouTube proxy config") from exc
+        return GenericProxyConfig
+    if name == "YouTubeTranscriptExtractor":
+        from .utils.youtube_utils import YouTubeTranscriptExtractor
+        return YouTubeTranscriptExtractor
+    if name == "TikTokTranscriptExtractor":
+        from .utils.tiktok_utils import TikTokTranscriptExtractor
+        return TikTokTranscriptExtractor
+    if name == "InstagramTranscriptExtractor":
+        from .utils.instagram_utils import InstagramTranscriptExtractor
+        return InstagramTranscriptExtractor
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

clipscribe/extractor.py

@@ -0,0 +1,295 @@
+"""Unified transcript extraction API for YouTube, TikTok, and Instagram."""
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+from typing import Callable
+
+from .utils.transcript_utils import PlatformExtractor, TranscriptResult, TranscriptSaver
+
+log = logging.getLogger(__name__)
+
+_NON_RETRYABLE = (ValueError, FileNotFoundError, ImportError)
+
+
+class ExtractionResponse:
+    """
+    Wraps a TranscriptResult alongside optional save paths.
+    This is what TranscriptExtractor.extract() always returns.
+
+    Attributes:
+        result:     The raw TranscriptResult (segments, metadata, full_text).
+        saved:      Dict of format -> Path if save=True, else None.
+        url:        The original URL that was extracted.
+        platform:   Detected TranscriptSource.
+    """
+
+    def __init__(self, result: TranscriptResult, saved: dict | None = None):
+        self.result = result
+        self.saved = saved
+        self.url = result.url
+        self.platform = result.source
+
+    @property
+    def full_text(self) -> str:
+        return self.result.full_text
+
+    @property
+    def segments(self):
+        return self.result.segments
+
+    @property
+    def metadata(self):
+        return self.result.metadata
+
+    def to_dict(self) -> dict:
+        data = self.result.to_dict()
+        if self.saved:
+            data["saved_files"] = {k: str(v) for k, v in self.saved.items()}
+        return data
+
+    def __repr__(self) -> str:
+        return (
+            f"<ExtractionResponse platform={self.platform.value!r} "
+            f"segments={len(self.segments)} url={self.url!r}>"
+        )
+
+
+class TranscriptExtractor:
+    """
+    Unified transcript extraction API.
+
+    Built-in extractors and extra platforms added via ``register()`` are
+    per-instance.
+
+    Args:
+        language:        Preferred transcript language for YouTube (default: "en").
+        whisper_model:   faster-whisper model size for TikTok/Instagram (default: "tiny").
+        whisper_device:  "cpu" or "cuda" (default: "cpu").
+        whisper_compute: Quantization type for faster-whisper (default: "int8").
+        output_dir:      Directory for saved transcripts (default: "output", relative to CWD).
+        allowed_outputs: File formats to write when save=True.
+                         Choose from "json", "txt", "srt" (default: json).
+        on_error:        Optional callback(url, exc) called on per-URL failures
+                         in extract_many(). Defaults to logging the error.
+        youtube_proxy_config: Optional youtube-transcript-api ProxyConfig.
+        tiktok_cookies_from_browser: Browser name for TikTok cookies.
+        tiktok_cookies:  Netscape cookies.txt path for TikTok.
+        instagram_cookies_from_browser: Browser name for Instagram cookies.
+        instagram_cookies: Netscape cookies.txt path for Instagram.
+        max_duration_s:  Reject TikTok/Instagram videos longer than this
+                         (default: 900). Pass None for no limit.
+    """
+
+    def __init__(
+        self,
+        language: str = "en",
+        whisper_model: str = "tiny",
+        whisper_device: str = "cpu",
+        whisper_compute: str = "int8",
+        output_dir: str = "output",
+        allowed_outputs: list[str] | None = None,
+        on_error: Callable[[str, Exception], None] | None = None,
+        instagram_cookies_from_browser: str | None = None,
+        instagram_cookies: str | None = None,
+        tiktok_cookies_from_browser: str | None = None,
+        tiktok_cookies: str | None = None,
+        youtube_proxy_config=None,
+        max_duration_s: int | None = 900,
+    ):
+        self._saver = TranscriptSaver(output_dir=output_dir, allowed_outputs=allowed_outputs)
+        self._on_error = on_error or self._default_error_handler
+        self._registry: list[tuple[re.Pattern, PlatformExtractor]] = []
+        self._extra: list[tuple[re.Pattern, PlatformExtractor]] = []
+
+        self._register_defaults(
+            language=language,
+            whisper_model=whisper_model,
+            whisper_device=whisper_device,
+            whisper_compute=whisper_compute,
+            instagram_cookies_from_browser=instagram_cookies_from_browser,
+            instagram_cookies=instagram_cookies,
+            tiktok_cookies_from_browser=tiktok_cookies_from_browser,
+            tiktok_cookies=tiktok_cookies,
+            youtube_proxy_config=youtube_proxy_config,
+            max_duration_s=max_duration_s,
+        )
+
+    def extract(
+        self,
+        url: str,
+        save: bool = False,
+        max_retries: int = 0,
+        allowed_outputs: list[str] | None = None,
+    ) -> ExtractionResponse:
+        """
+        Extract transcript from a single URL.
+
+        Args:
+            url:              A YouTube, TikTok, or Instagram URL (or any registered platform).
+            save:             If True, saves files to the configured output_dir.
+            max_retries:      Extra attempts after a failed extract (default: 0).
+            allowed_outputs:  Formats for this call only. Choose from
+                              "json", "txt", "srt". Defaults to the instance list.
+
+        Returns:
+            ExtractionResponse with .full_text, .segments, .metadata, .to_dict()
+
+        Raises:
+            ValueError:   If the URL doesn't match any registered platform.
+            RuntimeError: If extraction fails.
+            ImportError:  If the matching platform extra is not installed.
+        """
+        extractor = self._resolve(url)
+        if allowed_outputs is not None and not save:
+            log.warning("allowed_outputs is ignored because save=False")
+        max_retries = max(0, max_retries)
+        result: TranscriptResult | None = None
+        for attempt in range(max_retries + 1):
+            try:
+                result = extractor.extract(url)
+                break
+            except _NON_RETRYABLE:
+                raise
+            except Exception:
+                if attempt >= max_retries:
+                    raise
+                time.sleep(min(8, 2 ** attempt))
+        if result is None:
+            raise RuntimeError(f"Extraction failed for {url}")
+        saved = (
+            self._saver.save(result, self._slug(result), allowed_outputs=allowed_outputs)
+            if save
+            else None
+        )
+        return ExtractionResponse(result=result, saved=saved)
+
+    def extract_many(
+        self,
+        urls: list[str],
+        save: bool = False,
+        max_retries: int = 0,
+        allowed_outputs: list[str] | None = None,
+    ) -> list[ExtractionResponse]:
+        """
+        Extract transcripts from multiple URLs.
+        Failures are caught, passed to on_error, and skipped — processing continues.
+        """
+        if allowed_outputs is not None and not save:
+            log.warning("allowed_outputs is ignored because save=False")
+            allowed_outputs = None
+        responses = []
+        for url in urls:
+            try:
+                responses.append(
+                    self.extract(
+                        url,
+                        save=save,
+                        max_retries=max_retries,
+                        allowed_outputs=allowed_outputs,
+                    )
+                )
+            except Exception as exc:
+                self._on_error(url, exc)
+        return responses
+
+    def supports(self, url: str) -> bool:
+        """Return True if the URL matches a registered platform."""
+        try:
+            self._resolve(url)
+            return True
+        except ValueError:
+            return False
+
+    def register(self, url_pattern: str, extractor: PlatformExtractor) -> None:
+        """
+        Register a new platform extractor on this instance.
+
+        Args:
+            url_pattern: A regex string matched against the full URL.
+            extractor:   Any object with an .extract(url) -> TranscriptResult method.
+        """
+        self._extra.append((re.compile(url_pattern, re.IGNORECASE), extractor))
+
+    def _register_defaults(
+        self,
+        language: str,
+        whisper_model: str,
+        whisper_device: str,
+        whisper_compute: str,
+        instagram_cookies_from_browser: str | None = None,
+        instagram_cookies: str | None = None,
+        tiktok_cookies_from_browser: str | None = None,
+        tiktok_cookies: str | None = None,
+        youtube_proxy_config=None,
+        max_duration_s: int | None = 900,
+    ) -> None:
+        from .utils.youtube_utils import YouTubeTranscriptExtractor
+        from .utils.tiktok_utils import TikTokTranscriptExtractor
+
+        self._registry = [
+            (
+                re.compile(r"youtube\.com|youtu\.be", re.IGNORECASE),
+                YouTubeTranscriptExtractor(
+                    language=language,
+                    proxy_config=youtube_proxy_config,
+                ),
+            ),
+            (
+                re.compile(r"tiktok\.com", re.IGNORECASE),
+                TikTokTranscriptExtractor(
+                    whisper_model=whisper_model,
+                    device=whisper_device,
+                    compute_type=whisper_compute,
+                    cookies_from_browser=tiktok_cookies_from_browser,
+                    cookies_file=tiktok_cookies,
+                    max_duration_s=max_duration_s,
+                ),
+            ),
+        ]
+
+        if instagram_cookies_from_browser or instagram_cookies:
+            from .utils.instagram_utils import InstagramTranscriptExtractor
+
+            self._registry.append((
+                re.compile(r"instagram\.com", re.IGNORECASE),
+                InstagramTranscriptExtractor(
+                    cookies_from_browser=instagram_cookies_from_browser,
+                    cookies_file=instagram_cookies,
+                    whisper_model=whisper_model,
+                    device=whisper_device,
+                    compute_type=whisper_compute,
+                    max_duration_s=max_duration_s,
+                ),
+            ))
+
+    def _resolve(self, url: str) -> PlatformExtractor:
+        for pattern, extractor in (*self._registry, *self._extra):
+            if pattern.search(url):
+                return extractor
+        if "instagram.com" in url.lower():
+            raise ValueError(
+                "Instagram URLs require authentication. "
+                "Pass instagram_cookies_from_browser or instagram_cookies."
+            )
+        raise ValueError(
+            f"No extractor registered for URL: {url}\n"
+            f"Registered platforms: {self._registered_platforms()}"
+        )
+
+    def _slug(self, result: TranscriptResult) -> str:
+        media_id = (
+            result.metadata.get("video_id")
+            or result.metadata.get("post_id")
+            or "unknown"
+        )
+        return re.sub(r'[\\/*?:"<>|]', "_", f"{result.source.value}_{media_id}").strip()
+
+    def _registered_platforms(self) -> list[str]:
+        return [pattern.pattern for pattern, _ in (*self._registry, *self._extra)]
+
+    @staticmethod
+    def _default_error_handler(url: str, exc: Exception) -> None:
+        log.error("Failed (%s): %s", url, exc)

clipscribe/utils/extras.py

@@ -0,0 +1,10 @@
+"""Install-extra hints for optional platform dependencies."""
+
+def extra_install(platform: str) -> str:
+    return f'pip install "clipscribe[{platform}]" or "clipscribe[all]"'
+
+
+def missing_extra(platform: str, what: str) -> ImportError:
+    return ImportError(
+        f"{what} requires extra dependencies. Install with: {extra_install(platform)}"
+    )

clipscribe/utils/instagram_utils.py

@@ -0,0 +1,164 @@
+"""
+instagram_utils.py
+───────────────────
+Instagram transcript extraction via yt-dlp (audio download) + faster-whisper.
+
+Instagram requires authentication. Two options, in recommended order:
+
+  1. cookies_from_browser (BEST — always fresh, no manual export needed)
+  2. cookies_file (OK — but expires fast)
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import tempfile
+from urllib.parse import urlparse
+
+from .transcript_utils import (
+    BaseTranscriptExtractor,
+    TranscriptResult,
+    TranscriptSource,
+)
+from .whisper_backend import load_whisper_model, transcribe_to_segments
+from .ydl_audio import (
+    SUPPORTED_BROWSERS,
+    build_audio_ydl_opts,
+    download_audio,
+    validate_browser,
+    validate_cookies_file,
+)
+
+_URL_PATTERNS: list[tuple[re.Pattern[str], str]] = [
+    (re.compile(r"instagram\.com/share/reel/([A-Za-z0-9_-]+)", re.IGNORECASE), "reel"),
+    (re.compile(r"instagram\.com/share/p/([A-Za-z0-9_-]+)", re.IGNORECASE), "p"),
+    (re.compile(r"instagram\.com/reels/([A-Za-z0-9_-]+)", re.IGNORECASE), "reel"),
+    (re.compile(r"instagram\.com/reel/([A-Za-z0-9_-]+)", re.IGNORECASE), "reel"),
+    (re.compile(r"instagram\.com/p/([A-Za-z0-9_-]+)", re.IGNORECASE), "p"),
+    (re.compile(r"instagram\.com/tv/([A-Za-z0-9_-]+)", re.IGNORECASE), "tv"),
+]
+
+
+def parse_instagram_post(url: str) -> tuple[str | None, str]:
+    for pattern, kind in _URL_PATTERNS:
+        m = pattern.search(url)
+        if m:
+            return m.group(1), kind
+    return None, ""
+
+
+def canonical_instagram_media_url(url: str, post_id: str, kind: str) -> str:
+    host = (urlparse(url.strip()).hostname or "").lower()
+    if host not in ("www.instagram.com", "instagram.com", "m.instagram.com"):
+        raise ValueError(f"Instagram transcript URL host not allowed: {host!r}")
+    base = "https://www.instagram.com"
+    if kind == "p":
+        return f"{base}/p/{post_id}/"
+    if kind == "tv":
+        return f"{base}/tv/{post_id}/"
+    return f"{base}/reel/{post_id}/"
+
+
+class InstagramTranscriptExtractor(BaseTranscriptExtractor):
+    """
+    Downloads Instagram audio with yt-dlp and transcribes it with faster-whisper.
+
+    Requires: pip install "clipscribe[instagram]"
+
+    Instagram requires authentication — provide cookies_from_browser or cookies_file.
+
+    Args:
+        cookies_from_browser: Browser name to read live cookies from.
+        cookies_file:         Path to a Netscape-format cookies.txt file.
+        whisper_model:        Whisper model size (default: "tiny").
+        device:               "cpu" or "cuda" (default: "cpu").
+        compute_type:         Quantization — "int8", "float16", "float32".
+        ydl_opts_extra:       Optional dict merged into yt-dlp options.
+        max_duration_s:       Reject videos longer than this (default: 900).
+                              Pass None for no limit.
+        socket_timeout:       yt-dlp socket timeout in seconds (default: 30).
+    """
+
+    SUPPORTED_BROWSERS = SUPPORTED_BROWSERS
+
+    def __init__(
+        self,
+        cookies_from_browser: str | None = None,
+        cookies_file: str | None = None,
+        whisper_model: str = "tiny",
+        device: str = "cpu",
+        compute_type: str = "int8",
+        ydl_opts_extra: dict | None = None,
+        max_duration_s: int | None = 900,
+        socket_timeout: int = 30,
+    ):
+        if not any([cookies_from_browser, cookies_file]):
+            raise ValueError(
+                "Instagram requires authentication. Provide one of:\n"
+                "  - cookies_from_browser: 'chrome', 'firefox', 'edge', etc. (recommended)\n"
+                "  - cookies_file: path to a Netscape-format cookies.txt"
+            )
+
+        validate_browser(cookies_from_browser)
+        validate_cookies_file(cookies_file)
+
+        self.cookies_from_browser = cookies_from_browser
+        self.cookies_file = cookies_file
+        self.whisper_model = whisper_model
+        self.device = device
+        self.compute_type = compute_type
+        self.ydl_opts_extra = ydl_opts_extra or {}
+        self.max_duration_s = max_duration_s
+        self.socket_timeout = socket_timeout
+        self._model = None
+
+    def extract(self, url: str) -> TranscriptResult:
+        post_id, kind = parse_instagram_post(url)
+        if not post_id or not kind:
+            raise ValueError(
+                f"Unsupported Instagram URL: {url}\n"
+                "Supported formats: /reel/<id>/, /p/<id>/, /tv/<id>/"
+            )
+        fetch_url = canonical_instagram_media_url(url, post_id, kind)
+
+        with tempfile.TemporaryDirectory() as tmp:
+            audio_path, ydl_info = self._download_audio(fetch_url, tmp)
+            self.load_model()
+            segments, transcription_info = transcribe_to_segments(self._model, audio_path)
+
+        result = self._make_result(
+            fetch_url,
+            source=TranscriptSource.INSTAGRAM,
+            post_id=post_id,
+            title=ydl_info.get("title", ""),
+            author=ydl_info.get("uploader", ""),
+            language_detected=transcription_info.language,
+            language_probability=round(transcription_info.language_probability, 3),
+            whisper_model=self.whisper_model,
+        )
+        result.segments = segments
+        result.duration_s = ydl_info.get("duration")
+        return result
+
+    def load_model(self) -> None:
+        if self._model is None:
+            self._model = load_whisper_model(
+                self.whisper_model,
+                self.device,
+                self.compute_type,
+                extra="instagram",
+            )
+
+    def _download_audio(self, url: str, tmp_dir: str) -> tuple[str, dict]:
+        template = os.path.join(tmp_dir, "audio.%(ext)s")
+        opts = build_audio_ydl_opts(
+            template,
+            cookies_from_browser=self.cookies_from_browser,
+            cookies_file=self.cookies_file,
+            ydl_opts_extra=self.ydl_opts_extra,
+            socket_timeout=self.socket_timeout,
+            max_duration_s=self.max_duration_s,
+            extra="instagram",
+        )
+        return download_audio(url, tmp_dir, opts, platform="instagram")

clipscribe/utils/tiktok_utils.py

@@ -0,0 +1,142 @@
+"""
+tiktok_utils.py
+────────────────
+TikTok transcript extraction via yt-dlp (audio download) + faster-whisper.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import tempfile
+from urllib.parse import urlparse
+
+from .transcript_utils import (
+    BaseTranscriptExtractor,
+    TranscriptResult,
+    TranscriptSource,
+)
+from .whisper_backend import load_whisper_model, transcribe_to_segments
+from .ydl_audio import (
+    build_audio_ydl_opts,
+    download_audio,
+    validate_browser,
+    validate_cookies_file,
+)
+
+_SHORT_HOSTS = {"vm.tiktok.com", "vt.tiktok.com"}
+
+
+def assert_allowed_tiktok_url(url: str) -> None:
+    u = urlparse((url or "").strip())
+    host = (u.hostname or "").lower()
+    if not host:
+        raise ValueError("TikTok URL is missing a host")
+    if not (
+        host in ("tiktok.com", "www.tiktok.com", "m.tiktok.com", "vm.tiktok.com", "vt.tiktok.com")
+        or host.endswith(".tiktok.com")
+    ):
+        raise ValueError(f"TikTok transcript URL host not allowed: {host!r}")
+
+    path = u.path or ""
+    if host in _SHORT_HOSTS:
+        if path.strip("/"):
+            return
+        raise ValueError("TikTok short URL is missing a path")
+    if re.search(r"/video/\d+", path):
+        return
+    if re.search(r"/v/\d+", path):
+        return
+    if re.search(r"/t/[A-Za-z0-9]+", path):
+        return
+    raise ValueError(
+        "TikTok URL must be a single video (e.g. /@user/video/ID or vm.tiktok.com/...), "
+        "not a profile, tag, or playlist."
+    )
+
+
+class TikTokTranscriptExtractor(BaseTranscriptExtractor):
+    """
+    Downloads TikTok audio with yt-dlp and transcribes it with faster-whisper.
+
+    Requires: pip install "clipscribe[tiktok]"
+
+    Args:
+        whisper_model:   Whisper model size — "tiny", "base", "small",
+                         "medium", or "large" (default: "tiny").
+        device:          "cpu" or "cuda" (default: "cpu").
+        compute_type:    Quantization type — e.g. "int8", "float16", "float32".
+        cookies_from_browser: Browser name to read cookies from.
+        cookies_file:         Path to a Netscape-format cookies.txt file.
+        ydl_opts_extra:       Optional dict merged into yt-dlp options.
+        max_duration_s:  Reject videos longer than this (default: 900).
+                         Pass None for no limit.
+        socket_timeout:  yt-dlp socket timeout in seconds (default: 30).
+    """
+
+    def __init__(
+        self,
+        whisper_model: str = "tiny",
+        device: str = "cpu",
+        compute_type: str = "int8",
+        cookies_from_browser: str | None = None,
+        cookies_file: str | None = None,
+        ydl_opts_extra: dict | None = None,
+        max_duration_s: int | None = 900,
+        socket_timeout: int = 30,
+    ):
+        validate_browser(cookies_from_browser)
+        validate_cookies_file(cookies_file)
+
+        self.whisper_model = whisper_model
+        self.device = device
+        self.compute_type = compute_type
+        self.cookies_from_browser = cookies_from_browser
+        self.cookies_file = cookies_file
+        self.ydl_opts_extra = ydl_opts_extra or {}
+        self.max_duration_s = max_duration_s
+        self.socket_timeout = socket_timeout
+        self._model = None
+
+    def extract(self, url: str) -> TranscriptResult:
+        assert_allowed_tiktok_url(url)
+        with tempfile.TemporaryDirectory() as tmp:
+            audio_path, ydl_info = self._download_audio(url, tmp)
+            self.load_model()
+            segments, transcription_info = transcribe_to_segments(self._model, audio_path)
+
+        result = self._make_result(
+            url,
+            source=TranscriptSource.TIKTOK,
+            video_id=ydl_info.get("id", "unknown"),
+            title=ydl_info.get("title", ""),
+            author=ydl_info.get("uploader", ""),
+            language_detected=transcription_info.language,
+            language_probability=round(transcription_info.language_probability, 3),
+            whisper_model=self.whisper_model,
+        )
+        result.segments = segments
+        result.duration_s = ydl_info.get("duration")
+        return result
+
+    def load_model(self) -> None:
+        if self._model is None:
+            self._model = load_whisper_model(
+                self.whisper_model,
+                self.device,
+                self.compute_type,
+                extra="tiktok",
+            )
+
+    def _download_audio(self, url: str, tmp_dir: str) -> tuple[str, dict]:
+        template = os.path.join(tmp_dir, "audio.%(ext)s")
+        opts = build_audio_ydl_opts(
+            template,
+            cookies_from_browser=self.cookies_from_browser,
+            cookies_file=self.cookies_file,
+            ydl_opts_extra=self.ydl_opts_extra,
+            socket_timeout=self.socket_timeout,
+            max_duration_s=self.max_duration_s,
+            extra="tiktok",
+        )
+        return download_audio(url, tmp_dir, opts, platform="tiktok")