kabigon 0.14.2__py3-none-any.whl

kabigon/loaders/reddit.py

@@ -0,0 +1,86 @@
+from urllib.parse import urlparse
+from urllib.parse import urlunparse
+
+from playwright.async_api import async_playwright
+
+from kabigon.core.exception import InvalidURLError
+from kabigon.core.loader import Loader
+
+from .utils import html_to_markdown
+
+REDDIT_DOMAINS = [
+    "reddit.com",
+    "www.reddit.com",
+    "old.reddit.com",
+]
+
+USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+)
+
+
+def check_reddit_url(url: str) -> None:
+    """Check if URL is from Reddit.
+
+    Args:
+        url: The URL to check
+
+    Raises:
+        ValueError: If URL is not from Reddit
+    """
+    netloc = urlparse(url).netloc
+    if netloc not in REDDIT_DOMAINS:
+        raise InvalidURLError(url, "Reddit")
+
+
+def convert_to_old_reddit(url: str) -> str:
+    """Convert Reddit URL to old.reddit.com format.
+
+    Args:
+        url: Original Reddit URL
+
+    Returns:
+        URL with old.reddit.com domain
+    """
+    parsed = urlparse(url)
+    return str(urlunparse(parsed._replace(netloc="old.reddit.com")))
+
+
+class RedditLoader(Loader):
+    """Loader for Reddit posts and comments.
+
+    Uses old.reddit.com for better content extraction without CAPTCHA.
+    """
+
+    def __init__(self, timeout: float = 30_000) -> None:
+        """Initialize RedditLoader.
+
+        Args:
+            timeout: Timeout in milliseconds for page loading (default: 30 seconds)
+        """
+        self.timeout = timeout
+
+    async def load(self, url: str) -> str:
+        """Asynchronously load Reddit content from URL.
+
+        Args:
+            url: Reddit URL to load
+
+        Returns:
+            Loaded content as markdown
+
+        Raises:
+            ValueError: If URL is not from Reddit
+        """
+        check_reddit_url(url)
+        url = convert_to_old_reddit(url)
+
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            context = await browser.new_context(user_agent=USER_AGENT)
+            page = await context.new_page()
+            await page.goto(url, timeout=self.timeout, wait_until="networkidle")
+            content = await page.content()
+            await browser.close()
+
+            return html_to_markdown(content)

kabigon/loaders/reel.py

@@ -0,0 +1,24 @@
+from kabigon.core.exception import InvalidURLError
+from kabigon.core.loader import Loader
+
+from .httpx import HttpxLoader
+from .ytdlp import YtdlpLoader
+
+
+def check_reel_url(url: str) -> None:
+    if not url.startswith("https://www.instagram.com/reel"):
+        raise InvalidURLError(url, "Instagram Reel")
+
+
+class ReelLoader(Loader):
+    def __init__(self) -> None:
+        self.httpx_loader = HttpxLoader()
+        self.ytdlp_loader = YtdlpLoader()
+
+    async def load(self, url: str) -> str:
+        check_reel_url(url)
+
+        audio_content = await self.ytdlp_loader.load(url)
+        html_content = await self.httpx_loader.load(url)
+
+        return f"{audio_content}\n\n{html_content}"

kabigon/loaders/truthsocial.py

@@ -0,0 +1,71 @@
+from urllib.parse import urlparse
+
+from playwright.async_api import async_playwright
+
+from kabigon.core.exception import InvalidURLError
+from kabigon.core.loader import Loader
+
+from .utils import html_to_markdown
+
+TRUTHSOCIAL_DOMAINS = [
+    "truthsocial.com",
+    "www.truthsocial.com",
+]
+
+USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+)
+
+
+def check_truthsocial_url(url: str) -> None:
+    """Check if URL is from Truth Social.
+
+    Args:
+        url: The URL to check
+
+    Raises:
+        ValueError: If URL is not from Truth Social
+    """
+    netloc = urlparse(url).netloc
+    if netloc not in TRUTHSOCIAL_DOMAINS:
+        raise InvalidURLError(url, "Truth Social")
+
+
+class TruthSocialLoader(Loader):
+    """Loader for Truth Social posts.
+
+    Truth Social requires JavaScript rendering and longer wait times
+    for content to fully load.
+    """
+
+    def __init__(self, timeout: float = 60_000) -> None:
+        """Initialize TruthSocialLoader.
+
+        Args:
+            timeout: Timeout in milliseconds for page loading (default: 60 seconds)
+        """
+        self.timeout = timeout
+
+    async def load(self, url: str) -> str:
+        """Load Truth Social content from URL.
+
+        Args:
+            url: Truth Social URL to load
+
+        Returns:
+            Loaded content as markdown
+
+        Raises:
+            ValueError: If URL is not from Truth Social
+        """
+        check_truthsocial_url(url)
+
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            context = await browser.new_context(user_agent=USER_AGENT)
+            page = await context.new_page()
+            await page.goto(url, timeout=self.timeout, wait_until="networkidle")
+            content = await page.content()
+            await browser.close()
+
+            return html_to_markdown(content)

kabigon/loaders/twitter.py

@@ -0,0 +1,115 @@
+import asyncio
+import contextlib
+from urllib.parse import urlparse
+from urllib.parse import urlunparse
+
+from loguru import logger
+from playwright.async_api import Error as PlaywrightError
+from playwright.async_api import Page
+from playwright.async_api import Request
+from playwright.async_api import Route
+from playwright.async_api import TimeoutError
+from playwright.async_api import async_playwright
+
+from kabigon.core.exception import InvalidURLError
+from kabigon.core.loader import Loader
+
+from .utils import html_to_markdown
+
+TWITTER_DOMAINS = [
+    "twitter.com",
+    "x.com",
+    "fxtwitter.com",
+    "vxtwitter.com",
+    "fixvx.com",
+    "twittpr.com",
+    "api.fxtwitter.com",
+    "fixupx.com",
+]
+
+USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+)
+
+TWEET_READY_SELECTORS = [
+    'article [data-testid="tweetText"]',
+    'article [data-testid="tweet"]',
+    '[data-testid="tweetText"]',
+]
+
+
+def replace_domain(url: str, new_domain: str = "x.com") -> str:
+    return str(urlunparse(urlparse(url)._replace(netloc=new_domain)))
+
+
+def check_x_url(url: str) -> None:
+    if urlparse(url).netloc not in TWITTER_DOMAINS:
+        raise InvalidURLError(url, "Twitter/X")
+
+
+class TwitterLoader(Loader):
+    def __init__(self, timeout: float = 20_000, wait_for_tweet_timeout: float = 15_000) -> None:
+        self.timeout = timeout
+        self.wait_for_tweet_timeout = wait_for_tweet_timeout
+
+    async def _wait_for_any_selector(self, page: Page, *, selectors: list[str], timeout_ms: float) -> None:
+        async def wait_one(selector: str) -> None:
+            await page.wait_for_selector(selector, state="visible", timeout=timeout_ms)
+
+        tasks = [asyncio.create_task(wait_one(selector)) for selector in selectors]
+        try:
+            done, pending = await asyncio.wait(
+                tasks,
+                return_when=asyncio.FIRST_COMPLETED,
+                timeout=timeout_ms / 1000,
+            )
+            for task in pending:
+                task.cancel()
+            for task in done:
+                task.result()
+        finally:
+            for task in tasks:
+                if not task.done():
+                    task.cancel()
+
+    async def load(self, url: str) -> str:
+        check_x_url(url)
+
+        url = replace_domain(url)
+
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            context = await browser.new_context(user_agent=USER_AGENT)
+            page = await context.new_page()
+
+            async def route_handler(route: Route, request: Request) -> None:
+                if request.resource_type in {"image", "media", "font"}:
+                    await route.abort()
+                    return
+                await route.continue_()
+
+            await page.route("**/*", route_handler)
+
+            try:
+                await page.goto(url, timeout=self.timeout, wait_until="domcontentloaded")
+            except TimeoutError as e:
+                logger.warning("TimeoutError: {}, (url: {}, timeout: {})", e, url, self.timeout)
+
+            with contextlib.suppress(TimeoutError):
+                await self._wait_for_any_selector(
+                    page,
+                    selectors=TWEET_READY_SELECTORS,
+                    timeout_ms=min(self.timeout or self.wait_for_tweet_timeout, self.wait_for_tweet_timeout),
+                )
+
+            try:
+                tweet_articles = page.locator("article").filter(has=page.locator('[data-testid="tweetText"]'))
+                if await tweet_articles.count() > 0:
+                    content = await tweet_articles.nth(0).evaluate("el => el.outerHTML")
+                else:
+                    content = await page.content()
+            except (PlaywrightError, TimeoutError):
+                content = await page.content()
+
+            await browser.close()
+            return html_to_markdown(content)

kabigon/loaders/utils.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+
+import charset_normalizer
+from markdownify import markdownify
+
+
+def normalize_whitespace(text: str) -> str:
+    lines = []
+    for line in text.splitlines():
+        stripped = line.strip()
+        if stripped:
+            lines += [stripped]
+    return "\n".join(lines)
+
+
+def html_to_markdown(content: str | bytes) -> str:
+    """Convert HTML content to markdown format.
+
+    Args:
+        content: HTML content as string or bytes
+
+    Returns:
+        Converted markdown text with normalized whitespace
+    """
+    if isinstance(content, bytes):
+        content = str(charset_normalizer.from_bytes(content).best())
+
+    md = markdownify(content, strip=["a", "img"])
+    return normalize_whitespace(md)
+
+
+def read_html_content(f: str | Path) -> str:
+    content = str(charset_normalizer.from_path(f).best())
+
+    md = markdownify(content, strip=["a", "img"])
+    return normalize_whitespace(md)

kabigon/loaders/youtube.py

@@ -0,0 +1,165 @@
+from urllib.parse import parse_qs
+from urllib.parse import urlparse
+
+from youtube_transcript_api import YouTubeTranscriptApi
+
+from kabigon.core.exception import KabigonError
+from kabigon.core.loader import Loader
+
+DEFAULT_LANGUAGES = [
+    # 中文
+    "zh-TW",
+    "zh-Hant",
+    "zh",
+    "zh-Hans",
+    # 日韓英
+    "ja",
+    "ko",
+    "en",
+    # 歐洲主要語言
+    "fr",  # French
+    "de",  # German
+    "es",  # Spanish
+    "it",  # Italian
+    "pt",  # Portuguese
+    "pt-BR",  # Portuguese (Brazil)
+    "nl",  # Dutch
+    "sv",  # Swedish
+    "pl",  # Polish
+    # 東南亞
+    "th",  # Thai
+    "vi",  # Vietnamese
+    "id",  # Indonesian
+    "ms",  # Malay
+    "fil",  # Filipino / Tagalog
+    # 其他常見
+    "ru",  # Russian
+    "ar",  # Arabic
+    "hi",  # Hindi
+]
+ALLOWED_SCHEMES = {
+    "http",
+    "https",
+}
+ALLOWED_NETLOCS = {
+    "youtu.be",
+    "m.youtube.com",
+    "youtube.com",
+    "www.youtube.com",
+    "www.youtube-nocookie.com",
+    "vid.plus",
+}
+
+
+class UnsupportedURLSchemeError(KabigonError):
+    def __init__(self, scheme: str) -> None:
+        super().__init__(f"unsupported URL scheme: {scheme}")
+
+
+class UnsupportedURLNetlocError(KabigonError):
+    def __init__(self, netloc: str) -> None:
+        super().__init__(f"unsupported URL netloc: {netloc}")
+
+
+class VideoIDError(KabigonError):
+    def __init__(self, video_id: str) -> None:
+        super().__init__(f"invalid video ID: {video_id}")
+
+
+class NoVideoIDFoundError(KabigonError):
+    def __init__(self, url: str) -> None:
+        super().__init__(f"no video found in URL: {url}")
+
+
+def parse_video_id(url: str) -> str:
+    """Parse and extract the video ID from a YouTube URL.
+
+    Supports various YouTube URL formats including:
+    - https://www.youtube.com/watch?v=VIDEO_ID
+    - https://youtu.be/VIDEO_ID
+    - https://m.youtube.com/watch?v=VIDEO_ID
+    - https://www.youtube-nocookie.com/watch?v=VIDEO_ID
+    - https://vid.plus/VIDEO_ID
+
+    Args:
+        url: YouTube video URL.
+
+    Returns:
+        11-character video ID.
+
+    Raises:
+        UnsupportedURLSchemeError: If URL scheme is not http or https.
+        UnsupportedURLNetlocError: If URL domain is not a supported YouTube domain.
+        NoVideoIDFoundError: If no video ID parameter found in the URL.
+        VideoIDError: If extracted video ID is not exactly 11 characters.
+
+    Example:
+        >>> parse_video_id("https://www.youtube.com/watch?v=dQw4w9WgXcQ")
+        'dQw4w9WgXcQ'
+        >>> parse_video_id("https://youtu.be/dQw4w9WgXcQ")
+        'dQw4w9WgXcQ'
+    """
+    parsed_url = urlparse(url)
+
+    if parsed_url.scheme not in ALLOWED_SCHEMES:
+        raise UnsupportedURLSchemeError(parsed_url.scheme)
+
+    if parsed_url.netloc not in ALLOWED_NETLOCS:
+        raise UnsupportedURLNetlocError(parsed_url.netloc)
+
+    path = parsed_url.path
+
+    if path.endswith("/watch"):
+        query = parsed_url.query
+        parsed_query = parse_qs(query)
+        if "v" in parsed_query:
+            ids = parsed_query["v"]
+            video_id = ids[0]
+        else:
+            raise NoVideoIDFoundError(url)
+    else:
+        stripped_path = parsed_url.path.lstrip("/")
+        video_id = stripped_path.split("/")[-1]
+
+    if len(video_id) != 11:  # Video IDs are 11 characters long
+        raise VideoIDError(video_id)
+
+    return video_id
+
+
+def check_youtube_url(url: str) -> None:
+    """Validate that the given URL is a supported YouTube URL.
+
+    This delegates to ``parse_video_id`` to ensure that URL validation
+    (including scheme and netloc checks) is implemented in a single place.
+    Any validation failures are surfaced as ``ValueError`` to maintain
+    the previous public interface of this function.
+
+    Args:
+        url: YouTube video URL to validate.
+
+    Raises:
+        ValueError: If URL is invalid or not a supported YouTube URL.
+    """
+    try:
+        # We only care about validation here; the caller does not need the ID.
+        parse_video_id(url)
+    except (UnsupportedURLSchemeError, UnsupportedURLNetlocError, NoVideoIDFoundError, VideoIDError) as exc:
+        raise ValueError(str(exc)) from exc
+
+
+class YoutubeLoader(Loader):
+    def __init__(self, languages: list[str] | None = None) -> None:
+        self.languages = languages or DEFAULT_LANGUAGES
+
+    def load_sync(self, url: str) -> str:
+        video_id = parse_video_id(url)
+
+        fetched = YouTubeTranscriptApi().fetch(video_id, self.languages)
+
+        lines = []
+        for snippet in fetched.snippets:
+            text = str(snippet.text).strip()
+            if text:
+                lines.append(text)
+        return "\n".join(lines)

kabigon/loaders/youtube_ytdlp.py

@@ -0,0 +1,13 @@
+from kabigon.core.loader import Loader
+
+from .youtube import check_youtube_url
+from .ytdlp import YtdlpLoader
+
+
+class YoutubeYtdlpLoader(Loader):
+    def __init__(self) -> None:
+        self.ytdlp_loader = YtdlpLoader()
+
+    def load_sync(self, url: str) -> str:
+        check_youtube_url(url)
+        return self.ytdlp_loader.load_sync(url)

kabigon/loaders/ytdlp.py

@@ -0,0 +1,60 @@
+import os
+import uuid
+from pathlib import Path
+
+import yt_dlp
+from loguru import logger
+
+from kabigon.core.exception import WhisperNotInstalledError
+from kabigon.core.loader import Loader
+
+
+def download_audio(url: str, outtmpl: str | None = None) -> None:
+    ydl_opts = {
+        "format": "bestaudio/best",
+        "postprocessors": [
+            {
+                "key": "FFmpegExtractAudio",
+                "preferredcodec": "mp3",
+                "preferredquality": "192",
+            }
+        ],
+        "match_filter": yt_dlp.match_filter_func(["!is_live"]),
+    }
+
+    if outtmpl is not None:
+        ydl_opts["outtmpl"] = outtmpl
+
+    ffmpeg_path = os.getenv("FFMPEG_PATH")
+    if ffmpeg_path is not None:
+        ydl_opts["ffmpeg_location"] = ffmpeg_path
+
+    logger.info("Downloading audio from URL: {} with options: {}", url, ydl_opts)
+    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+        ydl.download([url])
+
+
+class YtdlpLoader(Loader):
+    def __init__(self, model: str = "tiny") -> None:
+        try:
+            import whisper
+        except ImportError as e:
+            raise WhisperNotInstalledError from e
+
+        self.model = whisper.load_model(model)
+        self.load_audio = whisper.load_audio
+
+    def load_sync(self, url: str) -> str:
+        outtmpl = uuid.uuid4().hex[:20]
+        path = str(Path(outtmpl).with_suffix(".mp3"))
+        download_audio(url, outtmpl=outtmpl)
+
+        try:
+            audio = self.load_audio(path)
+            logger.info("Transcribing audio file: {}", path)
+            result = self.model.transcribe(audio)
+        finally:
+            # Clean up the audio file
+            Path(path).unlink()
+
+        return result.get("text", "")