kabigon 0.14.2__py3-none-any.whl

kabigon/__init__.py

@@ -0,0 +1,45 @@
+import os
+import sys
+from typing import Final
+
+from loguru import logger
+
+from .api import load_url
+from .api import load_url_sync
+from .loaders import Compose
+from .loaders import FirecrawlLoader
+from .loaders import GitHubLoader
+from .loaders import HttpxLoader
+from .loaders import PDFLoader
+from .loaders import PlaywrightLoader
+from .loaders import PttLoader
+from .loaders import RedditLoader
+from .loaders import ReelLoader
+from .loaders import TruthSocialLoader
+from .loaders import TwitterLoader
+from .loaders import YoutubeLoader
+from .loaders import YoutubeYtdlpLoader
+from .loaders import YtdlpLoader
+
+__all__ = [
+    "Compose",
+    "FirecrawlLoader",
+    "GitHubLoader",
+    "HttpxLoader",
+    "PDFLoader",
+    "PlaywrightLoader",
+    "PttLoader",
+    "RedditLoader",
+    "ReelLoader",
+    "TruthSocialLoader",
+    "TwitterLoader",
+    "YoutubeLoader",
+    "YoutubeYtdlpLoader",
+    "YtdlpLoader",
+    "load_url",
+    "load_url_sync",
+]
+
+LOGURU_LEVEL: Final[str] = os.getenv("LOGURU_LEVEL", "INFO")
+logger.remove()
+logger.add(sys.stderr, level=LOGURU_LEVEL)

kabigon/api.py

@@ -0,0 +1,74 @@
+from . import loaders
+
+
+def _get_default_loader() -> loaders.Compose:
+    """Get the default loader composition used by CLI.
+
+    Returns:
+        Compose: Default loader chain with all available loaders
+    """
+    return loaders.Compose(
+        [
+            loaders.PttLoader(),
+            loaders.TwitterLoader(),
+            loaders.TruthSocialLoader(),
+            loaders.RedditLoader(),
+            loaders.YoutubeLoader(),
+            loaders.ReelLoader(),
+            loaders.YoutubeYtdlpLoader(),
+            loaders.PDFLoader(),
+            loaders.GitHubLoader(),
+            loaders.PlaywrightLoader(timeout=50_000, wait_until="networkidle"),
+            loaders.PlaywrightLoader(timeout=10_000),
+        ]
+    )
+
+
+def load_url_sync(url: str) -> str:
+    """Load content from a URL using the default loader chain.
+
+    This is a convenience function that uses the same loader chain as the CLI.
+    It tries each loader in sequence until one succeeds.
+
+    Args:
+        url: The URL to load content from
+
+    Returns:
+        str: Extracted content as markdown
+
+    Raises:
+        Exception: If all loaders fail to load the URL
+
+    Example:
+        >>> import kabigon
+        >>> text = kabigon.load_url_sync("https://example.com")
+        >>> print(text)
+    """
+    loader = _get_default_loader()
+    return loader.load_sync(url)
+
+
+async def load_url(url: str) -> str:
+    """Asynchronously load content from a URL using the default loader chain.
+
+    This is an async version of load_url() that can be used in async contexts.
+
+    Args:
+        url: The URL to load content from
+
+    Returns:
+        str: Extracted content as markdown
+
+    Raises:
+        Exception: If all loaders fail to load the URL
+
+    Example:
+        >>> import asyncio
+        >>> import kabigon
+        >>> async def main():
+        ...     text = await kabigon.load_url("https://example.com")
+        ...     print(text)
+        >>> asyncio.run(main())
+    """
+    loader = _get_default_loader()
+    return await loader.load(url)

kabigon/cli.py

@@ -0,0 +1,13 @@
+import typer
+from rich import print
+
+from .api import load_url_sync
+
+
+def run(url: str) -> None:
+    result = load_url_sync(url)
+    print(result)
+
+
+def main() -> None:
+    typer.run(run)

kabigon/core/__init__.py

@@ -0,0 +1,19 @@
+from .exception import ConfigurationError
+from .exception import FirecrawlAPIKeyNotSetError
+from .exception import InvalidURLError
+from .exception import KabigonError
+from .exception import LoaderError
+from .exception import MissingDependencyError
+from .exception import WhisperNotInstalledError
+from .loader import Loader
+
+__all__ = [
+    "ConfigurationError",
+    "FirecrawlAPIKeyNotSetError",
+    "InvalidURLError",
+    "KabigonError",
+    "Loader",
+    "LoaderError",
+    "MissingDependencyError",
+    "WhisperNotInstalledError",
+]

kabigon/core/exception.py

@@ -0,0 +1,41 @@
+class KabigonError(Exception):
+    """Base exception for all Kabigon errors."""
+
+
+class LoaderError(KabigonError):
+    """Raised when all loaders fail to load a URL."""
+
+    def __init__(self, url: str) -> None:
+        self.url = url
+        super().__init__(f"Failed to load URL: {url}")
+
+
+class InvalidURLError(KabigonError, ValueError):
+    """Raised when a URL is not valid for a specific loader."""
+
+    def __init__(self, url: str, expected: str) -> None:
+        self.url = url
+        self.expected = expected
+        super().__init__(f"URL is not a {expected} URL: {url}")
+
+
+class ConfigurationError(KabigonError):
+    """Raised when required configuration is missing."""
+
+
+class FirecrawlAPIKeyNotSetError(ConfigurationError):
+    """Raised when FIRECRAWL_API_KEY environment variable is not set."""
+
+    def __init__(self) -> None:
+        super().__init__("FIRECRAWL_API_KEY is not set.")
+
+
+class MissingDependencyError(KabigonError):
+    """Raised when a required dependency is not installed."""
+
+
+class WhisperNotInstalledError(MissingDependencyError):
+    """Raised when OpenAI Whisper is not installed."""
+
+    def __init__(self) -> None:
+        super().__init__("OpenAI Whisper not installed. Please install it with `pip install openai-whisper`.")

kabigon/core/loader.py

@@ -0,0 +1,9 @@
+import asyncio
+
+
+class Loader:
+    def load_sync(self, url: str) -> str:
+        return asyncio.run(self.load(url))
+
+    async def load(self, url: str) -> str:
+        return await asyncio.to_thread(self.load_sync, url)

kabigon/loaders/__init__.py

@@ -0,0 +1,31 @@
+from .compose import Compose
+from .firecrawl import FirecrawlLoader
+from .github import GitHubLoader
+from .httpx import HttpxLoader
+from .pdf import PDFLoader
+from .playwright import PlaywrightLoader
+from .ptt import PttLoader
+from .reddit import RedditLoader
+from .reel import ReelLoader
+from .truthsocial import TruthSocialLoader
+from .twitter import TwitterLoader
+from .youtube import YoutubeLoader
+from .youtube_ytdlp import YoutubeYtdlpLoader
+from .ytdlp import YtdlpLoader
+
+__all__ = [
+    "Compose",
+    "FirecrawlLoader",
+    "GitHubLoader",
+    "HttpxLoader",
+    "PDFLoader",
+    "PlaywrightLoader",
+    "PttLoader",
+    "RedditLoader",
+    "ReelLoader",
+    "TruthSocialLoader",
+    "TwitterLoader",
+    "YoutubeLoader",
+    "YoutubeYtdlpLoader",
+    "YtdlpLoader",
+]

kabigon/loaders/compose.py

@@ -0,0 +1,26 @@
+from loguru import logger
+
+from kabigon.core.exception import LoaderError
+from kabigon.core.loader import Loader
+
+
+class Compose(Loader):
+    def __init__(self, loaders: list[Loader]) -> None:
+        self.loaders = loaders
+
+    async def load(self, url: str) -> str:
+        for loader in self.loaders:
+            try:
+                result = await loader.load(url)
+            except Exception as e:  # noqa: BLE001
+                # We intentionally catch all exceptions to try the next loader in the chain
+                logger.info("[{}] Failed to load URL: {}, got error: {}", loader.__class__.__name__, url, e)
+            else:
+                if not result:
+                    logger.info("[{}] Failed to load URL: {}, got empty result", loader.__class__.__name__, url)
+                    continue
+
+                logger.info("[{}] Successfully loaded URL: {}", loader.__class__.__name__, url)
+                return result
+
+        raise LoaderError(url)

kabigon/loaders/firecrawl.py

@@ -0,0 +1,30 @@
+import os
+
+from firecrawl import FirecrawlApp
+
+from kabigon.core.exception import FirecrawlAPIKeyNotSetError
+from kabigon.core.exception import LoaderError
+from kabigon.core.loader import Loader
+
+
+class FirecrawlLoader(Loader):
+    def __init__(self, timeout: int | None = None) -> None:
+        self.timeout = timeout
+
+        api_key = os.getenv("FIRECRAWL_API_KEY")
+        if not api_key:
+            raise FirecrawlAPIKeyNotSetError
+
+        self.app = FirecrawlApp(api_key=api_key)
+
+    def load_sync(self, url: str) -> str:
+        result = self.app.scrape_url(  # ty:ignore[possibly-missing-attribute]
+            url,
+            formats=["markdown"],
+            timeout=self.timeout,
+        )
+
+        if not result.success:
+            raise LoaderError(url)
+
+        return result.markdown

kabigon/loaders/github.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+from html.parser import HTMLParser
+from urllib.parse import urlparse
+
+import httpx
+
+from kabigon.core.exception import InvalidURLError
+from kabigon.core.loader import Loader
+
+from .utils import html_to_markdown
+
+GITHUB_HOST = "github.com"
+RAW_GITHUB_HOST = "raw.githubusercontent.com"
+
+_VOID_TAGS = {
+    "area",
+    "base",
+    "br",
+    "col",
+    "embed",
+    "hr",
+    "img",
+    "input",
+    "link",
+    "meta",
+    "param",
+    "source",
+    "track",
+    "wbr",
+}
+
+_IGNORED_TAGS = {
+    "script",
+    "style",
+    "noscript",
+    "svg",
+    "nav",
+    "header",
+    "footer",
+}
+
+
+def check_github_url(url: str) -> None:
+    host = urlparse(url).netloc
+    if host not in {GITHUB_HOST, RAW_GITHUB_HOST}:
+        raise InvalidURLError(url, "GitHub")
+
+
+def to_raw_github_url(url: str) -> str:
+    """Convert a GitHub blob URL to a raw.githubusercontent.com URL.
+
+    Supports:
+      - https://github.com/<owner>/<repo>/blob/<ref>/<path>
+      - https://raw.githubusercontent.com/<owner>/<repo>/<ref>/<path>
+    """
+    parsed = urlparse(url)
+    if parsed.netloc == RAW_GITHUB_HOST:
+        return url
+
+    if parsed.netloc != GITHUB_HOST:
+        raise InvalidURLError(url, "GitHub")
+
+    parts = [p for p in parsed.path.split("/") if p]
+    if len(parts) < 5 or parts[2] != "blob":
+        raise InvalidURLError(url, "GitHub blob")
+
+    owner, repo, _, ref = parts[:4]
+    path = "/".join(parts[4:])
+    if not path:
+        raise InvalidURLError(url, "GitHub blob file")
+
+    return f"https://{RAW_GITHUB_HOST}/{owner}/{repo}/{ref}/{path}"
+
+
+class _SubtreeHTMLExtractor(HTMLParser):
+    def __init__(self, root_tag: str) -> None:
+        super().__init__(convert_charrefs=True)
+        self.root_tag = root_tag
+        self._capturing = False
+        self._depth = 0
+        self._ignored_depth = 0
+        self._out: list[str] = []
+
+    def get_html(self) -> str:
+        return "".join(self._out).strip()
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag == self.root_tag and not self._capturing:
+            self._capturing = True
+            self._depth = 1
+            self._out.append(self.get_starttag_text() or f"<{tag}>")
+            return
+
+        if not self._capturing:
+            return
+
+        if tag in _IGNORED_TAGS:
+            self._ignored_depth += 1
+            return
+
+        self._out.append(self.get_starttag_text() or f"<{tag}>")
+        if tag not in _VOID_TAGS:
+            self._depth += 1
+
+    def handle_endtag(self, tag: str) -> None:
+        if not self._capturing:
+            return
+
+        if self._ignored_depth:
+            if tag in _IGNORED_TAGS:
+                self._ignored_depth -= 1
+            return
+
+        self._out.append(f"</{tag}>")
+        if tag not in _VOID_TAGS:
+            self._depth -= 1
+
+        if self._depth <= 0:
+            self._capturing = False
+
+    def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if not self._capturing:
+            return
+        if self._ignored_depth or tag in _IGNORED_TAGS:
+            return
+        self._out.append(self.get_starttag_text() or f"<{tag} />")
+
+    def handle_data(self, data: str) -> None:
+        if not self._capturing or self._ignored_depth:
+            return
+        self._out.append(data)
+
+
+def extract_main_html(html: str) -> str:
+    """Extract GitHub's primary content area without site-specific selectors."""
+    for tag in ("main", "article"):
+        parser = _SubtreeHTMLExtractor(tag)
+        parser.feed(html)
+        extracted = parser.get_html()
+        if extracted:
+            return extracted
+    return html
+
+
+class GitHubLoader(Loader):
+    async def load(self, url: str) -> str:
+        check_github_url(url)
+        parsed = urlparse(url)
+
+        if parsed.netloc == RAW_GITHUB_HOST or "/blob/" in parsed.path:
+            raw_url = to_raw_github_url(url)
+
+            async with httpx.AsyncClient() as client:
+                response = await client.get(
+                    raw_url,
+                    follow_redirects=True,
+                    headers={"Accept": "text/plain, text/markdown;q=0.9, */*;q=0.1"},
+                )
+                response.raise_for_status()
+
+            content_type = response.headers.get("content-type", "")
+            if "text" not in content_type and "json" not in content_type and "xml" not in content_type:
+                raise InvalidURLError(url, f"GitHub text content-type (got {content_type!r})")
+
+            return response.text
+
+        async with httpx.AsyncClient() as client:
+            response = await client.get(
+                url,
+                follow_redirects=True,
+                headers={
+                    "Accept": "text/html,application/xhtml+xml",
+                    "User-Agent": "kabigon (httpx)",
+                },
+            )
+            response.raise_for_status()
+
+        content_type = response.headers.get("content-type", "")
+        if "html" not in content_type:
+            raise InvalidURLError(url, f"GitHub HTML content-type (got {content_type!r})")
+
+        main_html = extract_main_html(response.text)
+        return html_to_markdown(main_html)

kabigon/loaders/httpx.py

@@ -0,0 +1,16 @@
+import httpx
+
+from kabigon.core.loader import Loader
+
+from .utils import html_to_markdown
+
+
+class HttpxLoader(Loader):
+    def __init__(self, headers: dict[str, str] | None = None) -> None:
+        self.headers = headers
+
+    async def load(self, url: str) -> str:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(url, headers=self.headers, follow_redirects=True)
+            response.raise_for_status()
+            return html_to_markdown(response.content)

kabigon/loaders/pdf.py

@@ -0,0 +1,46 @@
+import io
+from pathlib import Path
+from typing import IO
+from typing import Any
+
+import httpx
+from pypdf import PdfReader
+
+from kabigon.core.loader import Loader
+
+DEFAULT_HEADERS = {
+    "Accept-Language": "zh-TW,zh;q=0.9,ja;q=0.8,en-US;q=0.7,en;q=0.6",
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",  # noqa
+}
+
+
+class NotPDFError(Exception):
+    def __init__(self, url: str) -> None:
+        super().__init__(f"URL is not a PDF: {url}")
+
+
+class PDFLoader(Loader):
+    async def load(self, url_or_file: str) -> str:  # ty:ignore[invalid-method-override]
+        if not url_or_file.startswith("http"):
+            return read_pdf_content(url_or_file)
+
+        async with httpx.AsyncClient() as client:
+            resp = await client.get(url_or_file, headers=DEFAULT_HEADERS, follow_redirects=True)
+            resp.raise_for_status()
+
+            if resp.headers.get("content-type") != "application/pdf":
+                raise NotPDFError(url_or_file)
+
+            return read_pdf_content(io.BytesIO(resp.content))
+
+
+def read_pdf_content(f: str | Path | IO[Any]) -> str:
+    lines = []
+    with PdfReader(f) as reader:
+        for page in reader.pages:
+            text = page.extract_text(extraction_mode="plain")
+            for line in text.splitlines():
+                stripped = line.strip()
+                if stripped:
+                    lines.append(stripped)
+    return "\n".join(lines)

kabigon/loaders/playwright.py

@@ -0,0 +1,37 @@
+from typing import Literal
+
+from loguru import logger
+from playwright.async_api import TimeoutError
+from playwright.async_api import async_playwright
+
+from kabigon.core.loader import Loader
+
+from .utils import html_to_markdown
+
+
+class PlaywrightLoader(Loader):
+    def __init__(
+        self,
+        timeout: float | None = 0,
+        wait_until: Literal["commit", "domcontentloaded", "load", "networkidle"] | None = None,
+        browser_headless: bool = False,
+    ) -> None:
+        self.timeout = timeout
+        self.wait_until = wait_until
+        self.browser_headless = browser_headless
+
+    async def load(self, url: str) -> str:
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=self.browser_headless)
+            context = await browser.new_context()
+            page = await context.new_page()
+
+            try:
+                await page.goto(url, timeout=self.timeout, wait_until=self.wait_until)
+            except TimeoutError as e:
+                logger.warning("TimeoutError: {}, (url: {}, timeout: {})", e, url, self.timeout)
+
+            content = await page.content()
+            await browser.close()
+
+            return html_to_markdown(content)

kabigon/loaders/ptt.py

@@ -0,0 +1,27 @@
+from urllib.parse import urlparse
+
+from kabigon.core.exception import InvalidURLError
+from kabigon.core.loader import Loader
+
+from .httpx import HttpxLoader
+
+
+def check_ptt_url(url: str) -> None:
+    if urlparse(url).netloc != "www.ptt.cc":
+        raise InvalidURLError(url, "PTT")
+
+
+class PttLoader(Loader):
+    def __init__(self) -> None:
+        self.httpx_loader = HttpxLoader(
+            headers={
+                "Accept-Language": "zh-TW,zh;q=0.9,ja;q=0.8,en-US;q=0.7,en;q=0.6",
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",  # noqa
+                "Cookie": "over18=1",
+            }
+        )
+
+    async def load(self, url: str) -> str:
+        check_ptt_url(url)
+
+        return await self.httpx_loader.load(url)