kabigon 0.1.0__py3-none-any.whl

kabigon/__init__.py

@@ -0,0 +1,19 @@
+import os
+import sys
+from typing import Final
+
+from loguru import logger
+
+from .cloudscraper import CloudscraperLoader
+from .httpx import HttpxLoader
+from .loader import Loader
+from .pdf import PDFLoader
+from .pipeline import PipelineLoader
+from .playwright import PlaywrightLoader
+from .reel import ReelLoader
+from .singlefile import SinglefileLoader
+from .youtube import YoutubeLoader
+from .ytdlp import YtdlpLoader
+
+LOGURU_LEVEL: Final[str] = os.getenv("LOGURU_LEVEL", "INFO")
+logger.configure(handlers=[{"sink": sys.stderr, "level": LOGURU_LEVEL}])

kabigon/cli.py

@@ -0,0 +1,11 @@
+import click
+from rich import print
+
+from .pipeline import PipelineLoader
+
+
+@click.command()
+@click.argument("url", type=click.STRING)
+def main(url: str) -> None:
+    loader = PipelineLoader().load(url)
+    print(loader)

kabigon/cloudscraper.py

@@ -0,0 +1,14 @@
+import cloudscraper
+import timeout_decorator
+
+from .loader import Loader
+from .utils import html_to_markdown
+
+
+class CloudscraperLoader(Loader):
+    @timeout_decorator.timeout(5)
+    def load(self, url: str) -> str:
+        client = cloudscraper.create_scraper()
+        response = client.get(url, allow_redirects=True)
+        response.raise_for_status()
+        return html_to_markdown(response.text)

kabigon/httpx.py

@@ -0,0 +1,19 @@
+import httpx
+import timeout_decorator
+
+from .loader import Loader
+from .utils import html_to_markdown
+
+DEFAULT_HEADERS = {
+    "Accept-Language": "zh-TW,zh;q=0.9,ja;q=0.8,en-US;q=0.7,en;q=0.6",
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",  # noqa
+    "Cookie": "over18=1",  # Required for some sites like PTT
+}
+
+
+class HttpxLoader(Loader):
+    @timeout_decorator.timeout(5)
+    def load(self, url: str) -> str:
+        response = httpx.get(url, headers=DEFAULT_HEADERS, follow_redirects=True)
+        response.raise_for_status()
+        return html_to_markdown(response.content)

kabigon/loader.py

@@ -0,0 +1,7 @@
+class Loader:
+    def load(self, url: str) -> str:
+        raise NotImplementedError
+
+
+class LoaderError(Exception):
+    pass

kabigon/pdf.py

@@ -0,0 +1,52 @@
+import tempfile
+from pathlib import Path
+
+import httpx
+import timeout_decorator
+from pypdf import PdfReader
+
+from .loader import Loader
+from .loader import LoaderError
+
+DEFAULT_HEADERS = {
+    "Accept-Language": "zh-TW,zh;q=0.9,ja;q=0.8,en-US;q=0.7,en;q=0.6",
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",  # noqa
+}
+
+
+class NotPDFError(LoaderError):
+    pass
+
+
+class PDFLoader(Loader):
+    @timeout_decorator.timeout(5)
+    def load(self, url_or_file: str) -> str:
+        if url_or_file.startswith("http"):
+            url_or_file = download_pdf_from_url(url_or_file)
+        return read_pdf_content(url_or_file)
+
+
+def download_pdf_from_url(url: str) -> str:
+    response = httpx.get(url=url, headers=DEFAULT_HEADERS, follow_redirects=True)
+    response.raise_for_status()
+
+    is_pdf = response.headers.get("content-type") == "application/pdf"
+    if not is_pdf:
+        raise NotPDFError(f"URL is not a PDF: {url}")
+
+    suffix = ".pdf" if is_pdf else None
+    with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as fp:
+        fp.write(response.content)
+        return fp.name
+
+
+def read_pdf_content(f: str | Path) -> str:
+    lines = []
+    with PdfReader(f) as reader:
+        for page in reader.pages:
+            text = page.extract_text(extraction_mode="plain")
+            for line in text.splitlines():
+                if not line.strip():
+                    continue
+                lines.append(line.strip())
+    return "\n".join(lines)

kabigon/pipeline.py

@@ -0,0 +1,67 @@
+from urllib.parse import urlparse
+from urllib.parse import urlunparse
+
+from loguru import logger
+
+from .cloudscraper import CloudscraperLoader
+from .httpx import HttpxLoader
+from .loader import Loader
+from .loader import LoaderError
+from .pdf import PDFLoader
+from .reel import ReelLoader
+from .singlefile import SinglefileLoader
+from .youtube import YoutubeLoader
+from .ytdlp import YtdlpLoader
+
+REPLACEMENTS = {
+    "api.fxtwitter.com": [
+        "twitter.com",
+        "x.com",
+        "fxtwitter.com",
+        "vxtwitter.com",
+        "fixvx.com",
+        "twittpr.com",
+        "fixupx.com",
+    ]
+}
+
+
+def replace_domain(url: str) -> str:
+    parsed = urlparse(url)
+    for target, source in REPLACEMENTS.items():
+        if parsed.netloc in source:
+            fixed_url = parsed._replace(netloc=target)
+            return urlunparse(fixed_url)
+    return url
+
+
+class PipelineLoader(Loader):
+    def __init__(self) -> None:
+        self.loaders: list[Loader] = [
+            YoutubeLoader(),
+            ReelLoader(),
+            YtdlpLoader(),
+            PDFLoader(),
+            CloudscraperLoader(),
+            HttpxLoader(),
+            SinglefileLoader(),
+        ]
+
+    def load(self, url: str) -> str:
+        url = replace_domain(url)
+
+        for loader in self.loaders:
+            try:
+                loaded_content = loader.load(url)
+
+                if not loaded_content:
+                    logger.info("[{}] Failed to load URL: {}, got empty result", loader.__class__.__name__, url)
+                    continue
+
+                logger.info("[{}] Successfully loaded URL: {}", loader.__class__.__name__, url)
+                return loaded_content
+
+            except Exception as e:
+                logger.info("[{}] Failed to load URL: {}, got error: {}", loader.__class__.__name__, url, e)
+
+        raise LoaderError(f"Failed to load URL: {url}")

kabigon/playwright.py

@@ -0,0 +1,38 @@
+from typing import Literal
+
+import timeout_decorator
+from loguru import logger
+from playwright.sync_api import TimeoutError
+from playwright.sync_api import sync_playwright
+
+from .loader import Loader
+from .utils import html_to_markdown
+
+
+class PlaywrightLoader(Loader):
+    def __init__(
+        self,
+        timeout: int = 10_000,
+        wait_until: Literal["commit", "domcontentloaded", "load", "networkidle"] = "networkidle",
+        browser_headless: bool = False,
+    ) -> None:
+        self.timeout = timeout
+        self.wait_until = wait_until
+        self.browser_headless = browser_headless
+
+    @timeout_decorator.timeout(5)
+    def load(self, url: str) -> str:
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=self.browser_headless)
+            page = browser.new_page()
+
+            try:
+                page.goto(url, timeout=self.timeout, wait_until=self.wait_until)
+            except TimeoutError as e:
+                logger.error("TimeoutError: {}", e)
+                page.goto(url)
+
+            content = page.content()
+            browser.close()
+
+            return html_to_markdown(content)

kabigon/py.typed

@@ -0,0 +1 @@
+

kabigon/reel.py

@@ -0,0 +1,31 @@
+import timeout_decorator
+
+from .httpx import HttpxLoader
+from .loader import Loader
+from .loader import LoaderError
+from .ytdlp import YtdlpLoader
+
+
+def is_reel_url(url: str) -> bool:
+    return url.startswith("https://www.instagram.com/reel")
+
+
+class NotReelURLError(LoaderError):
+    def __init__(self, url: str):
+        super().__init__(f"URL is not an Instagram Reel: {url}")
+
+
+class ReelLoader(Loader):
+    def __init__(self) -> None:
+        self.httpx_loader = HttpxLoader()
+        self.ytdlp_loader = YtdlpLoader()
+
+    @timeout_decorator.timeout(300)
+    def load(self, url: str) -> str:
+        if not is_reel_url(url):
+            raise NotReelURLError(url)
+
+        audio_content = self.ytdlp_loader.load(url)
+        html_content = self.httpx_loader.load(url)
+
+        return f"{audio_content}\n\n{html_content}"

kabigon/singlefile.py

@@ -0,0 +1,67 @@
+import os
+import subprocess
+import tempfile
+from functools import cache
+from pathlib import Path
+from typing import Final
+
+import charset_normalizer
+import timeout_decorator
+from loguru import logger
+
+from .loader import Loader
+from .utils import html_to_markdown
+
+DEFAULT_SINGLEFILE_PATH: Final[str] = "single-file"
+
+
+@cache
+def get_singlefile_path() -> str:
+    path = os.getenv("SINGLEFILE_PATH")
+    if not path:
+        path = DEFAULT_SINGLEFILE_PATH
+        logger.warning("SINGLEFILE_PATH not set, using default: {}", DEFAULT_SINGLEFILE_PATH)
+    return path
+
+
+class SinglefileLoader(Loader):
+    def __init__(self, cookies_file: str | None = None, browser_headless: bool = False) -> None:
+        self.cookies_file = cookies_file
+        self.browser_headless = browser_headless
+
+    @timeout_decorator.timeout(20)
+    def load(self, url: str) -> str:
+        filename = self.download(url)
+        content = str(charset_normalizer.from_path(filename).best())
+        return html_to_markdown(content)
+
+    def download(self, url: str) -> str:
+        logger.info("Downloading HTML using SingleFile: {}", url)
+
+        filename = tempfile.mktemp(suffix=".html")
+        singlefile_path = get_singlefile_path()
+
+        cmds = [singlefile_path]
+
+        if self.cookies_file is not None:
+            cookies_path = Path(self.cookies_file)
+            if not cookies_path.exists():
+                raise FileNotFoundError(f"Cookies file not found: {self.cookies_file}")
+
+            cmds += [
+                "--browser-cookies-file",
+                str(cookies_path),
+            ]
+
+        cmds += [
+            "--filename-conflict-action",
+            "overwrite",
+            "--browser-headless",
+            str(self.browser_headless).lower(),
+            url,
+            filename,
+        ]
+
+        subprocess.run(cmds)
+
+        return filename

kabigon/utils.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+
+import charset_normalizer
+from markdownify import markdownify
+
+
+def normalize_whitespace(text: str) -> str:
+    lines = []
+    for line in text.splitlines():
+        stripped = line.strip()
+        if stripped:
+            lines += [stripped]
+    return "\n".join(lines)
+
+
+def html_to_markdown(content: str | bytes) -> str:
+    """Convert HTML content to markdown format.
+
+    Args:
+        content: HTML content as string or bytes
+
+    Returns:
+        Converted markdown text with normalized whitespace
+    """
+    if isinstance(content, bytes):
+        content = str(charset_normalizer.from_bytes(content).best())
+
+    md = markdownify(content, strip=["a", "img"])
+    return normalize_whitespace(md)
+
+
+def read_html_content(f: str | Path) -> str:
+    content = str(charset_normalizer.from_path(f).best())
+
+    md = markdownify(content, strip=["a", "img"])
+    return normalize_whitespace(md)

kabigon/youtube.py

@@ -0,0 +1,92 @@
+from urllib.parse import parse_qs
+from urllib.parse import urlparse
+
+import timeout_decorator
+from youtube_transcript_api import YouTubeTranscriptApi
+
+from .loader import Loader
+from .loader import LoaderError
+
+DEFAULT_LANGUAGES = ["zh-TW", "zh-Hant", "zh", "zh-Hans", "ja", "en", "ko"]
+ALLOWED_SCHEMES = {
+    "http",
+    "https",
+}
+ALLOWED_NETLOCS = {
+    "youtu.be",
+    "m.youtube.com",
+    "youtube.com",
+    "www.youtube.com",
+    "www.youtube-nocookie.com",
+    "vid.plus",
+}
+
+
+class UnsupportedURLSchemeError(LoaderError):
+    def __init__(self, scheme: str) -> None:
+        super().__init__(f"unsupported URL scheme: {scheme}")
+
+
+class UnsupportedURLNetlocError(LoaderError):
+    def __init__(self, netloc: str) -> None:
+        super().__init__(f"unsupported URL netloc: {netloc}")
+
+
+class VideoIDError(LoaderError):
+    def __init__(self, video_id: str) -> None:
+        super().__init__(f"invalid video ID: {video_id}")
+
+
+class NoVideoIDFoundError(LoaderError):
+    def __init__(self, url: str) -> None:
+        super().__init__(f"no video found in URL: {url}")
+
+
+def parse_video_id(url: str) -> str:
+    """Parse a YouTube URL and return the video ID if valid, otherwise None."""
+    parsed_url = urlparse(url)
+
+    if parsed_url.scheme not in ALLOWED_SCHEMES:
+        raise UnsupportedURLSchemeError(parsed_url.scheme)
+
+    if parsed_url.netloc not in ALLOWED_NETLOCS:
+        raise UnsupportedURLNetlocError(parsed_url.netloc)
+
+    path = parsed_url.path
+
+    if path.endswith("/watch"):
+        query = parsed_url.query
+        parsed_query = parse_qs(query)
+        if "v" in parsed_query:
+            ids = parsed_query["v"]
+            video_id = ids if isinstance(ids, str) else ids[0]
+        else:
+            raise NoVideoIDFoundError(url)
+    else:
+        path = parsed_url.path.lstrip("/")
+        video_id = path.split("/")[-1]
+
+    if len(video_id) != 11:  # Video IDs are 11 characters long
+        raise VideoIDError(video_id)
+
+    return video_id
+
+
+class YoutubeLoader(Loader):
+    def __init__(self, languages: list[str] | None = None) -> None:
+        self.languages = languages or DEFAULT_LANGUAGES
+
+    @timeout_decorator.timeout(20)
+    def load(self, url: str) -> str:
+        video_id = parse_video_id(url)
+
+        transcript_pieces: list[dict[str, str | float]] = YouTubeTranscriptApi().get_transcript(
+            video_id, self.languages
+        )
+
+        lines = []
+        for transcript_piece in transcript_pieces:
+            text = str(transcript_piece.get("text", "")).strip()
+            if text:
+                lines.append(text)
+        return "\n".join(lines)

kabigon/ytdlp.py

@@ -0,0 +1,132 @@
+import functools
+import hashlib
+import os
+import subprocess
+import tempfile
+from typing import Final
+
+import numpy as np
+import timeout_decorator
+import whisper
+import yt_dlp
+from loguru import logger
+
+from .loader import Loader
+
+try:
+    import mlx_whisper  # noqa: F401
+
+    _mlx_whisper_installed = True
+except ImportError:
+    _mlx_whisper_installed = False
+
+
+DEFAULT_FFMPEG_PATH: Final[str] = "ffmpeg"
+
+
+def hash_url(url: str) -> str:
+    return hashlib.sha512(url.encode("utf-8")).hexdigest()
+
+
+def get_ffmpeg_path() -> str:
+    path = os.getenv("FFMPEG_PATH")
+    if not path:
+        path = DEFAULT_FFMPEG_PATH
+        logger.warning("FFMPEG_PATH not set, using default: {}", DEFAULT_FFMPEG_PATH)
+
+    return path
+
+
+def download_audio(url: str) -> str:
+    ffmpeg_path = get_ffmpeg_path()
+
+    filename = os.path.join(
+        tempfile.gettempdir(),
+        hash_url(url),
+    )
+
+    ydl_opts = {
+        "format": "bestaudio/best",
+        "postprocessors": [
+            {
+                "key": "FFmpegExtractAudio",
+                "preferredcodec": "mp3",
+                "preferredquality": "192",
+            }
+        ],
+        "outtmpl": filename,
+        "ffmpeg_location": ffmpeg_path,
+        "match_filter": yt_dlp.match_filter_func(["!is_live"]),
+    }
+
+    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+        ydl.download([url])
+
+    return filename + ".mp3"
+
+
+def load_audio(file: str, sr: int = 16000):
+    """
+    Open an audio file and read as mono waveform, resampling as necessary
+
+    Parameters
+    ----------
+    file: str
+        The audio file to open
+
+    sr: int
+        The sample rate to resample the audio if necessary
+
+    Returns
+    -------
+    A NumPy array containing the audio waveform, in float32 dtype.
+    """
+    ffmpeg_path = get_ffmpeg_path()
+
+    # This launches a subprocess to decode audio while down-mixing
+    # and resampling as necessary.  Requires the ffmpeg CLI in PATH.
+    # fmt: off
+    cmd = [
+        ffmpeg_path,
+        "-nostdin",
+        "-threads", "0",
+        "-i", file,
+        "-f", "s16le",
+        "-ac", "1",
+        "-acodec", "pcm_s16le",
+        "-ar", str(sr),
+        "-"
+    ]
+    # fmt: on
+    try:
+        out = subprocess.run(cmd, capture_output=True, check=True).stdout
+    except subprocess.CalledProcessError as e:
+        raise RuntimeError(f"Failed to load audio: {e.stderr.decode()}") from e
+
+    return np.frombuffer(out, np.int16).flatten().astype(np.float32) / 32768.0
+
+
+@functools.cache
+def _load_whisper_model() -> whisper.Whisper:
+    return whisper.load_model("tiny")
+
+
+def _transcribe(audio: np.ndarray) -> dict:
+    if _mlx_whisper_installed:
+        return mlx_whisper.transcribe(audio, path_or_hf_repo="mlx-community/whisper-tiny")
+
+    model = _load_whisper_model()
+    return model.transcribe(audio)
+
+
+class YtdlpLoader(Loader):
+    @timeout_decorator.timeout(300)
+    def load(self, url: str) -> str:
+        audio_file = download_audio(url)
+        audio = load_audio(audio_file)
+
+        # Clean up the audio file
+        os.remove(audio_file)
+
+        result = _transcribe(audio)
+        return result.get("text", "")

kabigon-0.1.0.dist-info/METADATA

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: kabigon
+Version: 0.1.0
+Author-email: narumi <toucans-cutouts0f@icloud.com>
+License-File: LICENSE
+Requires-Python: >=3.12
+Requires-Dist: click>=8.1.8
+Requires-Dist: cloudscraper>=1.2.71
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: markdownify>=0.14.1
+Requires-Dist: numba>=0.61.0
+Requires-Dist: openai-whisper>=20240930
+Requires-Dist: playwright>=1.50.0
+Requires-Dist: pypdf>=5.3.0
+Requires-Dist: rich>=13.9.4
+Requires-Dist: timeout-decorator>=0.5.0
+Requires-Dist: youtube-transcript-api>=0.6.3
+Requires-Dist: yt-dlp>=2025.1.26
+Description-Content-Type: text/markdown
+
+# python-template

kabigon-0.1.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+kabigon/__init__.py,sha256=XFwiwCa3r17Ehn4_unnWmTXZ54EFQRRTvNaCjE2f5I4,553
+kabigon/cli.py,sha256=eOto76gMuHSqpn9XuJsoUY1sG-lDb2Ny7DLbhSyqWAI,221
+kabigon/cloudscraper.py,sha256=viaIWATsS8nD9HN0RfBiveUHuL012OjuaKlKwLEteGw,403
+kabigon/httpx.py,sha256=SfhaJXNKlFOwWs_Eeadiegi5wNvZV0RX7lqQwR8nYGo,667
+kabigon/loader.py,sha256=_iRrjwW9SPWnqnsl8QFB1uEgh4Ed27zXio6GEl98b5A,126
+kabigon/pdf.py,sha256=9Oi_ZP7D0LLWs2D8KO3omRld1nYyhEbQMForZUw9YZg,1548
+kabigon/pipeline.py,sha256=XCjZMpOrbIfNcL6hTShvBHpsyOkKymvNRirp6PVIIDA,1874
+kabigon/playwright.py,sha256=0CkSDY90i7PHKxrJ6Zwad_NDCy5TPyCjeFvLrQuZXFU,1154
+kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+kabigon/reel.py,sha256=dkWXG2nBhIt0DpGJzevkIrRKLqJh_03-yrg_rjf6vnY,828
+kabigon/singlefile.py,sha256=2nTCTFgW5Gp3l0ExaVh2foUDVSgLaAssDp3tBoQ1MhY,1860
+kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
+kabigon/youtube.py,sha256=_wdKvRRAMrYnv3rUhkd_6JuOGCuQClYpj1UlVeYeojc,2615
+kabigon/ytdlp.py,sha256=kG1fXqU650otOWespjOSkGK_-jk1wO-sWiR60_UPJxY,3125
+kabigon-0.1.0.dist-info/METADATA,sha256=rtCDMxACLbzTuTKc2Yq4kPkteYSUjtbiyeVZHOmARmo,641
+kabigon-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kabigon-0.1.0.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
+kabigon-0.1.0.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
+kabigon-0.1.0.dist-info/RECORD,,

kabigon-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

kabigon-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+kabigon = kabigon.cli:main

kabigon-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 なるみ
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.