kabigon 0.6.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

kabigon/__init__.py

@@ -11,7 +11,7 @@ from .loader import Loader
 from .pdf import PDFLoader
 from .playwright import PlaywrightLoader
 from .reel import ReelLoader
-from .singlefile import SinglefileLoader
+from .twitter import TwitterLoader
 from .youtube import YoutubeLoader
 from .ytdlp import YtdlpLoader
 

kabigon/cli.py

@@ -6,6 +6,7 @@ from .httpx import HttpxLoader
 from .pdf import PDFLoader
 from .playwright import PlaywrightLoader
 from .reel import ReelLoader
+from .twitter import TwitterLoader
 from .youtube import YoutubeLoader
 from .ytdlp import YtdlpLoader
 
@@ -15,6 +16,7 @@ from .ytdlp import YtdlpLoader
 def main(url: str) -> None:
     loader = Compose(
         [
+            TwitterLoader(),
             YoutubeLoader(),
             ReelLoader(),
             YtdlpLoader(),

kabigon/compose.py

@@ -1,40 +1,14 @@
-from urllib.parse import urlparse
-from urllib.parse import urlunparse
-
 from loguru import logger
 
-from .errors import LoaderError
+from .errors import KabigonError
 from .loader import Loader
 
-REPLACEMENTS = {
-    "api.fxtwitter.com": [
-        "twitter.com",
-        "x.com",
-        "fxtwitter.com",
-        "vxtwitter.com",
-        "fixvx.com",
-        "twittpr.com",
-        "fixupx.com",
-    ]
-}
-
-
-def replace_domain(url: str) -> str:
-    parsed = urlparse(url)
-    for target, source in REPLACEMENTS.items():
-        if parsed.netloc in source:
-            fixed_url = parsed._replace(netloc=target)
-            return urlunparse(fixed_url)
-    return url
-
 
 class Compose(Loader):
     def __init__(self, loaders: list[Loader]) -> None:
         self.loaders = loaders
 
     def load(self, url: str) -> str:
-        url = replace_domain(url)
-
         for loader in self.loaders:
             try:
                 content = loader.load(url)
@@ -49,11 +23,9 @@ class Compose(Loader):
             except Exception as e:
                 logger.info("[{}] Failed to load URL: {}, got error: {}", loader.__class__.__name__, url, e)
 
-        raise LoaderError(f"Failed to load URL: {url}")
+        raise KabigonError(f"Failed to load URL: {url}")
 
     async def async_load(self, url: str) -> str:
-        url = replace_domain(url)
-
         for loader in self.loaders:
             try:
                 content = await loader.async_load(url)
@@ -68,4 +40,4 @@ class Compose(Loader):
             except Exception as e:
                 logger.info("[{}] Failed to load URL: {}, got error: {}", loader.__class__.__name__, url, e)
 
-        raise LoaderError(f"Failed to load URL: {url}")
+        raise KabigonError(f"Failed to load URL: {url}")

kabigon/errors.py

@@ -1,23 +1,28 @@
-class LoaderError(Exception):
+class KabigonError(Exception):
     pass
 
 
-class FirecrawlKeyError(LoaderError):
+class FirecrawlKeyError(KabigonError):
     def __init__(self) -> None:
         super().__init__("FIRECRAWL_API_KEY is not set.")
 
 
-class FirecrawlError(LoaderError):
+class FirecrawlError(KabigonError):
     def __init__(self, url: str, error: str) -> None:
         msg = f"Failed to load URL: {url}, got: {error}"
         super().__init__(msg)
 
 
-class NotPDFError(LoaderError):
+class NotPDFError(KabigonError):
     def __init__(self, url: str) -> None:
         super().__init__(f"URL is not a PDF: {url}")
 
 
-class NotReelURLError(LoaderError):
+class NotReelURLError(KabigonError):
     def __init__(self, url: str):
         super().__init__(f"URL is not an Instagram Reel: {url}")
+
+
+class NotTwitterURLError(KabigonError):
+    def __init__(self, url: str):
+        super().__init__(f"URL is not a Twitter URL: {url}")

kabigon/twitter.py

@@ -0,0 +1,46 @@
+from urllib.parse import urlparse
+from urllib.parse import urlunparse
+
+from .errors import NotTwitterURLError
+from .loader import Loader
+from .playwright import PlaywrightLoader
+
+TWITTER_DOMAINS = [
+    "twitter.com",
+    "x.com",
+    "fxtwitter.com",
+    "vxtwitter.com",
+    "fixvx.com",
+    "twittpr.com",
+    "api.fxtwitter.com",
+    "fixupx.com",
+]
+
+
+def replace_domain(url: str, new_domain: str = "x.com") -> str:
+    return urlunparse(urlparse(url)._replace(netloc=new_domain))
+
+
+def is_x_url(url: str) -> bool:
+    return urlparse(url).netloc in TWITTER_DOMAINS
+
+
+class TwitterLoader(Loader):
+    def __init__(self) -> None:
+        self.playwright_loader = PlaywrightLoader(wait_until="networkidle")
+
+    def load(self, url: str) -> str:
+        if not is_x_url(url):
+            raise NotTwitterURLError(url)
+
+        url = replace_domain(url)
+
+        return self.playwright_loader.load(url)
+
+    async def async_load(self, url: str):
+        if not is_x_url(url):
+            raise NotTwitterURLError(url)
+
+        url = replace_domain(url)
+
+        return await self.playwright_loader.async_load(url)

kabigon/ytdlp.py

@@ -14,7 +14,7 @@ from loguru import logger
 from .loader import Loader
 
 try:
-    import mlx_whisper  # noqa: F401
+    import mlx_whisper  # noqa: F401 # type: ignore
 
     _mlx_whisper_installed = True
 except ImportError:

{kabigon-0.6.0.dist-info → kabigon-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kabigon
-Version: 0.6.0
+Version: 0.7.0
 Author-email: narumi <toucans-cutouts0f@icloud.com>
 License-File: LICENSE
 Requires-Python: >=3.10
@@ -11,12 +11,12 @@ Requires-Dist: httpx>=0.28.1
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: markdownify>=0.14.1
 Requires-Dist: openai-whisper>=20240930
-Requires-Dist: playwright>=1.50.0
+Requires-Dist: playwright>=1.52.0
 Requires-Dist: pypdf>=5.3.0
 Requires-Dist: rich>=13.9.4
 Requires-Dist: timeout-decorator>=0.5.0
 Requires-Dist: youtube-transcript-api>=0.6.3
-Requires-Dist: yt-dlp>=2025.1.26
+Requires-Dist: yt-dlp>=2025.4.30
 Description-Content-Type: text/markdown
 
 # kabigon

kabigon-0.7.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+kabigon/__init__.py,sha256=L-MDyX7PrPZvq6mASyOCJnATebwn-Lj1Nobp9GAPeyU,533
+kabigon/cli.py,sha256=co9IczXx0bAnXo6EEzEdCMSfpBetkPe6pB5hd2jtzA8,675
+kabigon/compose.py,sha256=l2D5OK91VcN2a6DbjMdwBk3YSqzVV7fOVX0TqNm2gJo,1502
+kabigon/errors.py,sha256=iri_YS71UsOHwaVtfy5IA6iUfq30DCsptZsChmZaZic,755
+kabigon/firecrawl.py,sha256=Xnrlhco_R58x5kwGy9ZCKTnVqS4Pp-D3G0u-qnuGEsU,800
+kabigon/httpx.py,sha256=uDdLks6zVzirY7-mnsJkypX86kAI5XmUVfK-lFifdJA,895
+kabigon/loader.py,sha256=KhOJvlzLDM0o05o4VqMRgiyRVjofkzuGEcihQagj-8M,452
+kabigon/pdf.py,sha256=yJcgkdMMF52baFsFxJp9Jn89KsTKBboLTLwD3gs6U5U,1775
+kabigon/playwright.py,sha256=MZ-r0Ej2wWAOJkDLwYRvO77wcDvh38KXz2wgDsCTgm0,1358
+kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+kabigon/reel.py,sha256=J2QOxGMYi_HaEscQPIipPEoHGN_iksGxR6pV_XvryME,929
+kabigon/twitter.py,sha256=U07pa8xA0nHAaaDPeUelQRvXR5ZnUvYJZW35xRAvHA8,1114
+kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
+kabigon/youtube.py,sha256=HoiFNq0ookPL7_rO_wloBaY8yTIX6xP8A77F7y02q64,1166
+kabigon/ytdlp.py,sha256=_QRcyFx9s7NnI1MvcWdKKxlX-hHLnqtduCSL5_UH6dU,3140
+kabigon-0.7.0.dist-info/METADATA,sha256=4rng3Fe89ifNd6bQoBggRhCoqTMIa4sWhbU6N75puzo,1078
+kabigon-0.7.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kabigon-0.7.0.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
+kabigon-0.7.0.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
+kabigon-0.7.0.dist-info/RECORD,,

kabigon/singlefile.py

@@ -1,65 +0,0 @@
-import os
-import subprocess
-import tempfile
-from functools import cache
-from pathlib import Path
-from typing import Final
-
-import charset_normalizer
-from loguru import logger
-
-from .loader import Loader
-from .utils import html_to_markdown
-
-DEFAULT_SINGLEFILE_PATH: Final[str] = "single-file"
-
-
-@cache
-def get_singlefile_path() -> str:
-    path = os.getenv("SINGLEFILE_PATH")
-    if not path:
-        path = DEFAULT_SINGLEFILE_PATH
-        logger.warning("SINGLEFILE_PATH not set, using default: {}", DEFAULT_SINGLEFILE_PATH)
-    return path
-
-
-class SinglefileLoader(Loader):
-    def __init__(self, cookies_file: str | None = None, browser_headless: bool = False) -> None:
-        self.cookies_file = cookies_file
-        self.browser_headless = browser_headless
-
-    def load(self, url: str) -> str:
-        filename = self.download(url)
-        content = str(charset_normalizer.from_path(filename).best())
-        return html_to_markdown(content)
-
-    def download(self, url: str) -> str:
-        logger.info("Downloading HTML using SingleFile: {}", url)
-
-        filename = tempfile.mktemp(suffix=".html")
-        singlefile_path = get_singlefile_path()
-
-        cmds = [singlefile_path]
-
-        if self.cookies_file is not None:
-            cookies_path = Path(self.cookies_file)
-            if not cookies_path.exists():
-                raise FileNotFoundError(f"Cookies file not found: {self.cookies_file}")
-
-            cmds += [
-                "--browser-cookies-file",
-                str(cookies_path),
-            ]
-
-        cmds += [
-            "--filename-conflict-action",
-            "overwrite",
-            "--browser-headless",
-            str(self.browser_headless).lower(),
-            url,
-            filename,
-        ]
-
-        subprocess.run(cmds)
-
-        return filename

kabigon-0.6.0.dist-info/RECORD

@@ -1,20 +0,0 @@
-kabigon/__init__.py,sha256=L64FtFCQQxkqXXHDkW2rniWGAnufmC9WG6KBwutXU4w,539
-kabigon/cli.py,sha256=XSTyD1RFqq2Qok_52kSjJlBLUXl6t-K9QtsxCfB15o4,611
-kabigon/compose.py,sha256=Yhn9fe0mylVLpK01sT9OWWItIY1gD8TZLQ4qY8hAeSQ,2110
-kabigon/errors.py,sha256=tYokuDQ1-U25zq3hWu-GsYJBDpDq3VuObUijnd7Nc7M,613
-kabigon/firecrawl.py,sha256=Xnrlhco_R58x5kwGy9ZCKTnVqS4Pp-D3G0u-qnuGEsU,800
-kabigon/httpx.py,sha256=uDdLks6zVzirY7-mnsJkypX86kAI5XmUVfK-lFifdJA,895
-kabigon/loader.py,sha256=KhOJvlzLDM0o05o4VqMRgiyRVjofkzuGEcihQagj-8M,452
-kabigon/pdf.py,sha256=yJcgkdMMF52baFsFxJp9Jn89KsTKBboLTLwD3gs6U5U,1775
-kabigon/playwright.py,sha256=MZ-r0Ej2wWAOJkDLwYRvO77wcDvh38KXz2wgDsCTgm0,1358
-kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-kabigon/reel.py,sha256=J2QOxGMYi_HaEscQPIipPEoHGN_iksGxR6pV_XvryME,929
-kabigon/singlefile.py,sha256=CeTT2WPYm0vb1xWPNdyBN4uHRw9hRqfZm68D-nEcUA8,1800
-kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
-kabigon/youtube.py,sha256=HoiFNq0ookPL7_rO_wloBaY8yTIX6xP8A77F7y02q64,1166
-kabigon/ytdlp.py,sha256=kG1fXqU650otOWespjOSkGK_-jk1wO-sWiR60_UPJxY,3125
-kabigon-0.6.0.dist-info/METADATA,sha256=zsjAoTye8jH1NtI_ZylcCI8Vt-S0CqQWcN-ca39cbLY,1078
-kabigon-0.6.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kabigon-0.6.0.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
-kabigon-0.6.0.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
-kabigon-0.6.0.dist-info/RECORD,,