kabigon 0.5.3__py3-none-any.whl → 0.6.1__py3-none-any.whl

kabigon/__init__.py

@@ -5,12 +5,12 @@ from typing import Final
 from loguru import logger
 
 from .compose import Compose
+from .firecrawl import FirecrawlLoader
 from .httpx import HttpxLoader
 from .loader import Loader
 from .pdf import PDFLoader
 from .playwright import PlaywrightLoader
 from .reel import ReelLoader
-from .singlefile import SinglefileLoader
 from .youtube import YoutubeLoader
 from .ytdlp import YtdlpLoader
 

kabigon/compose.py

@@ -3,18 +3,19 @@ from urllib.parse import urlunparse
 
 from loguru import logger
 
+from .errors import KabigonError
 from .loader import Loader
-from .loader import LoaderError
 
 REPLACEMENTS = {
-    "api.fxtwitter.com": [
+    # fixupx.com seems better than api.fxtwitter.com
+    "fixupx.com": [
         "twitter.com",
         "x.com",
         "fxtwitter.com",
         "vxtwitter.com",
         "fixvx.com",
         "twittpr.com",
-        "fixupx.com",
+        "api.fxtwitter.com",
     ]
 }
 
@@ -49,7 +50,7 @@ class Compose(Loader):
             except Exception as e:
                 logger.info("[{}] Failed to load URL: {}, got error: {}", loader.__class__.__name__, url, e)
 
-        raise LoaderError(f"Failed to load URL: {url}")
+        raise KabigonError(f"Failed to load URL: {url}")
 
     async def async_load(self, url: str) -> str:
         url = replace_domain(url)
@@ -68,4 +69,4 @@ class Compose(Loader):
             except Exception as e:
                 logger.info("[{}] Failed to load URL: {}, got error: {}", loader.__class__.__name__, url, e)
 
-        raise LoaderError(f"Failed to load URL: {url}")
+        raise KabigonError(f"Failed to load URL: {url}")

kabigon/errors.py

@@ -0,0 +1,23 @@
+class KabigonError(Exception):
+    pass
+
+
+class FirecrawlKeyError(KabigonError):
+    def __init__(self) -> None:
+        super().__init__("FIRECRAWL_API_KEY is not set.")
+
+
+class FirecrawlError(KabigonError):
+    def __init__(self, url: str, error: str) -> None:
+        msg = f"Failed to load URL: {url}, got: {error}"
+        super().__init__(msg)
+
+
+class NotPDFError(KabigonError):
+    def __init__(self, url: str) -> None:
+        super().__init__(f"URL is not a PDF: {url}")
+
+
+class NotReelURLError(KabigonError):
+    def __init__(self, url: str):
+        super().__init__(f"URL is not an Instagram Reel: {url}")

kabigon/firecrawl.py

@@ -0,0 +1,33 @@
+import os
+
+from firecrawl import FirecrawlApp
+
+from .errors import FirecrawlError
+from .errors import FirecrawlKeyError
+from .loader import Loader
+
+
+class FirecrawlLoader(Loader):
+    def __init__(self, timeout: int | None = None) -> None:
+        self.timeout = timeout
+
+        api_key = os.getenv("FIRECRAWL_API_KEY")
+        if not api_key:
+            raise FirecrawlKeyError()
+
+        self.app = FirecrawlApp(api_key=api_key)
+
+    def load(self, url: str) -> str:
+        result = self.app.scrape_url(
+            url,
+            formats=["markdown"],
+            timeout=self.timeout,
+        )
+
+        if not result.success:
+            raise FirecrawlError(url, result.error)
+
+        return result.markdown
+
+    async def async_load(self, url: str) -> str:
+        return self.load(url)

kabigon/loader.py

@@ -14,7 +14,3 @@ class Loader:
         with concurrent.futures.ProcessPoolExecutor() as executor:
             result = await loop.run_in_executor(executor, self.load, url)
             return result
-
-
-class LoaderError(Exception):
-    pass

kabigon/pdf.py

@@ -6,8 +6,8 @@ from typing import Any
 import httpx
 from pypdf import PdfReader
 
+from .errors import NotPDFError
 from .loader import Loader
-from .loader import LoaderError
 
 DEFAULT_HEADERS = {
     "Accept-Language": "zh-TW,zh;q=0.9,ja;q=0.8,en-US;q=0.7,en;q=0.6",
@@ -15,11 +15,6 @@ DEFAULT_HEADERS = {
 }
 
 
-class NotPDFError(LoaderError):
-    def __init__(self, url: str) -> None:
-        super().__init__(f"URL is not a PDF: {url}")
-
-
 class PDFLoader(Loader):
     def load(self, url_or_file: str) -> str:
         if not url_or_file.startswith("http"):

kabigon/reel.py

@@ -1,6 +1,6 @@
+from .errors import NotReelURLError
 from .httpx import HttpxLoader
 from .loader import Loader
-from .loader import LoaderError
 from .ytdlp import YtdlpLoader
 
 
@@ -8,11 +8,6 @@ def is_reel_url(url: str) -> bool:
     return url.startswith("https://www.instagram.com/reel")
 
 
-class NotReelURLError(LoaderError):
-    def __init__(self, url: str):
-        super().__init__(f"URL is not an Instagram Reel: {url}")
-
-
 class ReelLoader(Loader):
     def __init__(self) -> None:
         self.httpx_loader = HttpxLoader()

kabigon/ytdlp.py

@@ -14,7 +14,7 @@ from loguru import logger
 from .loader import Loader
 
 try:
-    import mlx_whisper  # noqa: F401
+    import mlx_whisper  # noqa: F401 # type: ignore
 
     _mlx_whisper_installed = True
 except ImportError:

{kabigon-0.5.3.dist-info → kabigon-0.6.1.dist-info}/METADATA

@@ -1,11 +1,12 @@
 Metadata-Version: 2.4
 Name: kabigon
-Version: 0.5.3
+Version: 0.6.1
 Author-email: narumi <toucans-cutouts0f@icloud.com>
 License-File: LICENSE
 Requires-Python: >=3.10
 Requires-Dist: aioytt>=0.2.4
 Requires-Dist: click>=8.1.8
+Requires-Dist: firecrawl-py>=2.4.1
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: markdownify>=0.14.1

kabigon-0.6.1.dist-info/RECORD

@@ -0,0 +1,19 @@
+kabigon/__init__.py,sha256=_1LdKfp7qm0m7Fa_IJ9zcg4TRB14c9bB-r9M3fwASWI,498
+kabigon/cli.py,sha256=XSTyD1RFqq2Qok_52kSjJlBLUXl6t-K9QtsxCfB15o4,611
+kabigon/compose.py,sha256=5H_hWQ7ENUmEdeOpB2bkRv4U_U18_1IyNTran-cLYaM,2166
+kabigon/errors.py,sha256=3eSPQtLocreKuq9mhwRk7IMdA2xh8KkmIFEUhGmTPxg,618
+kabigon/firecrawl.py,sha256=Xnrlhco_R58x5kwGy9ZCKTnVqS4Pp-D3G0u-qnuGEsU,800
+kabigon/httpx.py,sha256=uDdLks6zVzirY7-mnsJkypX86kAI5XmUVfK-lFifdJA,895
+kabigon/loader.py,sha256=KhOJvlzLDM0o05o4VqMRgiyRVjofkzuGEcihQagj-8M,452
+kabigon/pdf.py,sha256=yJcgkdMMF52baFsFxJp9Jn89KsTKBboLTLwD3gs6U5U,1775
+kabigon/playwright.py,sha256=MZ-r0Ej2wWAOJkDLwYRvO77wcDvh38KXz2wgDsCTgm0,1358
+kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+kabigon/reel.py,sha256=J2QOxGMYi_HaEscQPIipPEoHGN_iksGxR6pV_XvryME,929
+kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
+kabigon/youtube.py,sha256=HoiFNq0ookPL7_rO_wloBaY8yTIX6xP8A77F7y02q64,1166
+kabigon/ytdlp.py,sha256=_QRcyFx9s7NnI1MvcWdKKxlX-hHLnqtduCSL5_UH6dU,3140
+kabigon-0.6.1.dist-info/METADATA,sha256=78J35ClbTdy2-vU8GtlQWjgSTitntFI8J-Cz07FGqoo,1078
+kabigon-0.6.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kabigon-0.6.1.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
+kabigon-0.6.1.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
+kabigon-0.6.1.dist-info/RECORD,,

kabigon/singlefile.py

@@ -1,65 +0,0 @@
-import os
-import subprocess
-import tempfile
-from functools import cache
-from pathlib import Path
-from typing import Final
-
-import charset_normalizer
-from loguru import logger
-
-from .loader import Loader
-from .utils import html_to_markdown
-
-DEFAULT_SINGLEFILE_PATH: Final[str] = "single-file"
-
-
-@cache
-def get_singlefile_path() -> str:
-    path = os.getenv("SINGLEFILE_PATH")
-    if not path:
-        path = DEFAULT_SINGLEFILE_PATH
-        logger.warning("SINGLEFILE_PATH not set, using default: {}", DEFAULT_SINGLEFILE_PATH)
-    return path
-
-
-class SinglefileLoader(Loader):
-    def __init__(self, cookies_file: str | None = None, browser_headless: bool = False) -> None:
-        self.cookies_file = cookies_file
-        self.browser_headless = browser_headless
-
-    def load(self, url: str) -> str:
-        filename = self.download(url)
-        content = str(charset_normalizer.from_path(filename).best())
-        return html_to_markdown(content)
-
-    def download(self, url: str) -> str:
-        logger.info("Downloading HTML using SingleFile: {}", url)
-
-        filename = tempfile.mktemp(suffix=".html")
-        singlefile_path = get_singlefile_path()
-
-        cmds = [singlefile_path]
-
-        if self.cookies_file is not None:
-            cookies_path = Path(self.cookies_file)
-            if not cookies_path.exists():
-                raise FileNotFoundError(f"Cookies file not found: {self.cookies_file}")
-
-            cmds += [
-                "--browser-cookies-file",
-                str(cookies_path),
-            ]
-
-        cmds += [
-            "--filename-conflict-action",
-            "overwrite",
-            "--browser-headless",
-            str(self.browser_headless).lower(),
-            url,
-            filename,
-        ]
-
-        subprocess.run(cmds)
-
-        return filename

kabigon-0.5.3.dist-info/RECORD

@@ -1,18 +0,0 @@
-kabigon/__init__.py,sha256=7ll3ePlHNbZq-CmrGMrQouLCUSmuRsZ9yAj2JOzr7HY,500
-kabigon/cli.py,sha256=XSTyD1RFqq2Qok_52kSjJlBLUXl6t-K9QtsxCfB15o4,611
-kabigon/compose.py,sha256=DO0hOJgEAX7ZLOS53dcE6V9zi7Tr9oGNW8koPHsx9eM,2110
-kabigon/httpx.py,sha256=uDdLks6zVzirY7-mnsJkypX86kAI5XmUVfK-lFifdJA,895
-kabigon/loader.py,sha256=D5xUPJb3uAygmBaN_sX56ZpGcGsVz-ueHOXC7gSGaxM,493
-kabigon/pdf.py,sha256=Q9XuBdKDrDQJ8BNvY7Lgt6dpGeA_ylGGHWOE3euiI_8,1904
-kabigon/playwright.py,sha256=MZ-r0Ej2wWAOJkDLwYRvO77wcDvh38KXz2wgDsCTgm0,1358
-kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-kabigon/reel.py,sha256=TP_oKYXABXYja2A9damTBWR3MVYA7aZyxbIvCuTcq40,1062
-kabigon/singlefile.py,sha256=CeTT2WPYm0vb1xWPNdyBN4uHRw9hRqfZm68D-nEcUA8,1800
-kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
-kabigon/youtube.py,sha256=HoiFNq0ookPL7_rO_wloBaY8yTIX6xP8A77F7y02q64,1166
-kabigon/ytdlp.py,sha256=kG1fXqU650otOWespjOSkGK_-jk1wO-sWiR60_UPJxY,3125
-kabigon-0.5.3.dist-info/METADATA,sha256=gzcVy_2l4kAo_heevYpN-NsHY9f7RMFlFGk6rkSCVQU,1043
-kabigon-0.5.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kabigon-0.5.3.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
-kabigon-0.5.3.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
-kabigon-0.5.3.dist-info/RECORD,,