kabigon 0.5.3__py3-none-any.whl → 0.6.0__py3-none-any.whl

kabigon/__init__.py

@@ -5,6 +5,7 @@ from typing import Final
 from loguru import logger
 
 from .compose import Compose
+from .firecrawl import FirecrawlLoader
 from .httpx import HttpxLoader
 from .loader import Loader
 from .pdf import PDFLoader

kabigon/compose.py

@@ -3,8 +3,8 @@ from urllib.parse import urlunparse
 
 from loguru import logger
 
+from .errors import LoaderError
 from .loader import Loader
-from .loader import LoaderError
 
 REPLACEMENTS = {
     "api.fxtwitter.com": [

kabigon/errors.py

@@ -0,0 +1,23 @@
+class LoaderError(Exception):
+    pass
+
+
+class FirecrawlKeyError(LoaderError):
+    def __init__(self) -> None:
+        super().__init__("FIRECRAWL_API_KEY is not set.")
+
+
+class FirecrawlError(LoaderError):
+    def __init__(self, url: str, error: str) -> None:
+        msg = f"Failed to load URL: {url}, got: {error}"
+        super().__init__(msg)
+
+
+class NotPDFError(LoaderError):
+    def __init__(self, url: str) -> None:
+        super().__init__(f"URL is not a PDF: {url}")
+
+
+class NotReelURLError(LoaderError):
+    def __init__(self, url: str):
+        super().__init__(f"URL is not an Instagram Reel: {url}")

kabigon/firecrawl.py

@@ -0,0 +1,33 @@
+import os
+
+from firecrawl import FirecrawlApp
+
+from .errors import FirecrawlError
+from .errors import FirecrawlKeyError
+from .loader import Loader
+
+
+class FirecrawlLoader(Loader):
+    def __init__(self, timeout: int | None = None) -> None:
+        self.timeout = timeout
+
+        api_key = os.getenv("FIRECRAWL_API_KEY")
+        if not api_key:
+            raise FirecrawlKeyError()
+
+        self.app = FirecrawlApp(api_key=api_key)
+
+    def load(self, url: str) -> str:
+        result = self.app.scrape_url(
+            url,
+            formats=["markdown"],
+            timeout=self.timeout,
+        )
+
+        if not result.success:
+            raise FirecrawlError(url, result.error)
+
+        return result.markdown
+
+    async def async_load(self, url: str) -> str:
+        return self.load(url)

kabigon/loader.py

@@ -14,7 +14,3 @@ class Loader:
         with concurrent.futures.ProcessPoolExecutor() as executor:
             result = await loop.run_in_executor(executor, self.load, url)
             return result
-
-
-class LoaderError(Exception):
-    pass

kabigon/pdf.py

@@ -6,8 +6,8 @@ from typing import Any
 import httpx
 from pypdf import PdfReader
 
+from .errors import NotPDFError
 from .loader import Loader
-from .loader import LoaderError
 
 DEFAULT_HEADERS = {
     "Accept-Language": "zh-TW,zh;q=0.9,ja;q=0.8,en-US;q=0.7,en;q=0.6",
@@ -15,11 +15,6 @@ DEFAULT_HEADERS = {
 }
 
 
-class NotPDFError(LoaderError):
-    def __init__(self, url: str) -> None:
-        super().__init__(f"URL is not a PDF: {url}")
-
-
 class PDFLoader(Loader):
     def load(self, url_or_file: str) -> str:
         if not url_or_file.startswith("http"):

kabigon/reel.py

@@ -1,6 +1,6 @@
+from .errors import NotReelURLError
 from .httpx import HttpxLoader
 from .loader import Loader
-from .loader import LoaderError
 from .ytdlp import YtdlpLoader
 
 
@@ -8,11 +8,6 @@ def is_reel_url(url: str) -> bool:
     return url.startswith("https://www.instagram.com/reel")
 
 
-class NotReelURLError(LoaderError):
-    def __init__(self, url: str):
-        super().__init__(f"URL is not an Instagram Reel: {url}")
-
-
 class ReelLoader(Loader):
     def __init__(self) -> None:
         self.httpx_loader = HttpxLoader()

{kabigon-0.5.3.dist-info → kabigon-0.6.0.dist-info}/METADATA

@@ -1,11 +1,12 @@
 Metadata-Version: 2.4
 Name: kabigon
-Version: 0.5.3
+Version: 0.6.0
 Author-email: narumi <toucans-cutouts0f@icloud.com>
 License-File: LICENSE
 Requires-Python: >=3.10
 Requires-Dist: aioytt>=0.2.4
 Requires-Dist: click>=8.1.8
+Requires-Dist: firecrawl-py>=2.4.1
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: markdownify>=0.14.1

kabigon-0.6.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+kabigon/__init__.py,sha256=L64FtFCQQxkqXXHDkW2rniWGAnufmC9WG6KBwutXU4w,539
+kabigon/cli.py,sha256=XSTyD1RFqq2Qok_52kSjJlBLUXl6t-K9QtsxCfB15o4,611
+kabigon/compose.py,sha256=Yhn9fe0mylVLpK01sT9OWWItIY1gD8TZLQ4qY8hAeSQ,2110
+kabigon/errors.py,sha256=tYokuDQ1-U25zq3hWu-GsYJBDpDq3VuObUijnd7Nc7M,613
+kabigon/firecrawl.py,sha256=Xnrlhco_R58x5kwGy9ZCKTnVqS4Pp-D3G0u-qnuGEsU,800
+kabigon/httpx.py,sha256=uDdLks6zVzirY7-mnsJkypX86kAI5XmUVfK-lFifdJA,895
+kabigon/loader.py,sha256=KhOJvlzLDM0o05o4VqMRgiyRVjofkzuGEcihQagj-8M,452
+kabigon/pdf.py,sha256=yJcgkdMMF52baFsFxJp9Jn89KsTKBboLTLwD3gs6U5U,1775
+kabigon/playwright.py,sha256=MZ-r0Ej2wWAOJkDLwYRvO77wcDvh38KXz2wgDsCTgm0,1358
+kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+kabigon/reel.py,sha256=J2QOxGMYi_HaEscQPIipPEoHGN_iksGxR6pV_XvryME,929
+kabigon/singlefile.py,sha256=CeTT2WPYm0vb1xWPNdyBN4uHRw9hRqfZm68D-nEcUA8,1800
+kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
+kabigon/youtube.py,sha256=HoiFNq0ookPL7_rO_wloBaY8yTIX6xP8A77F7y02q64,1166
+kabigon/ytdlp.py,sha256=kG1fXqU650otOWespjOSkGK_-jk1wO-sWiR60_UPJxY,3125
+kabigon-0.6.0.dist-info/METADATA,sha256=zsjAoTye8jH1NtI_ZylcCI8Vt-S0CqQWcN-ca39cbLY,1078
+kabigon-0.6.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kabigon-0.6.0.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
+kabigon-0.6.0.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
+kabigon-0.6.0.dist-info/RECORD,,

kabigon-0.5.3.dist-info/RECORD

@@ -1,18 +0,0 @@
-kabigon/__init__.py,sha256=7ll3ePlHNbZq-CmrGMrQouLCUSmuRsZ9yAj2JOzr7HY,500
-kabigon/cli.py,sha256=XSTyD1RFqq2Qok_52kSjJlBLUXl6t-K9QtsxCfB15o4,611
-kabigon/compose.py,sha256=DO0hOJgEAX7ZLOS53dcE6V9zi7Tr9oGNW8koPHsx9eM,2110
-kabigon/httpx.py,sha256=uDdLks6zVzirY7-mnsJkypX86kAI5XmUVfK-lFifdJA,895
-kabigon/loader.py,sha256=D5xUPJb3uAygmBaN_sX56ZpGcGsVz-ueHOXC7gSGaxM,493
-kabigon/pdf.py,sha256=Q9XuBdKDrDQJ8BNvY7Lgt6dpGeA_ylGGHWOE3euiI_8,1904
-kabigon/playwright.py,sha256=MZ-r0Ej2wWAOJkDLwYRvO77wcDvh38KXz2wgDsCTgm0,1358
-kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-kabigon/reel.py,sha256=TP_oKYXABXYja2A9damTBWR3MVYA7aZyxbIvCuTcq40,1062
-kabigon/singlefile.py,sha256=CeTT2WPYm0vb1xWPNdyBN4uHRw9hRqfZm68D-nEcUA8,1800
-kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
-kabigon/youtube.py,sha256=HoiFNq0ookPL7_rO_wloBaY8yTIX6xP8A77F7y02q64,1166
-kabigon/ytdlp.py,sha256=kG1fXqU650otOWespjOSkGK_-jk1wO-sWiR60_UPJxY,3125
-kabigon-0.5.3.dist-info/METADATA,sha256=gzcVy_2l4kAo_heevYpN-NsHY9f7RMFlFGk6rkSCVQU,1043
-kabigon-0.5.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kabigon-0.5.3.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
-kabigon-0.5.3.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
-kabigon-0.5.3.dist-info/RECORD,,