kabigon 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

kabigon/cli.py

@@ -1,7 +1,6 @@
 import click
 from rich import print
 
-from .cloudscraper import CloudscraperLoader
 from .compose import Compose
 from .httpx import HttpxLoader
 from .pdf import PDFLoader
@@ -20,7 +19,6 @@ def main(url: str) -> None:
             ReelLoader(),
             YtdlpLoader(),
             PDFLoader(),
-            CloudscraperLoader(),
             HttpxLoader(),
             SinglefileLoader(),
         ]

kabigon/cloudscraper.py

@@ -6,7 +6,7 @@ from .utils import html_to_markdown
 
 
 class CloudscraperLoader(Loader):
-    @timeout_decorator.timeout(5)
+    @timeout_decorator.timeout(10)
     def load(self, url: str) -> str:
         client = cloudscraper.create_scraper()
         response = client.get(url, allow_redirects=True)

kabigon/compose.py

@@ -37,14 +37,33 @@ class Compose(Loader):
 
         for loader in self.loaders:
             try:
-                loaded_content = loader.load(url)
+                content = loader.load(url)
 
-                if not loaded_content:
+                if not content:
                     logger.info("[{}] Failed to load URL: {}, got empty result", loader.__class__.__name__, url)
                     continue
 
                 logger.info("[{}] Successfully loaded URL: {}", loader.__class__.__name__, url)
-                return loaded_content
+                return content
+
+            except Exception as e:
+                logger.info("[{}] Failed to load URL: {}, got error: {}", loader.__class__.__name__, url, e)
+
+        raise LoaderError(f"Failed to load URL: {url}")
+
+    async def async_load(self, url: str) -> str:
+        url = replace_domain(url)
+
+        for loader in self.loaders:
+            try:
+                content = await loader.async_load(url)
+
+                if not content:
+                    logger.info("[{}] Failed to load URL: {}, got empty result", loader.__class__.__name__, url)
+                    continue
+
+                logger.info("[{}] Successfully loaded URL: {}", loader.__class__.__name__, url)
+                return content
 
             except Exception as e:
                 logger.info("[{}] Failed to load URL: {}, got error: {}", loader.__class__.__name__, url, e)

kabigon/httpx.py

@@ -12,8 +12,14 @@ DEFAULT_HEADERS = {
 
 
 class HttpxLoader(Loader):
-    @timeout_decorator.timeout(5)
+    @timeout_decorator.timeout(10)
     def load(self, url: str) -> str:
         response = httpx.get(url, headers=DEFAULT_HEADERS, follow_redirects=True)
         response.raise_for_status()
         return html_to_markdown(response.content)
+
+    async def async_load(self, url: str) -> str:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(url, headers=DEFAULT_HEADERS, follow_redirects=True)
+            response.raise_for_status()
+            return html_to_markdown(response.content)

kabigon/loader.py

@@ -1,3 +1,7 @@
+import asyncio
+import concurrent.futures
+
+
 class Loader:
     def __call__(self, url: str) -> str:
         return self.load(url)
@@ -5,6 +9,12 @@ class Loader:
     def load(self, url: str) -> str:
         raise NotImplementedError
 
+    async def async_load(self, url: str):
+        loop = asyncio.get_running_loop()
+        with concurrent.futures.ProcessPoolExecutor() as executor:
+            result = await loop.run_in_executor(executor, self.load, url)
+            return result
+
 
 class LoaderError(Exception):
     pass

kabigon/pdf.py

@@ -19,7 +19,7 @@ class NotPDFError(LoaderError):
 
 
 class PDFLoader(Loader):
-    @timeout_decorator.timeout(5)
+    @timeout_decorator.timeout(10)
     def load(self, url_or_file: str) -> str:
         if url_or_file.startswith("http"):
             url_or_file = download_pdf_from_url(url_or_file)

kabigon/playwright.py

@@ -1,8 +1,6 @@
 from typing import Literal
 
-import timeout_decorator
-from loguru import logger
-from playwright.sync_api import TimeoutError
+from playwright.async_api import async_playwright
 from playwright.sync_api import sync_playwright
 
 from .loader import Loader
@@ -20,19 +18,26 @@ class PlaywrightLoader(Loader):
         self.wait_until = wait_until
         self.browser_headless = browser_headless
 
-    @timeout_decorator.timeout(5)
     def load(self, url: str) -> str:
         with sync_playwright() as p:
             browser = p.chromium.launch(headless=self.browser_headless)
             page = browser.new_page()
 
-            try:
-                page.goto(url, timeout=self.timeout, wait_until=self.wait_until)
-            except TimeoutError as e:
-                logger.error("TimeoutError: {}", e)
-                page.goto(url)
+            page.goto(url, timeout=self.timeout, wait_until=self.wait_until)
 
             content = page.content()
             browser.close()
 
             return html_to_markdown(content)
+
+    async def async_load(self, url: str) -> str:
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=self.browser_headless)
+            page = await browser.new_page()
+
+            await page.goto(url, timeout=self.timeout, wait_until=self.wait_until)
+
+            content = await page.content()
+            await browser.close()
+
+            return html_to_markdown(content)

kabigon/reel.py

@@ -29,3 +29,12 @@ class ReelLoader(Loader):
         html_content = self.httpx_loader.load(url)
 
         return f"{audio_content}\n\n{html_content}"
+
+    async def async_load(self, url: str):
+        if not is_reel_url(url):
+            raise NotReelURLError(url)
+
+        audio_content = await self.ytdlp_loader.async_load(url)
+        html_content = await self.httpx_loader.async_load(url)
+
+        return f"{audio_content}\n\n{html_content}"

kabigon/singlefile.py

@@ -6,7 +6,6 @@ from pathlib import Path
 from typing import Final
 
 import charset_normalizer
-import timeout_decorator
 from loguru import logger
 
 from .loader import Loader
@@ -29,7 +28,6 @@ class SinglefileLoader(Loader):
         self.cookies_file = cookies_file
         self.browser_headless = browser_headless
 
-    @timeout_decorator.timeout(20)
     def load(self, url: str) -> str:
         filename = self.download(url)
         content = str(charset_normalizer.from_path(filename).best())

{kabigon-0.3.0.dist-info → kabigon-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kabigon
-Version: 0.3.0
+Version: 0.4.0
 Author-email: narumi <toucans-cutouts0f@icloud.com>
 License-File: LICENSE
 Requires-Python: >=3.10
@@ -9,7 +9,6 @@ Requires-Dist: cloudscraper>=1.2.71
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: markdownify>=0.14.1
-Requires-Dist: numpy>=2.1.3
 Requires-Dist: openai-whisper>=20240930
 Requires-Dist: playwright>=1.50.0
 Requires-Dist: pypdf>=5.3.0
@@ -20,3 +19,35 @@ Requires-Dist: yt-dlp>=2025.1.26
 Description-Content-Type: text/markdown
 
 # kabigon
+
+## Installation
+
+```shell
+pip install kabigon
+```
+
+## Usage
+
+```shell
+kabigon <url>
+```
+
+or
+
+```python
+import kabigon
+
+url = "https://www.google.com.tw"
+
+content = kabigon.Compose(
+    [
+        kabigon.YoutubeLoader(),
+        kabigon.ReelLoader(),
+        kabigon.YtdlpLoader(),
+        kabigon.PDFLoader(),
+        kabigon.HttpxLoader(),
+        kabigon.SinglefileLoader(),
+    ]
+).load(url)
+print(content)
+```

kabigon-0.4.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+kabigon/__init__.py,sha256=9RgyhhwjqrW2iQy9RBN2j7VZNhwA9xGo_atC7FKnZA4,545
+kabigon/cli.py,sha256=z3u2Msvi1SWf1fd9nCTzJULeO-rRb5oDKJfPxhUeYQ0,611
+kabigon/cloudscraper.py,sha256=0jzrXVXSZopExyxrDRbcI_2wsbHAg_dqOk4D3Re0jvk,404
+kabigon/compose.py,sha256=DO0hOJgEAX7ZLOS53dcE6V9zi7Tr9oGNW8koPHsx9eM,2110
+kabigon/httpx.py,sha256=B8_26rufJMbKSXINBEqyCIpaRueO_3Gk_PtEQmlOxQ4,955
+kabigon/loader.py,sha256=D5xUPJb3uAygmBaN_sX56ZpGcGsVz-ueHOXC7gSGaxM,493
+kabigon/pdf.py,sha256=oM5pwZJ2GCcHyQXg98-Mda-MHxarYVZQge30KdS_aHY,1549
+kabigon/playwright.py,sha256=ciNUlpMbwd47utCLT454wFSirXFmt3eCXN2Q-nAsiu8,1356
+kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+kabigon/reel.py,sha256=1JTcn7qVH7FcD0Oj-Rz-pnjI-xS1UtkoJcuClGb8ExQ,1124
+kabigon/singlefile.py,sha256=CeTT2WPYm0vb1xWPNdyBN4uHRw9hRqfZm68D-nEcUA8,1800
+kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
+kabigon/youtube.py,sha256=_wdKvRRAMrYnv3rUhkd_6JuOGCuQClYpj1UlVeYeojc,2615
+kabigon/ytdlp.py,sha256=kG1fXqU650otOWespjOSkGK_-jk1wO-sWiR60_UPJxY,3125
+kabigon-0.4.0.dist-info/METADATA,sha256=NRqd2kpi19xYz7TJx5TBXDm-uJcVcpu17P0sw3N-SiA,1020
+kabigon-0.4.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kabigon-0.4.0.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
+kabigon-0.4.0.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
+kabigon-0.4.0.dist-info/RECORD,,

kabigon-0.3.0.dist-info/RECORD

@@ -1,19 +0,0 @@
-kabigon/__init__.py,sha256=9RgyhhwjqrW2iQy9RBN2j7VZNhwA9xGo_atC7FKnZA4,545
-kabigon/cli.py,sha256=7qHklIhYUZ4E78C9MIB16AoVIYt2xLJS0Pz8sr51YPk,690
-kabigon/cloudscraper.py,sha256=viaIWATsS8nD9HN0RfBiveUHuL012OjuaKlKwLEteGw,403
-kabigon/compose.py,sha256=IqNm-Cxl6e8u7X9v5SoG7cjxOWCMIcSZG1lDVNNAfo8,1433
-kabigon/httpx.py,sha256=SfhaJXNKlFOwWs_Eeadiegi5wNvZV0RX7lqQwR8nYGo,667
-kabigon/loader.py,sha256=cV9ZqcWaNtS2WTDpgyNYK2kX5Cu1ZC-Sq-qS3PpPnJQ,198
-kabigon/pdf.py,sha256=9Oi_ZP7D0LLWs2D8KO3omRld1nYyhEbQMForZUw9YZg,1548
-kabigon/playwright.py,sha256=0CkSDY90i7PHKxrJ6Zwad_NDCy5TPyCjeFvLrQuZXFU,1154
-kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-kabigon/reel.py,sha256=dkWXG2nBhIt0DpGJzevkIrRKLqJh_03-yrg_rjf6vnY,828
-kabigon/singlefile.py,sha256=2nTCTFgW5Gp3l0ExaVh2foUDVSgLaAssDp3tBoQ1MhY,1860
-kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
-kabigon/youtube.py,sha256=_wdKvRRAMrYnv3rUhkd_6JuOGCuQClYpj1UlVeYeojc,2615
-kabigon/ytdlp.py,sha256=kG1fXqU650otOWespjOSkGK_-jk1wO-sWiR60_UPJxY,3125
-kabigon-0.3.0.dist-info/METADATA,sha256=w5bE9Wv-p4REXAC_7hPNIVilsxHK7tIZzg2EFz-vh_I,633
-kabigon-0.3.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kabigon-0.3.0.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
-kabigon-0.3.0.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
-kabigon-0.3.0.dist-info/RECORD,,