kabigon 0.3.1__py3-none-any.whl → 0.4.1__py3-none-any.whl

kabigon/cli.py

@@ -1,7 +1,6 @@
 import click
 from rich import print
 
-from .cloudscraper import CloudscraperLoader
 from .compose import Compose
 from .httpx import HttpxLoader
 from .pdf import PDFLoader
@@ -20,7 +19,6 @@ def main(url: str) -> None:
             ReelLoader(),
             YtdlpLoader(),
             PDFLoader(),
-            CloudscraperLoader(),
             HttpxLoader(),
             SinglefileLoader(),
         ]

kabigon/compose.py

@@ -37,14 +37,33 @@ class Compose(Loader):
 
         for loader in self.loaders:
             try:
-                loaded_content = loader.load(url)
+                content = loader.load(url)
 
-                if not loaded_content:
+                if not content:
                     logger.info("[{}] Failed to load URL: {}, got empty result", loader.__class__.__name__, url)
                     continue
 
                 logger.info("[{}] Successfully loaded URL: {}", loader.__class__.__name__, url)
-                return loaded_content
+                return content
+
+            except Exception as e:
+                logger.info("[{}] Failed to load URL: {}, got error: {}", loader.__class__.__name__, url, e)
+
+        raise LoaderError(f"Failed to load URL: {url}")
+
+    async def async_load(self, url: str) -> str:
+        url = replace_domain(url)
+
+        for loader in self.loaders:
+            try:
+                content = await loader.async_load(url)
+
+                if not content:
+                    logger.info("[{}] Failed to load URL: {}, got empty result", loader.__class__.__name__, url)
+                    continue
+
+                logger.info("[{}] Successfully loaded URL: {}", loader.__class__.__name__, url)
+                return content
 
             except Exception as e:
                 logger.info("[{}] Failed to load URL: {}, got error: {}", loader.__class__.__name__, url, e)

kabigon/httpx.py

@@ -17,3 +17,9 @@ class HttpxLoader(Loader):
         response = httpx.get(url, headers=DEFAULT_HEADERS, follow_redirects=True)
         response.raise_for_status()
         return html_to_markdown(response.content)
+
+    async def async_load(self, url: str) -> str:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(url, headers=DEFAULT_HEADERS, follow_redirects=True)
+            response.raise_for_status()
+            return html_to_markdown(response.content)

kabigon/loader.py

@@ -1,3 +1,7 @@
+import asyncio
+import concurrent.futures
+
+
 class Loader:
     def __call__(self, url: str) -> str:
         return self.load(url)
@@ -5,6 +9,12 @@ class Loader:
     def load(self, url: str) -> str:
         raise NotImplementedError
 
+    async def async_load(self, url: str):
+        loop = asyncio.get_running_loop()
+        with concurrent.futures.ProcessPoolExecutor() as executor:
+            result = await loop.run_in_executor(executor, self.load, url)
+            return result
+
 
 class LoaderError(Exception):
     pass

kabigon/playwright.py

@@ -1,7 +1,6 @@
 from typing import Literal
 
-from loguru import logger
-from playwright.sync_api import TimeoutError
+from playwright.async_api import async_playwright
 from playwright.sync_api import sync_playwright
 
 from .loader import Loader
@@ -24,13 +23,21 @@ class PlaywrightLoader(Loader):
             browser = p.chromium.launch(headless=self.browser_headless)
             page = browser.new_page()
 
-            try:
-                page.goto(url, timeout=self.timeout, wait_until=self.wait_until)
-            except TimeoutError as e:
-                logger.error("TimeoutError: {}", e)
-                page.goto(url)
+            page.goto(url, timeout=self.timeout, wait_until=self.wait_until)
 
             content = page.content()
             browser.close()
 
             return html_to_markdown(content)
+
+    async def async_load(self, url: str) -> str:
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=self.browser_headless)
+            page = await browser.new_page()
+
+            await page.goto(url, timeout=self.timeout, wait_until=self.wait_until)
+
+            content = await page.content()
+            await browser.close()
+
+            return html_to_markdown(content)

kabigon/reel.py

@@ -29,3 +29,12 @@ class ReelLoader(Loader):
         html_content = self.httpx_loader.load(url)
 
         return f"{audio_content}\n\n{html_content}"
+
+    async def async_load(self, url: str):
+        if not is_reel_url(url):
+            raise NotReelURLError(url)
+
+        audio_content = await self.ytdlp_loader.async_load(url)
+        html_content = await self.httpx_loader.async_load(url)
+
+        return f"{audio_content}\n\n{html_content}"

kabigon/youtube.py

@@ -1,6 +1,7 @@
 from urllib.parse import parse_qs
 from urllib.parse import urlparse
 
+import aioytt
 import timeout_decorator
 from youtube_transcript_api import YouTubeTranscriptApi
 
@@ -90,3 +91,12 @@ class YoutubeLoader(Loader):
             if text:
                 lines.append(text)
         return "\n".join(lines)
+
+    async def async_load(self, url: str) -> str:
+        transcript = await aioytt.get_transcript_from_url(url)
+        lines = []
+        for piece in transcript:
+            text = piece.text.strip()
+            if text:
+                lines += text
+        return "\n".join(lines)

{kabigon-0.3.1.dist-info → kabigon-0.4.1.dist-info}/METADATA

@@ -1,9 +1,10 @@
 Metadata-Version: 2.4
 Name: kabigon
-Version: 0.3.1
+Version: 0.4.1
 Author-email: narumi <toucans-cutouts0f@icloud.com>
 License-File: LICENSE
 Requires-Python: >=3.10
+Requires-Dist: aioytt>=0.2.4
 Requires-Dist: click>=8.1.8
 Requires-Dist: cloudscraper>=1.2.71
 Requires-Dist: httpx>=0.28.1
@@ -19,3 +20,35 @@ Requires-Dist: yt-dlp>=2025.1.26
 Description-Content-Type: text/markdown
 
 # kabigon
+
+## Installation
+
+```shell
+pip install kabigon
+```
+
+## Usage
+
+```shell
+kabigon <url>
+```
+
+or
+
+```python
+import kabigon
+
+url = "https://www.google.com.tw"
+
+content = kabigon.Compose(
+    [
+        kabigon.YoutubeLoader(),
+        kabigon.ReelLoader(),
+        kabigon.YtdlpLoader(),
+        kabigon.PDFLoader(),
+        kabigon.HttpxLoader(),
+        kabigon.SinglefileLoader(),
+    ]
+).load(url)
+print(content)
+```

kabigon-0.4.1.dist-info/RECORD

@@ -0,0 +1,19 @@
+kabigon/__init__.py,sha256=9RgyhhwjqrW2iQy9RBN2j7VZNhwA9xGo_atC7FKnZA4,545
+kabigon/cli.py,sha256=z3u2Msvi1SWf1fd9nCTzJULeO-rRb5oDKJfPxhUeYQ0,611
+kabigon/cloudscraper.py,sha256=0jzrXVXSZopExyxrDRbcI_2wsbHAg_dqOk4D3Re0jvk,404
+kabigon/compose.py,sha256=DO0hOJgEAX7ZLOS53dcE6V9zi7Tr9oGNW8koPHsx9eM,2110
+kabigon/httpx.py,sha256=B8_26rufJMbKSXINBEqyCIpaRueO_3Gk_PtEQmlOxQ4,955
+kabigon/loader.py,sha256=D5xUPJb3uAygmBaN_sX56ZpGcGsVz-ueHOXC7gSGaxM,493
+kabigon/pdf.py,sha256=oM5pwZJ2GCcHyQXg98-Mda-MHxarYVZQge30KdS_aHY,1549
+kabigon/playwright.py,sha256=ciNUlpMbwd47utCLT454wFSirXFmt3eCXN2Q-nAsiu8,1356
+kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+kabigon/reel.py,sha256=1JTcn7qVH7FcD0Oj-Rz-pnjI-xS1UtkoJcuClGb8ExQ,1124
+kabigon/singlefile.py,sha256=CeTT2WPYm0vb1xWPNdyBN4uHRw9hRqfZm68D-nEcUA8,1800
+kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
+kabigon/youtube.py,sha256=M1v7t4VS72ItqaNJTwlv59bafqORqldGjNzptZOeybA,2915
+kabigon/ytdlp.py,sha256=kG1fXqU650otOWespjOSkGK_-jk1wO-sWiR60_UPJxY,3125
+kabigon-0.4.1.dist-info/METADATA,sha256=IfnrNBC17ac0E4aI9Y-VOWhaiOnMN_RZxhoS-_EvhzA,1049
+kabigon-0.4.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kabigon-0.4.1.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
+kabigon-0.4.1.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
+kabigon-0.4.1.dist-info/RECORD,,

kabigon-0.3.1.dist-info/RECORD

@@ -1,19 +0,0 @@
-kabigon/__init__.py,sha256=9RgyhhwjqrW2iQy9RBN2j7VZNhwA9xGo_atC7FKnZA4,545
-kabigon/cli.py,sha256=7qHklIhYUZ4E78C9MIB16AoVIYt2xLJS0Pz8sr51YPk,690
-kabigon/cloudscraper.py,sha256=0jzrXVXSZopExyxrDRbcI_2wsbHAg_dqOk4D3Re0jvk,404
-kabigon/compose.py,sha256=IqNm-Cxl6e8u7X9v5SoG7cjxOWCMIcSZG1lDVNNAfo8,1433
-kabigon/httpx.py,sha256=ECNYGLOZ1uPz15ziABuziz8mQR3x1nF3cGcEOAsSmjo,668
-kabigon/loader.py,sha256=cV9ZqcWaNtS2WTDpgyNYK2kX5Cu1ZC-Sq-qS3PpPnJQ,198
-kabigon/pdf.py,sha256=oM5pwZJ2GCcHyQXg98-Mda-MHxarYVZQge30KdS_aHY,1549
-kabigon/playwright.py,sha256=g4cI3sIC_tVsk__x-1m-YA2LQs2_pWX8588rsVAmEOA,1095
-kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-kabigon/reel.py,sha256=dkWXG2nBhIt0DpGJzevkIrRKLqJh_03-yrg_rjf6vnY,828
-kabigon/singlefile.py,sha256=CeTT2WPYm0vb1xWPNdyBN4uHRw9hRqfZm68D-nEcUA8,1800
-kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
-kabigon/youtube.py,sha256=_wdKvRRAMrYnv3rUhkd_6JuOGCuQClYpj1UlVeYeojc,2615
-kabigon/ytdlp.py,sha256=kG1fXqU650otOWespjOSkGK_-jk1wO-sWiR60_UPJxY,3125
-kabigon-0.3.1.dist-info/METADATA,sha256=KWad9d5t-7NMX2xXhZWsQHpXlavwvEAkzDxNUO1mk8U,605
-kabigon-0.3.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kabigon-0.3.1.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
-kabigon-0.3.1.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
-kabigon-0.3.1.dist-info/RECORD,,