kabigon 0.4.2__py3-none-any.whl → 0.5.1__py3-none-any.whl

kabigon/__init__.py

@@ -4,7 +4,6 @@ from typing import Final
 
 from loguru import logger
 
-from .cloudscraper import CloudscraperLoader
 from .compose import Compose
 from .httpx import HttpxLoader
 from .loader import Loader

kabigon/cli.py

@@ -4,8 +4,8 @@ from rich import print
 from .compose import Compose
 from .httpx import HttpxLoader
 from .pdf import PDFLoader
+from .playwright import PlaywrightLoader
 from .reel import ReelLoader
-from .singlefile import SinglefileLoader
 from .youtube import YoutubeLoader
 from .ytdlp import YtdlpLoader
 
@@ -20,7 +20,7 @@ def main(url: str) -> None:
             YtdlpLoader(),
             PDFLoader(),
             HttpxLoader(),
-            SinglefileLoader(),
+            PlaywrightLoader(),
         ]
     )
     result = loader.load(url)

kabigon/httpx.py

@@ -1,5 +1,4 @@
 import httpx
-import timeout_decorator
 
 from .loader import Loader
 from .utils import html_to_markdown
@@ -12,7 +11,6 @@ DEFAULT_HEADERS = {
 
 
 class HttpxLoader(Loader):
-    @timeout_decorator.timeout(10)
     def load(self, url: str) -> str:
         response = httpx.get(url, headers=DEFAULT_HEADERS, follow_redirects=True)
         response.raise_for_status()

kabigon/pdf.py

@@ -1,8 +1,9 @@
-import tempfile
+import io
 from pathlib import Path
+from typing import IO
+from typing import Any
 
 import httpx
-import timeout_decorator
 from pypdf import PdfReader
 
 from .loader import Loader
@@ -15,38 +16,44 @@ DEFAULT_HEADERS = {
 
 
 class NotPDFError(LoaderError):
-    pass
+    def __init__(self, url: str) -> None:
+        super().__init__(f"URL is not a PDF: {url}")
 
 
 class PDFLoader(Loader):
-    @timeout_decorator.timeout(10)
     def load(self, url_or_file: str) -> str:
-        if url_or_file.startswith("http"):
-            url_or_file = download_pdf_from_url(url_or_file)
-        return read_pdf_content(url_or_file)
+        if not url_or_file.startswith("http"):
+            return read_pdf_content(url_or_file)
 
+        resp = httpx.get(url_or_file, headers=DEFAULT_HEADERS, follow_redirects=True)
+        resp.raise_for_status()
 
-def download_pdf_from_url(url: str) -> str:
-    response = httpx.get(url=url, headers=DEFAULT_HEADERS, follow_redirects=True)
-    response.raise_for_status()
+        if resp.headers.get("content-type") != "application/pdf":
+            raise NotPDFError(url_or_file)
 
-    is_pdf = response.headers.get("content-type") == "application/pdf"
-    if not is_pdf:
-        raise NotPDFError(f"URL is not a PDF: {url}")
+        return read_pdf_content(io.BytesIO(resp.content))
 
-    suffix = ".pdf" if is_pdf else None
-    with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as fp:
-        fp.write(response.content)
-        return fp.name
+    async def async_load(self, url_or_file: str) -> str:
+        if not url_or_file.startswith("http"):
+            return read_pdf_content(url_or_file)
 
+        async with httpx.AsyncClient() as client:
+            resp = await client.get(url_or_file, headers=DEFAULT_HEADERS, follow_redirects=True)
+            resp.raise_for_status()
 
-def read_pdf_content(f: str | Path) -> str:
+            if resp.headers.get("content-type") != "application/pdf":
+                raise NotPDFError(url_or_file)
+
+            return read_pdf_content(io.BytesIO(resp.content))
+
+
+def read_pdf_content(f: str | Path | IO[Any]) -> str:
     lines = []
     with PdfReader(f) as reader:
         for page in reader.pages:
             text = page.extract_text(extraction_mode="plain")
             for line in text.splitlines():
-                if not line.strip():
-                    continue
-                lines.append(line.strip())
+                stripped = line.strip()
+                if stripped:
+                    lines.append(stripped)
     return "\n".join(lines)

kabigon/playwright.py

@@ -1,5 +1,7 @@
 from typing import Literal
 
+from loguru import logger
+from playwright.async_api import TimeoutError
 from playwright.async_api import async_playwright
 from playwright.sync_api import sync_playwright
 
@@ -35,7 +37,11 @@ class PlaywrightLoader(Loader):
             browser = await p.chromium.launch(headless=self.browser_headless)
             page = await browser.new_page()
 
-            await page.goto(url, timeout=self.timeout, wait_until=self.wait_until)
+            try:
+                await page.goto(url, timeout=self.timeout, wait_until=self.wait_until)
+            except TimeoutError as e:
+                logger.error("Unable to load url: {}, got error: {}", url, e)
+                await page.goto(url)
 
             content = await page.content()
             await browser.close()

kabigon/reel.py

@@ -1,5 +1,3 @@
-import timeout_decorator
-
 from .httpx import HttpxLoader
 from .loader import Loader
 from .loader import LoaderError
@@ -20,7 +18,6 @@ class ReelLoader(Loader):
         self.httpx_loader = HttpxLoader()
         self.ytdlp_loader = YtdlpLoader()
 
-    @timeout_decorator.timeout(300)
     def load(self, url: str) -> str:
         if not is_reel_url(url):
             raise NotReelURLError(url)

{kabigon-0.4.2.dist-info → kabigon-0.5.1.dist-info}/METADATA

@@ -1,12 +1,11 @@
 Metadata-Version: 2.4
 Name: kabigon
-Version: 0.4.2
+Version: 0.5.1
 Author-email: narumi <toucans-cutouts0f@icloud.com>
 License-File: LICENSE
 Requires-Python: >=3.10
 Requires-Dist: aioytt>=0.2.4
 Requires-Dist: click>=8.1.8
-Requires-Dist: cloudscraper>=1.2.71
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: markdownify>=0.14.1
@@ -25,6 +24,7 @@ Description-Content-Type: text/markdown
 
 ```shell
 pip install kabigon
+playwright install chromium
 ```
 
 ## Usage
@@ -46,8 +46,8 @@ content = kabigon.Compose(
         kabigon.ReelLoader(),
         kabigon.YtdlpLoader(),
         kabigon.PDFLoader(),
-        kabigon.HttpxLoader(),
-        kabigon.SinglefileLoader(),
+        # kabigon.HttpxLoader(),
+        kabigon.PlaywrightLoader(),
     ]
 ).load(url)
 print(content)

kabigon-0.5.1.dist-info/RECORD

@@ -0,0 +1,18 @@
+kabigon/__init__.py,sha256=7ll3ePlHNbZq-CmrGMrQouLCUSmuRsZ9yAj2JOzr7HY,500
+kabigon/cli.py,sha256=XSTyD1RFqq2Qok_52kSjJlBLUXl6t-K9QtsxCfB15o4,611
+kabigon/compose.py,sha256=DO0hOJgEAX7ZLOS53dcE6V9zi7Tr9oGNW8koPHsx9eM,2110
+kabigon/httpx.py,sha256=uDdLks6zVzirY7-mnsJkypX86kAI5XmUVfK-lFifdJA,895
+kabigon/loader.py,sha256=D5xUPJb3uAygmBaN_sX56ZpGcGsVz-ueHOXC7gSGaxM,493
+kabigon/pdf.py,sha256=Q9XuBdKDrDQJ8BNvY7Lgt6dpGeA_ylGGHWOE3euiI_8,1904
+kabigon/playwright.py,sha256=R-NCW9sJs3O-asKk_iSxBI8pg15XqgQ-hEXMTr1F2q0,1602
+kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+kabigon/reel.py,sha256=TP_oKYXABXYja2A9damTBWR3MVYA7aZyxbIvCuTcq40,1062
+kabigon/singlefile.py,sha256=CeTT2WPYm0vb1xWPNdyBN4uHRw9hRqfZm68D-nEcUA8,1800
+kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
+kabigon/youtube.py,sha256=HoiFNq0ookPL7_rO_wloBaY8yTIX6xP8A77F7y02q64,1166
+kabigon/ytdlp.py,sha256=kG1fXqU650otOWespjOSkGK_-jk1wO-sWiR60_UPJxY,3125
+kabigon-0.5.1.dist-info/METADATA,sha256=3YUnj7TzdYOAfMV4X9vHwO1HMQrM8iIEDyTDqMdnDFM,1043
+kabigon-0.5.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kabigon-0.5.1.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
+kabigon-0.5.1.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
+kabigon-0.5.1.dist-info/RECORD,,

kabigon/cloudscraper.py

@@ -1,14 +0,0 @@
-import cloudscraper
-import timeout_decorator
-
-from .loader import Loader
-from .utils import html_to_markdown
-
-
-class CloudscraperLoader(Loader):
-    @timeout_decorator.timeout(10)
-    def load(self, url: str) -> str:
-        client = cloudscraper.create_scraper()
-        response = client.get(url, allow_redirects=True)
-        response.raise_for_status()
-        return html_to_markdown(response.text)

kabigon-0.4.2.dist-info/RECORD

@@ -1,19 +0,0 @@
-kabigon/__init__.py,sha256=9RgyhhwjqrW2iQy9RBN2j7VZNhwA9xGo_atC7FKnZA4,545
-kabigon/cli.py,sha256=z3u2Msvi1SWf1fd9nCTzJULeO-rRb5oDKJfPxhUeYQ0,611
-kabigon/cloudscraper.py,sha256=0jzrXVXSZopExyxrDRbcI_2wsbHAg_dqOk4D3Re0jvk,404
-kabigon/compose.py,sha256=DO0hOJgEAX7ZLOS53dcE6V9zi7Tr9oGNW8koPHsx9eM,2110
-kabigon/httpx.py,sha256=B8_26rufJMbKSXINBEqyCIpaRueO_3Gk_PtEQmlOxQ4,955
-kabigon/loader.py,sha256=D5xUPJb3uAygmBaN_sX56ZpGcGsVz-ueHOXC7gSGaxM,493
-kabigon/pdf.py,sha256=oM5pwZJ2GCcHyQXg98-Mda-MHxarYVZQge30KdS_aHY,1549
-kabigon/playwright.py,sha256=ciNUlpMbwd47utCLT454wFSirXFmt3eCXN2Q-nAsiu8,1356
-kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-kabigon/reel.py,sha256=1JTcn7qVH7FcD0Oj-Rz-pnjI-xS1UtkoJcuClGb8ExQ,1124
-kabigon/singlefile.py,sha256=CeTT2WPYm0vb1xWPNdyBN4uHRw9hRqfZm68D-nEcUA8,1800
-kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
-kabigon/youtube.py,sha256=HoiFNq0ookPL7_rO_wloBaY8yTIX6xP8A77F7y02q64,1166
-kabigon/ytdlp.py,sha256=kG1fXqU650otOWespjOSkGK_-jk1wO-sWiR60_UPJxY,3125
-kabigon-0.4.2.dist-info/METADATA,sha256=JHbf13Nnhr05WfSS1hijT-YkeCewuWr5kYbzxjlJ-M8,1049
-kabigon-0.4.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kabigon-0.4.2.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
-kabigon-0.4.2.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
-kabigon-0.4.2.dist-info/RECORD,,