kabigon 0.4.2__py3-none-any.whl → 0.5.0__py3-none-any.whl

kabigon/__init__.py

@@ -4,7 +4,6 @@ from typing import Final
 
 from loguru import logger
 
-from .cloudscraper import CloudscraperLoader
 from .compose import Compose
 from .httpx import HttpxLoader
 from .loader import Loader

kabigon/httpx.py

@@ -1,5 +1,4 @@
 import httpx
-import timeout_decorator
 
 from .loader import Loader
 from .utils import html_to_markdown
@@ -12,7 +11,6 @@ DEFAULT_HEADERS = {
 
 
 class HttpxLoader(Loader):
-    @timeout_decorator.timeout(10)
     def load(self, url: str) -> str:
         response = httpx.get(url, headers=DEFAULT_HEADERS, follow_redirects=True)
         response.raise_for_status()

kabigon/pdf.py

@@ -1,8 +1,9 @@
-import tempfile
+import io
 from pathlib import Path
+from typing import IO
+from typing import Any
 
 import httpx
-import timeout_decorator
 from pypdf import PdfReader
 
 from .loader import Loader
@@ -15,38 +16,44 @@ DEFAULT_HEADERS = {
 
 
 class NotPDFError(LoaderError):
-    pass
+    def __init__(self, url: str) -> None:
+        super().__init__(f"URL is not a PDF: {url}")
 
 
 class PDFLoader(Loader):
-    @timeout_decorator.timeout(10)
     def load(self, url_or_file: str) -> str:
-        if url_or_file.startswith("http"):
-            url_or_file = download_pdf_from_url(url_or_file)
-        return read_pdf_content(url_or_file)
+        if not url_or_file.startswith("http"):
+            return read_pdf_content(url_or_file)
 
+        resp = httpx.get(url_or_file, headers=DEFAULT_HEADERS, follow_redirects=True)
+        resp.raise_for_status()
 
-def download_pdf_from_url(url: str) -> str:
-    response = httpx.get(url=url, headers=DEFAULT_HEADERS, follow_redirects=True)
-    response.raise_for_status()
+        if resp.headers.get("content-type") != "application/pdf":
+            raise NotPDFError(url_or_file)
 
-    is_pdf = response.headers.get("content-type") == "application/pdf"
-    if not is_pdf:
-        raise NotPDFError(f"URL is not a PDF: {url}")
+        return read_pdf_content(io.BytesIO(resp.content))
 
-    suffix = ".pdf" if is_pdf else None
-    with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as fp:
-        fp.write(response.content)
-        return fp.name
+    async def async_load(self, url_or_file: str) -> str:
+        if not url_or_file.startswith("http"):
+            return read_pdf_content(url_or_file)
 
+        async with httpx.AsyncClient() as client:
+            resp = await client.get(url_or_file, headers=DEFAULT_HEADERS, follow_redirects=True)
+            resp.raise_for_status()
 
-def read_pdf_content(f: str | Path) -> str:
+            if resp.headers.get("content-type") != "application/pdf":
+                raise NotPDFError(url_or_file)
+
+            return read_pdf_content(io.BytesIO(resp.content))
+
+
+def read_pdf_content(f: str | Path | IO[Any]) -> str:
     lines = []
     with PdfReader(f) as reader:
         for page in reader.pages:
             text = page.extract_text(extraction_mode="plain")
             for line in text.splitlines():
-                if not line.strip():
-                    continue
-                lines.append(line.strip())
+                stripped = line.strip()
+                if stripped:
+                    lines.append(stripped)
     return "\n".join(lines)

kabigon/reel.py

@@ -1,5 +1,3 @@
-import timeout_decorator
-
 from .httpx import HttpxLoader
 from .loader import Loader
 from .loader import LoaderError
@@ -20,7 +18,6 @@ class ReelLoader(Loader):
         self.httpx_loader = HttpxLoader()
         self.ytdlp_loader = YtdlpLoader()
 
-    @timeout_decorator.timeout(300)
     def load(self, url: str) -> str:
         if not is_reel_url(url):
             raise NotReelURLError(url)

{kabigon-0.4.2.dist-info → kabigon-0.5.0.dist-info}/METADATA

@@ -1,12 +1,11 @@
 Metadata-Version: 2.4
 Name: kabigon
-Version: 0.4.2
+Version: 0.5.0
 Author-email: narumi <toucans-cutouts0f@icloud.com>
 License-File: LICENSE
 Requires-Python: >=3.10
 Requires-Dist: aioytt>=0.2.4
 Requires-Dist: click>=8.1.8
-Requires-Dist: cloudscraper>=1.2.71
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: markdownify>=0.14.1

{kabigon-0.4.2.dist-info → kabigon-0.5.0.dist-info}/RECORD

@@ -1,19 +1,18 @@
-kabigon/__init__.py,sha256=9RgyhhwjqrW2iQy9RBN2j7VZNhwA9xGo_atC7FKnZA4,545
+kabigon/__init__.py,sha256=7ll3ePlHNbZq-CmrGMrQouLCUSmuRsZ9yAj2JOzr7HY,500
 kabigon/cli.py,sha256=z3u2Msvi1SWf1fd9nCTzJULeO-rRb5oDKJfPxhUeYQ0,611
-kabigon/cloudscraper.py,sha256=0jzrXVXSZopExyxrDRbcI_2wsbHAg_dqOk4D3Re0jvk,404
 kabigon/compose.py,sha256=DO0hOJgEAX7ZLOS53dcE6V9zi7Tr9oGNW8koPHsx9eM,2110
-kabigon/httpx.py,sha256=B8_26rufJMbKSXINBEqyCIpaRueO_3Gk_PtEQmlOxQ4,955
+kabigon/httpx.py,sha256=uDdLks6zVzirY7-mnsJkypX86kAI5XmUVfK-lFifdJA,895
 kabigon/loader.py,sha256=D5xUPJb3uAygmBaN_sX56ZpGcGsVz-ueHOXC7gSGaxM,493
-kabigon/pdf.py,sha256=oM5pwZJ2GCcHyQXg98-Mda-MHxarYVZQge30KdS_aHY,1549
+kabigon/pdf.py,sha256=Q9XuBdKDrDQJ8BNvY7Lgt6dpGeA_ylGGHWOE3euiI_8,1904
 kabigon/playwright.py,sha256=ciNUlpMbwd47utCLT454wFSirXFmt3eCXN2Q-nAsiu8,1356
 kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-kabigon/reel.py,sha256=1JTcn7qVH7FcD0Oj-Rz-pnjI-xS1UtkoJcuClGb8ExQ,1124
+kabigon/reel.py,sha256=TP_oKYXABXYja2A9damTBWR3MVYA7aZyxbIvCuTcq40,1062
 kabigon/singlefile.py,sha256=CeTT2WPYm0vb1xWPNdyBN4uHRw9hRqfZm68D-nEcUA8,1800
 kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
 kabigon/youtube.py,sha256=HoiFNq0ookPL7_rO_wloBaY8yTIX6xP8A77F7y02q64,1166
 kabigon/ytdlp.py,sha256=kG1fXqU650otOWespjOSkGK_-jk1wO-sWiR60_UPJxY,3125
-kabigon-0.4.2.dist-info/METADATA,sha256=JHbf13Nnhr05WfSS1hijT-YkeCewuWr5kYbzxjlJ-M8,1049
-kabigon-0.4.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kabigon-0.4.2.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
-kabigon-0.4.2.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
-kabigon-0.4.2.dist-info/RECORD,,
+kabigon-0.5.0.dist-info/METADATA,sha256=TVL8rTtEdjtIkjl7L9pC7vrv3-VOx6sYBafeXVUlSN8,1013
+kabigon-0.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kabigon-0.5.0.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
+kabigon-0.5.0.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
+kabigon-0.5.0.dist-info/RECORD,,

kabigon/cloudscraper.py

@@ -1,14 +0,0 @@
-import cloudscraper
-import timeout_decorator
-
-from .loader import Loader
-from .utils import html_to_markdown
-
-
-class CloudscraperLoader(Loader):
-    @timeout_decorator.timeout(10)
-    def load(self, url: str) -> str:
-        client = cloudscraper.create_scraper()
-        response = client.get(url, allow_redirects=True)
-        response.raise_for_status()
-        return html_to_markdown(response.text)