kabigon 0.4.1__py3-none-any.whl → 0.5.0__py3-none-any.whl

kabigon/__init__.py

@@ -4,7 +4,6 @@ from typing import Final
 
 from loguru import logger
 
-from .cloudscraper import CloudscraperLoader
 from .compose import Compose
 from .httpx import HttpxLoader
 from .loader import Loader

kabigon/httpx.py

@@ -1,5 +1,4 @@
 import httpx
-import timeout_decorator
 
 from .loader import Loader
 from .utils import html_to_markdown
@@ -12,7 +11,6 @@ DEFAULT_HEADERS = {
 
 
 class HttpxLoader(Loader):
-    @timeout_decorator.timeout(10)
     def load(self, url: str) -> str:
         response = httpx.get(url, headers=DEFAULT_HEADERS, follow_redirects=True)
         response.raise_for_status()

kabigon/pdf.py

@@ -1,8 +1,9 @@
-import tempfile
+import io
 from pathlib import Path
+from typing import IO
+from typing import Any
 
 import httpx
-import timeout_decorator
 from pypdf import PdfReader
 
 from .loader import Loader
@@ -15,38 +16,44 @@ DEFAULT_HEADERS = {
 
 
 class NotPDFError(LoaderError):
-    pass
+    def __init__(self, url: str) -> None:
+        super().__init__(f"URL is not a PDF: {url}")
 
 
 class PDFLoader(Loader):
-    @timeout_decorator.timeout(10)
     def load(self, url_or_file: str) -> str:
-        if url_or_file.startswith("http"):
-            url_or_file = download_pdf_from_url(url_or_file)
-        return read_pdf_content(url_or_file)
+        if not url_or_file.startswith("http"):
+            return read_pdf_content(url_or_file)
 
+        resp = httpx.get(url_or_file, headers=DEFAULT_HEADERS, follow_redirects=True)
+        resp.raise_for_status()
 
-def download_pdf_from_url(url: str) -> str:
-    response = httpx.get(url=url, headers=DEFAULT_HEADERS, follow_redirects=True)
-    response.raise_for_status()
+        if resp.headers.get("content-type") != "application/pdf":
+            raise NotPDFError(url_or_file)
 
-    is_pdf = response.headers.get("content-type") == "application/pdf"
-    if not is_pdf:
-        raise NotPDFError(f"URL is not a PDF: {url}")
+        return read_pdf_content(io.BytesIO(resp.content))
 
-    suffix = ".pdf" if is_pdf else None
-    with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as fp:
-        fp.write(response.content)
-        return fp.name
+    async def async_load(self, url_or_file: str) -> str:
+        if not url_or_file.startswith("http"):
+            return read_pdf_content(url_or_file)
 
+        async with httpx.AsyncClient() as client:
+            resp = await client.get(url_or_file, headers=DEFAULT_HEADERS, follow_redirects=True)
+            resp.raise_for_status()
 
-def read_pdf_content(f: str | Path) -> str:
+            if resp.headers.get("content-type") != "application/pdf":
+                raise NotPDFError(url_or_file)
+
+            return read_pdf_content(io.BytesIO(resp.content))
+
+
+def read_pdf_content(f: str | Path | IO[Any]) -> str:
     lines = []
     with PdfReader(f) as reader:
         for page in reader.pages:
             text = page.extract_text(extraction_mode="plain")
             for line in text.splitlines():
-                if not line.strip():
-                    continue
-                lines.append(line.strip())
+                stripped = line.strip()
+                if stripped:
+                    lines.append(stripped)
     return "\n".join(lines)

kabigon/reel.py

@@ -1,5 +1,3 @@
-import timeout_decorator
-
 from .httpx import HttpxLoader
 from .loader import Loader
 from .loader import LoaderError
@@ -20,7 +18,6 @@ class ReelLoader(Loader):
         self.httpx_loader = HttpxLoader()
         self.ytdlp_loader = YtdlpLoader()
 
-    @timeout_decorator.timeout(300)
     def load(self, url: str) -> str:
         if not is_reel_url(url):
             raise NotReelURLError(url)

kabigon/youtube.py

@@ -1,76 +1,11 @@
-from urllib.parse import parse_qs
-from urllib.parse import urlparse
-
 import aioytt
+import aioytt.video_id
 import timeout_decorator
 from youtube_transcript_api import YouTubeTranscriptApi
 
 from .loader import Loader
-from .loader import LoaderError
 
 DEFAULT_LANGUAGES = ["zh-TW", "zh-Hant", "zh", "zh-Hans", "ja", "en", "ko"]
-ALLOWED_SCHEMES = {
-    "http",
-    "https",
-}
-ALLOWED_NETLOCS = {
-    "youtu.be",
-    "m.youtube.com",
-    "youtube.com",
-    "www.youtube.com",
-    "www.youtube-nocookie.com",
-    "vid.plus",
-}
-
-
-class UnsupportedURLSchemeError(LoaderError):
-    def __init__(self, scheme: str) -> None:
-        super().__init__(f"unsupported URL scheme: {scheme}")
-
-
-class UnsupportedURLNetlocError(LoaderError):
-    def __init__(self, netloc: str) -> None:
-        super().__init__(f"unsupported URL netloc: {netloc}")
-
-
-class VideoIDError(LoaderError):
-    def __init__(self, video_id: str) -> None:
-        super().__init__(f"invalid video ID: {video_id}")
-
-
-class NoVideoIDFoundError(LoaderError):
-    def __init__(self, url: str) -> None:
-        super().__init__(f"no video found in URL: {url}")
-
-
-def parse_video_id(url: str) -> str:
-    """Parse a YouTube URL and return the video ID if valid, otherwise None."""
-    parsed_url = urlparse(url)
-
-    if parsed_url.scheme not in ALLOWED_SCHEMES:
-        raise UnsupportedURLSchemeError(parsed_url.scheme)
-
-    if parsed_url.netloc not in ALLOWED_NETLOCS:
-        raise UnsupportedURLNetlocError(parsed_url.netloc)
-
-    path = parsed_url.path
-
-    if path.endswith("/watch"):
-        query = parsed_url.query
-        parsed_query = parse_qs(query)
-        if "v" in parsed_query:
-            ids = parsed_query["v"]
-            video_id = ids if isinstance(ids, str) else ids[0]
-        else:
-            raise NoVideoIDFoundError(url)
-    else:
-        path = parsed_url.path.lstrip("/")
-        video_id = path.split("/")[-1]
-
-    if len(video_id) != 11:  # Video IDs are 11 characters long
-        raise VideoIDError(video_id)
-
-    return video_id
 
 
 class YoutubeLoader(Loader):
@@ -79,7 +14,7 @@ class YoutubeLoader(Loader):
 
     @timeout_decorator.timeout(20)
     def load(self, url: str) -> str:
-        video_id = parse_video_id(url)
+        video_id = aioytt.video_id.parse_video_id(url)
 
         transcript_pieces: list[dict[str, str | float]] = YouTubeTranscriptApi().get_transcript(
             video_id, self.languages
@@ -98,5 +33,5 @@ class YoutubeLoader(Loader):
         for piece in transcript:
             text = piece.text.strip()
             if text:
-                lines += text
+                lines.append(text)
         return "\n".join(lines)

{kabigon-0.4.1.dist-info → kabigon-0.5.0.dist-info}/METADATA

@@ -1,12 +1,11 @@
 Metadata-Version: 2.4
 Name: kabigon
-Version: 0.4.1
+Version: 0.5.0
 Author-email: narumi <toucans-cutouts0f@icloud.com>
 License-File: LICENSE
 Requires-Python: >=3.10
 Requires-Dist: aioytt>=0.2.4
 Requires-Dist: click>=8.1.8
-Requires-Dist: cloudscraper>=1.2.71
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: markdownify>=0.14.1

kabigon-0.5.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+kabigon/__init__.py,sha256=7ll3ePlHNbZq-CmrGMrQouLCUSmuRsZ9yAj2JOzr7HY,500
+kabigon/cli.py,sha256=z3u2Msvi1SWf1fd9nCTzJULeO-rRb5oDKJfPxhUeYQ0,611
+kabigon/compose.py,sha256=DO0hOJgEAX7ZLOS53dcE6V9zi7Tr9oGNW8koPHsx9eM,2110
+kabigon/httpx.py,sha256=uDdLks6zVzirY7-mnsJkypX86kAI5XmUVfK-lFifdJA,895
+kabigon/loader.py,sha256=D5xUPJb3uAygmBaN_sX56ZpGcGsVz-ueHOXC7gSGaxM,493
+kabigon/pdf.py,sha256=Q9XuBdKDrDQJ8BNvY7Lgt6dpGeA_ylGGHWOE3euiI_8,1904
+kabigon/playwright.py,sha256=ciNUlpMbwd47utCLT454wFSirXFmt3eCXN2Q-nAsiu8,1356
+kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+kabigon/reel.py,sha256=TP_oKYXABXYja2A9damTBWR3MVYA7aZyxbIvCuTcq40,1062
+kabigon/singlefile.py,sha256=CeTT2WPYm0vb1xWPNdyBN4uHRw9hRqfZm68D-nEcUA8,1800
+kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
+kabigon/youtube.py,sha256=HoiFNq0ookPL7_rO_wloBaY8yTIX6xP8A77F7y02q64,1166
+kabigon/ytdlp.py,sha256=kG1fXqU650otOWespjOSkGK_-jk1wO-sWiR60_UPJxY,3125
+kabigon-0.5.0.dist-info/METADATA,sha256=TVL8rTtEdjtIkjl7L9pC7vrv3-VOx6sYBafeXVUlSN8,1013
+kabigon-0.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kabigon-0.5.0.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
+kabigon-0.5.0.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
+kabigon-0.5.0.dist-info/RECORD,,

kabigon/cloudscraper.py

@@ -1,14 +0,0 @@
-import cloudscraper
-import timeout_decorator
-
-from .loader import Loader
-from .utils import html_to_markdown
-
-
-class CloudscraperLoader(Loader):
-    @timeout_decorator.timeout(10)
-    def load(self, url: str) -> str:
-        client = cloudscraper.create_scraper()
-        response = client.get(url, allow_redirects=True)
-        response.raise_for_status()
-        return html_to_markdown(response.text)

kabigon-0.4.1.dist-info/RECORD

@@ -1,19 +0,0 @@
-kabigon/__init__.py,sha256=9RgyhhwjqrW2iQy9RBN2j7VZNhwA9xGo_atC7FKnZA4,545
-kabigon/cli.py,sha256=z3u2Msvi1SWf1fd9nCTzJULeO-rRb5oDKJfPxhUeYQ0,611
-kabigon/cloudscraper.py,sha256=0jzrXVXSZopExyxrDRbcI_2wsbHAg_dqOk4D3Re0jvk,404
-kabigon/compose.py,sha256=DO0hOJgEAX7ZLOS53dcE6V9zi7Tr9oGNW8koPHsx9eM,2110
-kabigon/httpx.py,sha256=B8_26rufJMbKSXINBEqyCIpaRueO_3Gk_PtEQmlOxQ4,955
-kabigon/loader.py,sha256=D5xUPJb3uAygmBaN_sX56ZpGcGsVz-ueHOXC7gSGaxM,493
-kabigon/pdf.py,sha256=oM5pwZJ2GCcHyQXg98-Mda-MHxarYVZQge30KdS_aHY,1549
-kabigon/playwright.py,sha256=ciNUlpMbwd47utCLT454wFSirXFmt3eCXN2Q-nAsiu8,1356
-kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-kabigon/reel.py,sha256=1JTcn7qVH7FcD0Oj-Rz-pnjI-xS1UtkoJcuClGb8ExQ,1124
-kabigon/singlefile.py,sha256=CeTT2WPYm0vb1xWPNdyBN4uHRw9hRqfZm68D-nEcUA8,1800
-kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
-kabigon/youtube.py,sha256=M1v7t4VS72ItqaNJTwlv59bafqORqldGjNzptZOeybA,2915
-kabigon/ytdlp.py,sha256=kG1fXqU650otOWespjOSkGK_-jk1wO-sWiR60_UPJxY,3125
-kabigon-0.4.1.dist-info/METADATA,sha256=IfnrNBC17ac0E4aI9Y-VOWhaiOnMN_RZxhoS-_EvhzA,1049
-kabigon-0.4.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kabigon-0.4.1.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
-kabigon-0.4.1.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
-kabigon-0.4.1.dist-info/RECORD,,