kabigon 0.8.11__py3-none-any.whl → 0.8.13__py3-none-any.whl

kabigon/playwright.py

@@ -1,5 +1,7 @@
 from typing import Literal
 
+from loguru import logger
+
 from .loader import Loader
 from .utils import html_to_markdown
 
@@ -17,6 +19,7 @@ class PlaywrightLoader(Loader):
 
     def load(self, url: str) -> str:
         try:
+            from playwright.sync_api import TimeoutError
             from playwright.sync_api import sync_playwright
         except ImportError as e:
             raise ImportError(
@@ -29,7 +32,10 @@ class PlaywrightLoader(Loader):
             context = browser.new_context()
             page = context.new_page()
 
-            page.goto(url, timeout=self.timeout, wait_until=self.wait_until)
+            try:
+                page.goto(url, timeout=self.timeout, wait_until=self.wait_until)
+            except TimeoutError as e:
+                logger.warning("TimeoutError: {}, (url: {}, timeout: {})", e, url, self.timeout)
 
             content = page.content()
             browser.close()
@@ -38,6 +44,7 @@ class PlaywrightLoader(Loader):
 
     async def async_load(self, url: str) -> str:
         try:
+            from playwright.async_api import TimeoutError
             from playwright.async_api import async_playwright
         except ImportError as e:
             raise ImportError(
@@ -50,7 +57,10 @@ class PlaywrightLoader(Loader):
             context = await browser.new_context()
             page = await context.new_page()
 
-            await page.goto(url, timeout=self.timeout, wait_until=self.wait_until)
+            try:
+                await page.goto(url, timeout=self.timeout, wait_until=self.wait_until)
+            except TimeoutError as e:
+                logger.warning("TimeoutError: {}, (url: {}, timeout: {})", e, url, self.timeout)
 
             content = await page.content()
             await browser.close()

kabigon/twitter.py

@@ -17,7 +17,7 @@ TWITTER_DOMAINS = [
 
 
 def replace_domain(url: str, new_domain: str = "x.com") -> str:
-    return urlunparse(urlparse(url)._replace(netloc=new_domain))
+    return str(urlunparse(urlparse(url)._replace(netloc=new_domain)))
 
 
 def check_x_url(url: str) -> None:
@@ -27,7 +27,7 @@ def check_x_url(url: str) -> None:
 
 class TwitterLoader(Loader):
     def __init__(self) -> None:
-        self.playwright_loader = PlaywrightLoader(wait_until="networkidle")
+        self.playwright_loader = PlaywrightLoader(wait_until="networkidle", timeout=10_000)
 
     def load(self, url: str) -> str:
         check_x_url(url)

kabigon/youtube.py

@@ -14,13 +14,11 @@ class YoutubeLoader(Loader):
     def load(self, url: str) -> str:
         video_id = aioytt.video_id.parse_video_id(url)
 
-        transcript_pieces: list[dict[str, str | float]] = YouTubeTranscriptApi().get_transcript(
-            video_id, self.languages
-        )
+        fetched = YouTubeTranscriptApi().fetch(video_id, self.languages)
 
         lines = []
-        for transcript_piece in transcript_pieces:
-            text = str(transcript_piece.get("text", "")).strip()
+        for snippet in fetched.snippets:
+            text = str(snippet.text).strip()
             if text:
                 lines.append(text)
         return "\n".join(lines)

{kabigon-0.8.11.dist-info → kabigon-0.8.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kabigon
-Version: 0.8.11
+Version: 0.8.13
 Author-email: narumi <toucans-cutouts0f@icloud.com>
 License-File: LICENSE
 Requires-Python: >=3.10
@@ -12,7 +12,7 @@ Requires-Dist: markdownify>=0.14.1
 Requires-Dist: pypdf>=5.3.0
 Requires-Dist: rich>=13.9.4
 Requires-Dist: typer>=0.15.3
-Requires-Dist: youtube-transcript-api>=0.6.3
+Requires-Dist: youtube-transcript-api>=1.2.2
 Requires-Dist: yt-dlp>=2025.4.30
 Provides-Extra: all
 Requires-Dist: openai-whisper>=20240930; extra == 'all'

{kabigon-0.8.11.dist-info → kabigon-0.8.13.dist-info}/RECORD

@@ -5,17 +5,17 @@ kabigon/firecrawl.py,sha256=-5AI9tla_684dtpubY_BRudqLgw28158WdwA1RjJvAA,778
 kabigon/httpx.py,sha256=Zup9DURyWLqoWzaxBbCYAaV-5LSlHUuAcNyyUsZTVag,696
 kabigon/loader.py,sha256=W2ZdSINQKjG-5XNYweM8lIl19JA4wwd7AE2pueHeO70,380
 kabigon/pdf.py,sha256=PNOx-Dz_VpN-EVnVar_wJJZdxWrDZrAFE-gSuUR9q7o,1870
-kabigon/playwright.py,sha256=etzxzAmwpATnB_ckxGU3IE4jgKBdlFDwtP1fN0u4Pr8,1959
+kabigon/playwright.py,sha256=gECaDJhsIrY53lwG7OrH8gUHvIsVe7p5jgBTUy7KPJg,2413
 kabigon/ptt.py,sha256=Gyp2nJrjptkjbwZJ9VEQHX0DEgKBe5QRQOmGVHUUgNA,896
 kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 kabigon/reel.py,sha256=qOwWCvcp7xNKg0JDunq_Bsl8yqqMzrnAOI9k5mSqrOU,874
-kabigon/twitter.py,sha256=aRqAiFxIwln6lteWdoF6SmvbzO62yBTQRzcB7UcVJwk,1046
+kabigon/twitter.py,sha256=-EcwbvgIEXXsGqDHaVZQ-VKJHbxhmT8FFn8bGTIdXiw,1067
 kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
-kabigon/youtube.py,sha256=HKvMqQ5x6qGqao1QHsy9e5f8J2TKE3bsRv-vlSipmvE,1122
+kabigon/youtube.py,sha256=Vh4cBidScUruTgvm4BbTxqaEDxnotMI3uM3FuFlx5J0,1021
 kabigon/youtube_ytdlp.py,sha256=Y6h55wYF-5PdxLRxsznFtJDypCedD8g-hY9fgCnfvbg,722
 kabigon/ytdlp.py,sha256=4AjPY0W8IxbEhCNVXpi_KbBQLGrWn5ThrchhffcJIIU,1718
-kabigon-0.8.11.dist-info/METADATA,sha256=6K2_uFhQVCpWiFnTJE7bHSEe4CR5jD4NS4cwl_fKnQ0,1265
-kabigon-0.8.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kabigon-0.8.11.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
-kabigon-0.8.11.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
-kabigon-0.8.11.dist-info/RECORD,,
+kabigon-0.8.13.dist-info/METADATA,sha256=37MjwEQKzcXbHdsF3mfIFePL3rN0n7xUJenVWkivz6w,1265
+kabigon-0.8.13.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kabigon-0.8.13.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
+kabigon-0.8.13.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
+kabigon-0.8.13.dist-info/RECORD,,