kabigon 0.6.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

kabigon/__init__.py

@@ -11,6 +11,7 @@ from .loader import Loader
 from .pdf import PDFLoader
 from .playwright import PlaywrightLoader
 from .reel import ReelLoader
+from .twitter import TwitterLoader
 from .youtube import YoutubeLoader
 from .ytdlp import YtdlpLoader
 

kabigon/cli.py

@@ -6,6 +6,7 @@ from .httpx import HttpxLoader
 from .pdf import PDFLoader
 from .playwright import PlaywrightLoader
 from .reel import ReelLoader
+from .twitter import TwitterLoader
 from .youtube import YoutubeLoader
 from .ytdlp import YtdlpLoader
 
@@ -15,6 +16,7 @@ from .ytdlp import YtdlpLoader
 def main(url: str) -> None:
     loader = Compose(
         [
+            TwitterLoader(),
             YoutubeLoader(),
             ReelLoader(),
             YtdlpLoader(),

kabigon/compose.py

@@ -1,41 +1,14 @@
-from urllib.parse import urlparse
-from urllib.parse import urlunparse
-
 from loguru import logger
 
 from .errors import KabigonError
 from .loader import Loader
 
-REPLACEMENTS = {
-    # fixupx.com seems better than api.fxtwitter.com
-    "fixupx.com": [
-        "twitter.com",
-        "x.com",
-        "fxtwitter.com",
-        "vxtwitter.com",
-        "fixvx.com",
-        "twittpr.com",
-        "api.fxtwitter.com",
-    ]
-}
-
-
-def replace_domain(url: str) -> str:
-    parsed = urlparse(url)
-    for target, source in REPLACEMENTS.items():
-        if parsed.netloc in source:
-            fixed_url = parsed._replace(netloc=target)
-            return urlunparse(fixed_url)
-    return url
-
 
 class Compose(Loader):
     def __init__(self, loaders: list[Loader]) -> None:
         self.loaders = loaders
 
     def load(self, url: str) -> str:
-        url = replace_domain(url)
-
         for loader in self.loaders:
             try:
                 content = loader.load(url)
@@ -53,8 +26,6 @@ class Compose(Loader):
         raise KabigonError(f"Failed to load URL: {url}")
 
     async def async_load(self, url: str) -> str:
-        url = replace_domain(url)
-
         for loader in self.loaders:
             try:
                 content = await loader.async_load(url)

kabigon/errors.py

@@ -21,3 +21,8 @@ class NotPDFError(KabigonError):
 class NotReelURLError(KabigonError):
     def __init__(self, url: str):
         super().__init__(f"URL is not an Instagram Reel: {url}")
+
+
+class NotTwitterURLError(KabigonError):
+    def __init__(self, url: str):
+        super().__init__(f"URL is not a Twitter URL: {url}")

kabigon/twitter.py

@@ -0,0 +1,46 @@
+from urllib.parse import urlparse
+from urllib.parse import urlunparse
+
+from .errors import NotTwitterURLError
+from .loader import Loader
+from .playwright import PlaywrightLoader
+
+TWITTER_DOMAINS = [
+    "twitter.com",
+    "x.com",
+    "fxtwitter.com",
+    "vxtwitter.com",
+    "fixvx.com",
+    "twittpr.com",
+    "api.fxtwitter.com",
+    "fixupx.com",
+]
+
+
+def replace_domain(url: str, new_domain: str = "x.com") -> str:
+    return urlunparse(urlparse(url)._replace(netloc=new_domain))
+
+
+def is_x_url(url: str) -> bool:
+    return urlparse(url).netloc in TWITTER_DOMAINS
+
+
+class TwitterLoader(Loader):
+    def __init__(self) -> None:
+        self.playwright_loader = PlaywrightLoader(wait_until="networkidle")
+
+    def load(self, url: str) -> str:
+        if not is_x_url(url):
+            raise NotTwitterURLError(url)
+
+        url = replace_domain(url)
+
+        return self.playwright_loader.load(url)
+
+    async def async_load(self, url: str):
+        if not is_x_url(url):
+            raise NotTwitterURLError(url)
+
+        url = replace_domain(url)
+
+        return await self.playwright_loader.async_load(url)

{kabigon-0.6.1.dist-info → kabigon-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kabigon
-Version: 0.6.1
+Version: 0.7.0
 Author-email: narumi <toucans-cutouts0f@icloud.com>
 License-File: LICENSE
 Requires-Python: >=3.10
@@ -11,12 +11,12 @@ Requires-Dist: httpx>=0.28.1
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: markdownify>=0.14.1
 Requires-Dist: openai-whisper>=20240930
-Requires-Dist: playwright>=1.50.0
+Requires-Dist: playwright>=1.52.0
 Requires-Dist: pypdf>=5.3.0
 Requires-Dist: rich>=13.9.4
 Requires-Dist: timeout-decorator>=0.5.0
 Requires-Dist: youtube-transcript-api>=0.6.3
-Requires-Dist: yt-dlp>=2025.1.26
+Requires-Dist: yt-dlp>=2025.4.30
 Description-Content-Type: text/markdown
 
 # kabigon

{kabigon-0.6.1.dist-info → kabigon-0.7.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
-kabigon/__init__.py,sha256=_1LdKfp7qm0m7Fa_IJ9zcg4TRB14c9bB-r9M3fwASWI,498
-kabigon/cli.py,sha256=XSTyD1RFqq2Qok_52kSjJlBLUXl6t-K9QtsxCfB15o4,611
-kabigon/compose.py,sha256=5H_hWQ7ENUmEdeOpB2bkRv4U_U18_1IyNTran-cLYaM,2166
-kabigon/errors.py,sha256=3eSPQtLocreKuq9mhwRk7IMdA2xh8KkmIFEUhGmTPxg,618
+kabigon/__init__.py,sha256=L-MDyX7PrPZvq6mASyOCJnATebwn-Lj1Nobp9GAPeyU,533
+kabigon/cli.py,sha256=co9IczXx0bAnXo6EEzEdCMSfpBetkPe6pB5hd2jtzA8,675
+kabigon/compose.py,sha256=l2D5OK91VcN2a6DbjMdwBk3YSqzVV7fOVX0TqNm2gJo,1502
+kabigon/errors.py,sha256=iri_YS71UsOHwaVtfy5IA6iUfq30DCsptZsChmZaZic,755
 kabigon/firecrawl.py,sha256=Xnrlhco_R58x5kwGy9ZCKTnVqS4Pp-D3G0u-qnuGEsU,800
 kabigon/httpx.py,sha256=uDdLks6zVzirY7-mnsJkypX86kAI5XmUVfK-lFifdJA,895
 kabigon/loader.py,sha256=KhOJvlzLDM0o05o4VqMRgiyRVjofkzuGEcihQagj-8M,452
@@ -9,11 +9,12 @@ kabigon/pdf.py,sha256=yJcgkdMMF52baFsFxJp9Jn89KsTKBboLTLwD3gs6U5U,1775
 kabigon/playwright.py,sha256=MZ-r0Ej2wWAOJkDLwYRvO77wcDvh38KXz2wgDsCTgm0,1358
 kabigon/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 kabigon/reel.py,sha256=J2QOxGMYi_HaEscQPIipPEoHGN_iksGxR6pV_XvryME,929
+kabigon/twitter.py,sha256=U07pa8xA0nHAaaDPeUelQRvXR5ZnUvYJZW35xRAvHA8,1114
 kabigon/utils.py,sha256=eNTLtHLSB2erDac2HH3jWemgfr8Ou_ozwVb8h9BD-4g,922
 kabigon/youtube.py,sha256=HoiFNq0ookPL7_rO_wloBaY8yTIX6xP8A77F7y02q64,1166
 kabigon/ytdlp.py,sha256=_QRcyFx9s7NnI1MvcWdKKxlX-hHLnqtduCSL5_UH6dU,3140
-kabigon-0.6.1.dist-info/METADATA,sha256=78J35ClbTdy2-vU8GtlQWjgSTitntFI8J-Cz07FGqoo,1078
-kabigon-0.6.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kabigon-0.6.1.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
-kabigon-0.6.1.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
-kabigon-0.6.1.dist-info/RECORD,,
+kabigon-0.7.0.dist-info/METADATA,sha256=4rng3Fe89ifNd6bQoBggRhCoqTMIa4sWhbU6N75puzo,1078
+kabigon-0.7.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kabigon-0.7.0.dist-info/entry_points.txt,sha256=O3FYAO9w-NQvlGMJrBvtrnGHSK2QkUnQBTa30YXRbVE,45
+kabigon-0.7.0.dist-info/licenses/LICENSE,sha256=H2T3_RTgmcngMeC7p_SXT3GwBLkd2DaNgAZuxulcfiA,1066
+kabigon-0.7.0.dist-info/RECORD,,