parsehub 2.0.11__tar.gz → 2.0.12__tar.gz

{parsehub-2.0.11/src/parsehub.egg-info → parsehub-2.0.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: parsehub
-Version: 2.0.11
+Version: 2.0.12
 Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
 Author-email: 梓澪 <zilingmio@gmail.com>
 License: MIT
@@ -98,7 +98,7 @@ print(result)
 
 | 平台              | 视频 | 图文 |  其他   |
 |:----------------|:--:|:--:|:-----:|
-| **Twitter / X** | ✅  | ✅  |       |
+| **Twitter / X** | ✅  | ✅  | 📝 文章 |
 | **Instagram**   | ✅  | ✅  |       |
 | **YouTube**     | ✅  |    | 🎵 音乐 |
 | **Facebook**    | ✅  |    |       |

{parsehub-2.0.11 → parsehub-2.0.12}/README.md

@@ -61,7 +61,7 @@ print(result)
 
 | 平台              | 视频 | 图文 |  其他   |
 |:----------------|:--:|:--:|:-----:|
-| **Twitter / X** | ✅  | ✅  |       |
+| **Twitter / X** | ✅  | ✅  | 📝 文章 |
 | **Instagram**   | ✅  | ✅  |       |
 | **YouTube**     | ✅  |    | 🎵 音乐 |
 | **Facebook**    | ✅  |    |       |

{parsehub-2.0.11 → parsehub-2.0.12}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "parsehub"
-version = "2.0.11"
+version = "2.0.12"
 description = "轻量、异步、开箱即用的社交媒体聚合解析库"
 readme = "README.md"
 requires-python = ">=3.12.0"

{parsehub-2.0.11 → parsehub-2.0.12}/src/parsehub/parsers/parser/twitter.py

@@ -7,7 +7,7 @@ from ...provider_api.twitter import (
     TwitterTweet,
     TwitterVideo,
 )
-from ...types import AniRef, ImageRef, MultimediaParseResult, ParseError, Platform, VideoRef
+from ...types import AniRef, ImageRef, MultimediaParseResult, ParseError, Platform, RichTextParseResult, VideoRef
 from ...utils.utils import cookie_ellipsis
 from ..base.base import BaseParser
 
@@ -48,21 +48,24 @@ class TwitterParser(BaseParser):
     @staticmethod
     async def media_parse(tweet: TwitterTweet):
         media = []
-        for m in tweet.media:
-            match m:
-                case TwitterPhoto():
-                    path = ImageRef(url=m.url, height=m.height, width=m.width, thumb_url=m.thumb_url)
-                case TwitterVideo():
-                    path = VideoRef(
-                        url=m.url,
-                        height=m.height,
-                        width=m.width,
-                        duration=int(m.duration_millis / 1000),
-                        thumb_url=m.thumb_url,
-                    )
-                case TwitterAni():
-                    path = AniRef(url=m.url, ext="mp4", height=m.height, width=m.width, thumb_url=m.thumb_url)
-            media.append(path)
+        if tweet.media:
+            for m in tweet.media:
+                match m:
+                    case TwitterPhoto():
+                        path = ImageRef(url=m.url, height=m.height, width=m.width, thumb_url=m.thumb_url)
+                    case TwitterVideo():
+                        path = VideoRef(
+                            url=m.url,
+                            height=m.height,
+                            width=m.width,
+                            duration=int(m.duration_millis / 1000),
+                            thumb_url=m.thumb_url,
+                        )
+                    case TwitterAni():
+                        path = AniRef(url=m.url, ext="mp4", height=m.height, width=m.width, thumb_url=m.thumb_url)
+                media.append(path)
+        if article := tweet.article:
+            return RichTextParseResult(markdown_content=article.content, title=article.title, media=media)
         return MultimediaParseResult(content=tweet.full_text, media=media)
 
 

parsehub-2.0.12/src/parsehub/provider_api/twitter.py

@@ -0,0 +1,408 @@
+import re
+from dataclasses import dataclass
+from typing import Literal, NamedTuple, Union
+
+import httpx
+from loguru import logger
+
+from ..config import GlobalConfig
+from ..types import ParseError
+
+
+class Twitter:
+    def __init__(self, proxy: str | None = None, cookie: dict = None):
+        self.proxy = proxy
+        self.authorization = (
+            "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOu"
+            "H5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
+        )
+        self.cookie = cookie
+
+    async def fetch_tweet(self, url: str) -> "TwitterTweet":
+        tweet_id = self.get_id_by_url(url)
+        headers = {
+            "accept-language": "zh-CN,zh;q=0.9",
+            "authorization": self.authorization,
+            "content-type": "application/json",
+            "user-agent": GlobalConfig.ua,
+            "x-guest-token": await self.get_guest_token(url),
+            "x-twitter-active-user": "yes",
+            "x-twitter-client-language": "zh-cn",
+        }
+
+        cookie = None
+        if self.cookie and self.check_cookie():
+            headers["x-csrf-token"] = self.cookie.get("ct0")
+            cookie = self.cookie
+
+        params = {
+            "variables": f'{{"tweetId":"{tweet_id}","withComm'
+            f'unity":false,"includePromotedContent":false,"withVoice":false}}',
+            "features": '{"creator_subscriptions_tweet_preview_api_enabled":true,'
+            '"communities_web_enable_tweet_community_results_fetch":true,'
+            '"c9s_tweet_anatomy_moderator_badge_enabled":true,"tweetypie_unmention_optimization_enabled":true,'
+            '"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled"'
+            ":true,"
+            '"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,'
+            '"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,'
+            '"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled"'
+            ":true,"
+            '"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enable'
+            'd":true,'
+            '"tweet_with_visibility_results_prefer_gql_media_interstitial_enabled":false,"rweb_video_timestamps_enabled'
+            '":true,'
+            '"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,'
+            '"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_'
+            'phone_label_enabled":false,'
+            '"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline'
+            '_navigation_enabled":true,'
+            '"responsive_web_enhance_cards_enabled":false}',
+            "fieldToggles": '{"withArticleRichContentState":true,"withArticlePlainText":false}',
+        }
+
+        async with httpx.AsyncClient(proxy=self.proxy) as client:
+            response = await client.get(
+                "https://api.twitter.com/graphql/kPLTRmMnzbPTv70___D06w/TweetResultByRestId",
+                params=params,
+                headers=headers,
+                cookies=cookie,
+            )
+        response.raise_for_status()
+        return self.parse(response.json())
+
+    def parse(self, result: dict):
+        if e := result.get("errors"):
+            raise Exception(f"error -1: {e[0]['message']}")
+
+        result = result["data"]["tweetResult"].get("result")
+        if not result:
+            raise ParseError("error -4: 帖子或用户不存在")
+
+        if tweet := result.get("tweet"):
+            tweet_id = tweet.get("rest_id", {})
+            legacy: dict = tweet.get("legacy")
+        else:
+            tweet_id = result.get("rest_id", {})
+            legacy = result.get("legacy")
+
+        if not legacy:
+            if result.get("__typename") == "TweetTombstone":
+                raise Exception("error -2: 该推文开启了限制, 匿名用户无法查看")
+            raise Exception(f"error -3: {result.get('reason')}")
+
+        if article := result.get("article", {}):
+            ta = ArticleRenderer(article["article_results"]["result"]).render()
+            return TwitterTweet(tweet_id=tweet_id, article=ta)
+
+        if note_tweet := result.get("note_tweet"):
+            full_text = note_tweet.get("note_tweet_results", {}).get("result", {}).get("text", None)
+            if not full_text:
+                full_text = legacy.get("full_text", "")
+        else:
+            full_text = legacy.get("full_text", "")
+
+        media = legacy["entities"].get("media", [])
+        media_list = []
+        for i in media:
+            original_info = i.get("original_info", {})
+            height = original_info.get("height", 0)
+            width = original_info.get("width", 0)
+            media_url_https = i["media_url_https"]
+
+            match i["type"]:
+                case "photo":
+                    media_list.append(
+                        TwitterPhoto(
+                            url=self._build_img_url(media_url_https, "orig"),
+                            width=width,
+                            height=height,
+                            thumb_url=self._build_img_url(media_url_https, "small"),
+                        )
+                    )
+                case "video":
+                    video_info = i.get("video_info", {})
+                    media_list.append(
+                        TwitterVideo(
+                            url=video_info["variants"][-1]["url"],
+                            height=height,
+                            width=width,
+                            duration_millis=video_info.get("duration_millis", 0),
+                            thumb_url=self._build_img_url(media_url_https, "medium"),
+                        )
+                    )
+                case "animated_gif":
+                    media_list.append(
+                        TwitterAni(
+                            url=i["video_info"]["variants"][-1]["url"],
+                            height=height,
+                            width=width,
+                            thumb_url=self._build_img_url(media_url_https, "small"),
+                        )
+                    )
+
+        return TwitterTweet(tweet_id=tweet_id, full_text=full_text, media=media_list or None)
+
+    @staticmethod
+    def _build_img_url(url: str, size: Literal["orig", "large", "medium", "small", "thumb"]):
+        p = "&" if "?" in url else "?"
+        return f"{url}{p}name={size}"
+
+    @staticmethod
+    def get_id_by_url(url: str):
+        return re.search(r"status/(\d+)", url)[1]
+
+    async def get_guest_token(self, url: str):
+        async with httpx.AsyncClient(proxy=self.proxy) as client:
+            response = await client.post(url)
+            response.raise_for_status()
+        guest_token = re.search(r'cookie="gt=(\d+);', response.text)
+        if not guest_token:
+            raise Exception("error -5: 获取 guest_token 失败")
+        return guest_token[1]
+
+    def check_cookie(self):
+        if not self.cookie.get("ct0"):
+            logger.warning("cookie 缺少必要参数: ct0")
+            return False
+        if not self.cookie.get("auth_token"):
+            logger.warning("cookie 缺少必要参数: auth_token")
+            return False
+        return True
+
+
+class TwitterTweet:
+    def __init__(
+        self,
+        tweet_id: str,
+        full_text: str | None = None,
+        media: list[Union["TwitterVideo", "TwitterPhoto", "TwitterAni"]] | None = None,
+        article: Union["TwitterArticle"] = None,
+    ):
+        self.tweet_id = tweet_id
+        self.full_text = re.sub(r"https://t\.co/[^\s,]+$", "", full_text) if media else full_text
+        self.media = media
+        self.article = article
+
+
+@dataclass
+class TwitterArticle:
+    title: str
+    content: str
+    media: list[Union["TwitterVideo", "TwitterPhoto"]] | None = None
+
+
+@dataclass
+class TwitterVideo:
+    url: str
+    height: int
+    width: int
+    duration_millis: int
+    thumb_url: str | None = None
+
+
+@dataclass
+class TwitterPhoto:
+    url: str
+    height: int
+    width: int
+    thumb_url: str | None = None
+
+
+@dataclass
+class TwitterAni:
+    url: str
+    height: int
+    width: int
+    thumb_url: str | None = None
+
+
+class _Insertion(NamedTuple):
+    """待插入原文的 Markdown 标记。"""
+
+    idx: int
+    text: str
+    kind: str  # "start" | "end" | "atomic"
+    length: int = 0
+
+
+class ArticleRenderer:
+    """将 Twitter Article JSON 解析并渲染为 Markdown。"""
+
+    # 行内样式 → Markdown 标记
+    _INLINE_STYLES: dict[str, str] = {
+        "Bold": "**",
+        "Italic": "*",
+        "Strikethrough": "~~",
+    }
+
+    # 块级类型 → 格式化函数
+    _BLOCK_FORMATTERS: dict[str, callable] = {
+        "header-one": lambda t: f"# {t}",
+        "header-two": lambda t: f"## {t}",
+        "header-three": lambda t: f"### {t}",
+        "blockquote": lambda t: "\n".join(f"> {line}" for line in t.split("\n")),
+        "ordered-list-item": lambda t: f"1. {t}",
+        "unordered-list-item": lambda t: f"- {t}",
+    }
+
+    def __init__(self, article_data: dict):
+        self._data = article_data
+        self._media_dict: dict = {}
+        self._media_result: list[TwitterPhoto | TwitterVideo] = []
+
+    # ── 公共入口 ──────────────────────────────
+
+    def render(self) -> "TwitterArticle":
+        content_state = self._data.get("content_state", {})
+        blocks = content_state.get("blocks", [])
+        entity_map = {str(item["key"]): item["value"] for item in content_state.get("entityMap", [])}
+        title = self._data.get("title", "")
+
+        self._parse_media_entities()
+        cover_url = self._data.get("cover_media", {}).get("media_info", {}).get("original_img_url", "")
+
+        md_lines: list[str] = []
+        if cover_url:
+            md_lines.append(f"![Cover Image]({cover_url})\n")
+
+        for block in blocks:
+            md_lines.append(self._render_block(block, entity_map))
+
+        return TwitterArticle(
+            title=title,
+            content="\n\n".join(md_lines),
+            media=self._media_result or None,
+        )
+
+    # ── 媒体解析 ──────────────────────────────
+
+    def _parse_media_entities(self) -> None:
+        for media in self._data.get("media_entities", []):
+            media_id = media.get("media_id")
+            media_info = media.get("media_info", {})
+            typename = media_info.get("__typename")
+
+            if typename == "ApiImage":
+                self._parse_image(media_id, media_info)
+            elif typename == "ApiVideo":
+                self._parse_video(media_id, media_info)
+
+    def _parse_image(self, media_id, info: dict) -> None:
+        url = info.get("original_img_url", "")
+        if media_id and url:
+            self._media_dict[media_id] = {"type": "image", "url": url}
+        self._media_result.append(
+            TwitterPhoto(
+                url=url,
+                height=info.get("original_img_height", 0),
+                width=info.get("original_img_width", 0),
+            )
+        )
+
+    def _parse_video(self, media_id, info: dict) -> None:
+        preview = info.get("preview_image", {})
+        preview_url = preview.get("original_img_url", "")
+        video_url = self._best_mp4_url(info.get("variants", []))
+
+        if media_id and preview_url:
+            self._media_dict[media_id] = {
+                "type": "video",
+                "preview_url": preview_url,
+                "video_url": video_url,
+            }
+        self._media_result.append(
+            TwitterVideo(
+                url=video_url,
+                height=preview.get("original_img_height", 0),
+                width=preview.get("original_img_width", 0),
+                duration_millis=info.get("duration_millis", 0),
+                thumb_url=preview_url,
+            )
+        )
+
+    @staticmethod
+    def _best_mp4_url(variants: list) -> str:
+        mp4s = [v for v in variants if v.get("content_type") == "video/mp4"]
+        if not mp4s:
+            return ""
+        return max(mp4s, key=lambda v: v.get("bit_rate", 0)).get("url", "")
+
+    # ── Block 渲染 ────────────────────────────
+
+    def _render_block(self, block: dict, entity_map: dict) -> str:
+        b_type = block.get("type", "unstyled")
+        text = block.get("text", "")
+
+        insertions = self._collect_inline_styles(block)
+        insertions += self._collect_entities(block, entity_map)
+        insertions.sort(key=self._insertion_sort_key)
+
+        final_text = self._apply_insertions(text, insertions)
+        formatter = self._BLOCK_FORMATTERS.get(b_type)
+        return formatter(final_text) if formatter else final_text
+
+    @staticmethod
+    def _collect_inline_styles(block: dict) -> list[_Insertion]:
+        result: list[_Insertion] = []
+        for style in block.get("inlineStyleRanges", []):
+            marker = ArticleRenderer._INLINE_STYLES.get(style["style"])
+            if not marker:
+                continue
+            offset, length = style["offset"], style["length"]
+            result.append(_Insertion(offset, marker, "start", length))
+            result.append(_Insertion(offset + length, marker, "end", length))
+        return result
+
+    def _collect_entities(self, block: dict, entity_map: dict) -> list[_Insertion]:
+        result: list[_Insertion] = []
+        for ent in block.get("entityRanges", []):
+            offset, length = ent["offset"], ent["length"]
+            ent_data = entity_map.get(str(ent["key"]), {})
+            ent_type = ent_data.get("type")
+
+            if ent_type == "LINK":
+                url = ent_data.get("data", {}).get("url", "")
+                result.append(_Insertion(offset, "[", "start", length))
+                result.append(_Insertion(offset + length, f"]({url})", "end", length))
+
+            elif ent_type == "MEDIA":
+                md = self._media_entity_to_md(ent_data)
+                if md:
+                    result.append(_Insertion(offset, md, "atomic", length))
+
+            elif ent_type == "DIVIDER":
+                result.append(_Insertion(offset, "\n---\n", "atomic", length))
+
+        return result
+
+    def _media_entity_to_md(self, ent_data: dict) -> str:
+        media_items = ent_data.get("data", {}).get("mediaItems", [])
+        if not media_items:
+            return ""
+        obj = self._media_dict.get(media_items[0].get("mediaId"))
+        if not obj:
+            return ""
+
+        if obj["type"] == "image":
+            return f"![Image]({obj['url']})"
+        if obj["type"] == "video":
+            p, v = obj["preview_url"], obj["video_url"]
+            return f"[![Video]({p})]({v})" if v else f"![Video Preview]({p})"
+        return ""
+
+    # ── 文本拼装 ──────────────────────────────
+
+    @staticmethod
+    def _insertion_sort_key(ins: _Insertion) -> tuple:
+        weight = {"end": 1, "atomic": 0, "start": -1}.get(ins.kind, 0)
+        return -ins.idx, weight, ins.length
+
+    @staticmethod
+    def _apply_insertions(text: str, insertions: list[_Insertion]) -> str:
+        chars = list(text)
+        for ins in insertions:
+            if ins.kind == "atomic" and ins.idx < len(chars):
+                chars[ins.idx] = ins.text
+            else:
+                chars.insert(ins.idx, ins.text)
+        return "".join(chars)

{parsehub-2.0.11 → parsehub-2.0.12}/src/parsehub/provider_api/xiaoheihe.py

@@ -10,104 +10,90 @@ from urllib.parse import parse_qs, urlparse
 import httpx
 from markdownify import MarkdownConverter
 
+# TODO: 逆向 EP 和 DATA
 V4_EP = (
-    "V1ZCERzVgMWrKv+VcTl5QmS9JuPWLOQ8A0mACeTyYXtTbiguOrHhwaqnagZ6zdAgF"
-    "4WpAYBvUH3EDnPRlNWut4CTDU1tCa80BSnvTMC9X1j9Kh6IMlGmzPIqpBzzx9r7Nt"
-    "9XtUhv2WiQ2BgPnUwOFe7gN9r8Yj3184qxn1btJL8="
+    "CFcLOAE8E7Ew0J7yxtc9hPtklLIOym8yh1eU5jpB6D0M86gJERnbWbE7wPEWM95v8cWsxACqGq7iU"
+    "OEnrD2ODeFIj5VZdvbD3zhhOgT4FB6QfskCkuCN+JP/+aLz0rg/B+c/9fd5513ESuZxFVqUmrwe/v"
+    "jqZh5nS6Bsyt50VN8="
 )
 V4_DATA = (
-    "abbbe96a1579aa6fe4fa84e875851b7d7a843a14c5c9573c771d9c1443c9b3a"
-    "d7603a8d9d67dbc9bd001bf42702ac82e4a6979323ff305eecd74b9620ee140"
-    "0c135f840b35d9402ec3e3a93fcb3d0d3d6b3e740f5176b72225b6fb8a0d483"
-    "cab753aa71062dc9b59bc8de950628f23607301c6cd94e75f680b86485a11ac"
-    "36eba1413e9f14b274eadff30114dfb1cedadc4bd08ef83c5b2d048970d07d3"
-    "943afef809b44e3b9fee602c91e274fee1523a8beee7e7cec85680b279d616d"
-    "da15e98b1b0aa718276bcdb05d4ac3e44e72da220e0ea798ad7452aec01d0db"
-    "c31ad6bf147eab7f7e539d35fe5149110aae5c7069a67eba4aae638505819f8"
-    "9e2a58bc3b5001c8a5045334121ef04a8e442d7dbb7776bd6013674d2c0028a"
-    "f131bf6bde47b90dce5c8b9463c9f83d0e7264145c2f6f259d70c4d63a4996b"
-    "b7c0074e8a59fa298ad144ec139cb29bc94074fbe2f4a88400d85c003793e2b"
-    "e2077184c3ba2e792926fce25f24d3a764a7c2667446173c74aa704d0d517f2"
-    "10926aaef05376230b43c3a676dad6ff1c9603553d66eadfb492445eac44745"
-    "acc620b325560d4941c10e05f3099a17a553fd763a1b7d6ef29f512e436bdfa"
-    "9fa7c5a70b6a5f91bbcb21946fc2ce92db0c92930008b0fc82e90c3c73f9265"
-    "2ca388f77b262a918cf59160fa88e481138ee7fe9a9b51d7949a74d22d1dab4"
-    "e865c12325bfb5b9e748526afb6d8a05c543fd6dc72e81b06a4ebbf8149fca5"
-    "37a19330da2011eec0229e2302babe239397aa1c2292ab3807cf0aa129d078a"
-    "a9da010003eac5bb2c06435fbbe9bee7543290c1224745bb485d78f42ee4e82"
-    "afb27a38befc60a688fb2514795064926bf205357bd46b7c14dd15aea2cab48"
-    "5c993f0df5a20811d0a7b3bfb1fcb0737c8305675e9bdac396ef8cffb0b6bc4"
-    "700c3d881c1945329b721b9080bed46b18105b7c9fea4f8276f0fcd09fe99ec"
-    "52fa50b11e12a19eb9d091ecde701ab2879e2d7727386b28bbde8d62832e1ad"
-    "822ea57b383cdd3767e8ee64e201bf00fe9cc8428ece3262550764fea47c69e"
-    "e4339de98767f034d8852993fdefa315d9dcda71a74b665804706d4f9a8c139"
-    "3670c2220e4ceac833620e0dc8175eb7a77b8b37c1a9d9940c67d44c8bc6b5f"
-    "9e46273e2f5149d3d3148e8f7a02c4a4c3c998924b7d0e93528952034adc20d"
-    "c342404a8606f0c07cb2b98c4a5434e69b69282daf952f586b9eed4b4f1ef0c"
-    "fe5c6d156d14fb5057c8c32a355d07e2f56737d1ccfad573d42c840bbe8b750"
-    "388211f2c0c5d6a1e34e7741389a742dff58bb0b9f339707a349a09519ca78d"
-    "5e4f1baaf2598ab9001c15824494eecc17735e69a193e5437cbe44c6f156a0b"
-    "b8df4fed5edefd4f56f4ef0b4d8cc40fe623836da3c5e662005825c9d344074"
-    "be2306d6241c163fe92a6ce40ff60538d7464f5a06b6bb9ca1e6f18491ca3c7"
-    "d6c00e299cbb1ca1c525a981fc6c6f2bb05f709101099b8bd0d2c2a628d94c6"
-    "1aa97fdd58c9f357359fbd5be9e8f0f534f4481fb780d58e3e599e01fdd5a7f"
-    "c5fb7e01b76fd58b2f264947d2149fefa57577ef326e264fc827939329031d9"
-    "01be7579ecf5fccdab11c615c1a053f198297c0723faf8b17ea3335d49df2bf"
-    "dd17271c2b64745b1f412d87297edd4404a4ae5312debf73b66afcc3d884b93"
-    "8de41b6ee87265ce624897f3557ebe2d97e6fb17f1dc6a893e48dfa16ef2bff"
-    "d8f3e06f0a1fcf44c7f2efa372e0ff61344c93f4a2a66538fcc134cd0bf94d5"
-    "4c969cda4392af70608cbab6cfa340b674ba3a59385c0ed9bb236ff6ed10e1e"
-    "5a9d4b6529c075dc1ac23cfdae18ab1651a5ee747322e51e3cc6035ca929789"
-    "00924e661a2694a47873569baa95fd821711dc53a1e0299ed707e337b570591"
-    "a3f61a5e39f8a75771da1613e8236c9b1b94cb5617fdaf2424d68a7fbd83ebf"
-    "356fc87e8a805bee5bbd20a55a70881394d7624b1dcf5a135f1cf40b842eca3"
-    "3d46b72447e0a2e85adf6c26efa6cc73b63573840f7b6229fb03ab45a8b639b"
-    "5a66bbd6f63d10e59db49d7a9c9af3e3aeb79b7b756e24d5002917e7e788018"
-    "4f80fcc605a1ba825c779e6083fd7fb0920bbcee021ec8e35427391b871b149"
-    "c306c2dbda602044cd53ec424dd70cfd1c14a23c9964c039258cff4b75112f8"
-    "15d9717433c1989ec398cd2acd67c89be82a409e0ef8f3e9ea8ec8b51b5ea5a"
-    "005b5e735978d9a2987a76d62a2af230e30dc6327f7c0d153add27c7e8a320e"
-    "4df6c05ab91fe0b9f6f9e13c50f39454066776503eb2ec84b74b4b2d5228627"
-    "d81c938f7201610c9b703e4fd283a94835b7387db2880443a050d3eb0859aa1"
-    "efd0f9bb7613b6b918ec2f7b5bb3e7722105b595e7973a93e3de8153a0f8e5b"
-    "fd1aa6cefc6285fea85e8381ddcce98b31dda33db2a3c80ac04df14b872c805"
-    "15373f231c3653fb2db799b32e83e59fb0f5763febca3d291b49bf83dd7ebd6"
-    "1229300b65d44964d9e679f6061a0b2ea1bcd9f5af9bf710047237d87d13394"
-    "ea8b4627c6997589d0b58379d025b076460eab88d6615ee92b0aa6c47f721f9"
-    "7e0b5bbe721f06544d0a1bb81402697f2d72ad32c791dab45064b4d18460602"
-    "9494b268feaebb268e7f92352dc3482f857c14885aabbad98a43e5f8fa5d77d"
-    "61dc22f23080b9e6403c76f5fb862d7520ab85ae7c1d0e339729f664e7d668f"
-    "4b9d1301acabb62fda5940db236ea9d2ca896cbb6a13eda6120fa5881453cb4"
-    "490438460c00db4cd4bdf5df993d3a8d5726c756015eed542e0a4b910570f39"
-    "7211c3f84f6a0d038e82270f94543e8da1e8d0cffd8f4f561daaf6003ad1fad"
-    "fdd89c50f057a79225d8647aead74b33216e328c4204686b4ae93ce5f7ee25e"
-    "1c83fe2cb72c67589aa4865d278ff7a112d09c16707de8acd61b49b901a3266"
-    "e8ef55f1351fdc3013154635e51e649cbf31fc9b32f6956800834ca73e0b75b"
-    "2b54d7125257eb6c24ebff52b741109be6da99bb6e0ffab85c3c219550ec3fc"
-    "b12e2e4d0234627b061193c290baa1be73241be70925c08d33e6efdd44eca9a"
-    "5160bdc5b47bd1f9d3f2cbf38848cf1aaa2a4827f86e43e06246b3bf94cb0b9"
-    "f050c89533a3be9ffecefebd1a92e04197f18d7fadc0bfc8664de18425d5c03"
-    "59b58049267934756f513bd68ea427b38f15213f42cce05cd59f5ea502967ec"
-    "6a096daaa5e5d2a373227f2fe4514e27dfa012d708f7e94a286452972b5fab4"
-    "581ecee3df40bad802cbb50b1a5d9dd3323a5f7c61ab893b16782a0ba64fd42"
-    "10c30ac00f9d21b9124e5e5b323f43badf56761e1eea5c86ff61f19ce1485f4"
-    "2cf6cadd751bbfb2ef87229eee5068ef6e209f123d29a571a374974ceac2e77"
-    "f143faba60fc5d16f88d801fa01d879420b5d1393ad5b2bc913e3b0ba7155a6"
-    "7648196573126273cccc79f2eac32ab68d72cc0f7170feca9c9726af9d65962"
-    "663d5281372386ec88bd2fa82316f687535ecd39f00658523708ca4785529f5"
-    "93baf100597ed00c15ae8ff87baa295871680b4096ac03a550f0f015297198b"
-    "1a93f38cfefbeceabc099c1026664d77f616b4f069cf8bf53d2684b9a4d933c"
-    "3c65a3aef21559527bfc6586e0247efa244a0a355b43751bc09be8012699468"
-    "a8c332d60b11bb4881bf56b92ead10e059ac40f83a4d6725cacbc1bb307c839"
-    "c4edc8b5484b9e2935842e867e739223f2eaaaff04d9701cfa49e3f80be4f2d"
-    "1b7e8eb76fd7f33dfa79831f75ee65a75b7c7fff98254818f1ab77bca856656"
-    "4d48e0012733dd426bf841f27f960394b1bacb8a3e36b96c41d751584cd580f"
-    "ef1b6a8bf990487268348f682a27549ecbb9674b14f2fc97f203f3468f248ec"
-    "3cf5171aa5e8a8d31a9a433c4f7644736aaf6695b28771fe66b4736e3afb322"
-    "11ad534b05641600d2cdc79a251fc4c4e5540df9a40aaad329fedd49a429b20"
-    "70e1345a4146c297ee2a03f056675054e83207d17de21242032c30398259440"
-    "84e60cbd70eb4c469859824cd7d04340de0d19e614a0826a63c63e15c3372b1"
-    "7515d4b6951ff6c612f65c3e6538fd0515bcb4814bb641fca5a45c7dae9"
+    "7ccf4483919143daa17cca371b849651ab10c58aa97415e3fcc9b2f4c0bc776844997f4059"
+    "512c213b3cc965e84693188b08f1ddb8924922598173e0cfa0bab40f242bcd20e11c728da7"
+    "5a75d64b75d4070affa0d64831d0b32efde8c74ac4e6adeef18bbcbd1d21131746d131e30c"
+    "8ba5939ea8247e79534f6688fed7545d5060b069e85c19d11c0277ee8015d2a989d84ce1bd"
+    "01ed2754a365959496343de0152044cef7db82d0353a091f566253f2f8ca14a192c64b610f"
+    "643309079d235d355438c84f566943df3df71c2cc979a68c6f36ce62861d6ddb64874d03f8"
+    "b596b1380de9f84a60aff650ec59e4b2427ba7492f541354ee4dfe09b02c7296539978d281"
+    "2269a7d37121ba96133b7e2b5fdba4922efc6f4bacd31855ca2604b86096ed5abfa6b87656"
+    "8298f4bd75c1de979608714b5f0ec2bd852ec6974f929891cfff70392b0c42c7efd9f53e6b"
+    "52541d08d654f85d92b29b553b3ea4de3c0ddf88ea77815871e476d5ba8b61dfbb427e3147"
+    "62b58a306479eeeb7831864bb593c91af9c85004e891efe5d495b3d1cb4885996ffeda2d50"
+    "7f747be1022544cf6ca1e4663bba30d7e7be129b23c5dd4ee1b56d2c48969eeee5b7b0e062"
+    "8cfc0527c5e2880c43a61dd753c72b76a0ec1556cac7682f54f0582b50419dbfaa504a9363"
+    "54dcb289d282dfa94ded53d926a4385cf437e35afed207c8ccf9eea2e2d493b645034a79ea"
+    "115b5df365cc3c6b160d0de25d5d94efd576538386521cc617058831a39bd9009555fe8bc3"
+    "419e1f4c9c51271d3996dd5616d0071d850a36799296abb9084a8a6b406f62341ebe581d50"
+    "11029c18e88074a2cd7e9fb6be16b948da4d696c624412a8adb4651af89e43db779ed90114"
+    "001c7ad552a6baf80447c751c39ce85ce713a661dd7b67be37aa749b46d8827b2187401e8c"
+    "3e26a5993b654d3b7e6a6323a512a00f925f887d7ce231f20788d999c527b63160b6b1893a"
+    "5891ab183760ca28c95232c164563857a98b963838d385b9638295ded7b69eeb7a43185463"
+    "d2278bd59409f5badc24abffcf5cab137f93d89657992b72c340d1a87ddec55a828d33857d"
+    "ae8b27fc0aad082e14cd8ef294938dddd095f11dd842f94aa055f3b0ba880cc87771f0d61d"
+    "cdc419027c010afb23d668b337cf63ce8359f51623326a81e7513beebfd98d3531b8c701b4"
+    "cc58b42937245244228fcefe0c74b491e765e98ec0f71814788c347b5340163aaa8aae7c97"
+    "332acb3270583f0d77c15c3216696ad4951e24a19107fd5fe150fc275198fe4c9794f2785d"
+    "a3b0b840ebfe75e823b997f0d2eda75f5debbeced24462f1b976e5fc9d643858143d1b0ee4"
+    "6dc3936991f50b5d9d7040a5d9f1cb202fbbc06420cdee16fafc0a6929789088ce8e695332"
+    "b0178a64761a352b15d87aa3a40529febc881d46a3ae80933e407fc2b28c5e0771dd426b02"
+    "1cf177e2ef53c94a0cc5fcc83212843955af3e5f3bb8b24e9ed121669dadd689d54644b507"
+    "1581b0e882d4513220cdf1fd5345b76d1fe1d824357bf3acd8a1c58d4bfb4fe3f39922f72e"
+    "2eb9a74ee4b5f248bf7e279569597f45ef0e7fcbefa2619dcf367fe3638cc93fe90583a72e"
+    "4190729c8c5ab6dd6fb6a37b43eaa90c2e25530ac9d9e923492037f1f14c0da73e4968391f"
+    "c96fe10e2bfbefd620bbd6ea4e948cf04d6219e2c32ee6875cb0c2515b3a9ff993438412d3"
+    "b1b71ba4c50ea98216b50778a1c909cbb7802acc8348aad6a9118a91a9be87f8610a1ab363"
+    "ba06beb726e0a5ce56820e6baf9de2d87a10ce1d5cdd2d94c9e0bab0a3b7b8809d52dd3926"
+    "873caf244ab322a0f2f4c4d9c119153d0b3105c8321dd30378b5345418c5a509fe731aef31"
+    "7b156cdd606d71b291954181fc3efd71467d809b90d2b02a876ddbe7c758c3189ff6ecca21"
+    "44b2a63ef949d7b8b643e3ca7a20c2e5c843e6e34f0260d3963982510a8c077dd7f47158de"
+    "ee71befbab650ef1fad54a622bf4d1c297d9a39995fb1420bdba52d20a939b2da9ec3d8a13"
+    "b156a597f9de8a683ad68a5725a3d2afbdfdbf9c024793558ba6bbd1f6d5f520988358f6d8"
+    "02c0ea8580d4f93218d729cf3bbec52e6224175a0f37dd5bb4901ec5efaa6625c6b6c3b452"
+    "752584d2e634fdee181ef7772857de3831725a6bbb6a22c29a4ddee5e8d1bf5c9aebc1b863"
+    "5ee14584163dae9d4fb2c28be4220a23bb889d1965b870c32273b0166f3195b22cb85fc570"
+    "fb3b13335c49792aaef7b675135a5ced82efe0c36713d7b40123254a7cb0099139bc6634c3"
+    "c1af20595392a6436b192b8e6bb43038a33dff4d22f6f11497cbcb5662e11f2d1510a77b61"
+    "0d1150b15a76b6c916767f1f7f0883db4a0f7b96e9d9b0884249f965212ec1cb54056ee26d"
+    "a2a883f29acdfc7040d4e2e99c4ffd42a8bb1c7852cb5b4c758cdc295baaf973eebd6e720c"
+    "bf0bd6b30ad4a7133929e4b1223c4a579dc1dde1f4fdc1fec5a83c0e3d5335f2dc79e57efc"
+    "74f64b4d69d0151d4025ee5392fd844f783e2c614903e0b3685362f142fa091dce36382c1d"
+    "dc3a6a63815fe062c59e86cad9d26bb54dbf93297ad4ae75039719eddf659c22f0922f08fe"
+    "9a2241200f87bfe60f92d9983062d868d5eaced8df5b2851f86b9ee00055d386bf1276ad9b"
+    "b27f2fa4b04ca6e773ff7348eb078e7b3b20ac5f878552133a652793f630304d28f1dc8ecb"
+    "eedf571f743ffb494c9b34a47df86df8530af4243f0fedfea466c374ff920571a998ebb799"
+    "6c9b0ec4ef5780bd519f19106ad1a5b16183bf62cbc0d7d7e4c297df6c0870fd07825d29c9"
+    "b51ecdc227efeda8848eaca34a4c65ef35c0d5d3fa6e02f416cf25c84ef054206906e0950e"
+    "24250b6e8cbea114c42de785f2ac69204ff675c7bd8f89bb1f683b9adb1c08d73cea3b5cfe"
+    "420fa46a893b9b4ba5674c502bebc59d492942af6eef30a09eb9ff94ead00ebc2007702868"
+    "63ec52c88a45ec7cbe5414485d28c64112aca5015f1976c2bd772cacb7baa5ae267035c7a1"
+    "d9703289821b84ef386f6998777f72f44392f28daa1dc23d26445ed5ca382405ae8b2b47a0"
+    "06d56a040b55c6796328ace7d8faa040d3009e5b627e12c30ec6c02bff8de7173b9f393320"
+    "3e0fb8e06f812ee8ba5a673f3fa31c27e5309a3f7e0a8a55829c0f5c8c7433bbc4db4cfce9"
+    "aea6f37058dd0bcaef20b54546466bdef7b5f69745d4d4ba59c61bc64fd4202f9ce95cc8e1"
+    "a56273db05551b6de959c5e2d5f2ccc6d9893d99e48a1ff043889c5bdcb96512ccff7237bd"
+    "95fd344d3dd46e8d19743a65cde0aeace9ec6563f4c5d2a1dd6e72a32b48dc9444246d6d9e"
+    "a5a9a8d4216b9e0b41f1e54179c52c9f456dbe6c4e8872627b54d7ca6957a270bac31a98cb"
+    "2bacf895f30ed6a508b9bdeb288ccfbf5166cec8535ab73c5fa90b41f4ba5d8a55a7cfb8d9"
+    "783e00356ee534676215463f0aa1333b3388c13c8c0f176af6d7d2a01e2dd01cac2eb73574"
+    "bd6c0930c412cf12bcb80708706cc94b2b9546621f64547b8543179a203d9d871dfc4d5cd4"
+    "8334f42598f62e7c8199782bd605c75dd719c0db51ed801a47938746caf258966fc3132f6c"
+    "77b0a97ba78ece0e150fee450a90433d2b8534d276b07e8d4586043de0ffe1af106f026d45"
+    "41ad961aea6f69fa92344ed9a93f76f2a9f0f29110a4f0a7bda6a84a46d815c68784ab6685"
+    "466059376f0f8866107623c49d59acf60a010c923a73177ea9f58e187bcec2d6feb94a5220"
+    "56325e1651b5499fd28c17456a756e171840b7f8f1d6785e3e63d0bb5a690cc148f45ba0b0"
+    "6b5e0c8da2c6711a6b5011fdfc57221767bce9925d149f357cfa8f108965f9f6037f9b3bc9"
+    "46d90499ec8c40108216ed10eea155cb8d8e7bf76cc17efc1fda962101dc22114ca7b3b39c"
+    "44c3345d0e1c525e4cbdc1f49dbb66ad1f5874bb91a577cf66428fa861624febfb03c369d1"
+    "9d794544"
 )
 
 

{parsehub-2.0.11 → parsehub-2.0.12/src/parsehub.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: parsehub
-Version: 2.0.11
+Version: 2.0.12
 Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
 Author-email: 梓澪 <zilingmio@gmail.com>
 License: MIT
@@ -98,7 +98,7 @@ print(result)
 
 | 平台              | 视频 | 图文 |  其他   |
 |:----------------|:--:|:--:|:-----:|
-| **Twitter / X** | ✅  | ✅  |       |
+| **Twitter / X** | ✅  | ✅  | 📝 文章 |
 | **Instagram**   | ✅  | ✅  |       |
 | **YouTube**     | ✅  |    | 🎵 音乐 |
 | **Facebook**    | ✅  |    |       |

parsehub-2.0.11/src/parsehub/provider_api/twitter.py

@@ -1,203 +0,0 @@
-import re
-from dataclasses import dataclass
-from typing import Literal, Union
-
-import httpx
-from loguru import logger
-
-from ..config import GlobalConfig
-from ..types import ParseError
-
-
-class Twitter:
-    def __init__(self, proxy: str | None = None, cookie: dict = None):
-        self.proxy = proxy
-        self.authorization = (
-            "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOu"
-            "H5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
-        )
-        self.cookie = cookie
-
-    async def fetch_tweet(self, url: str) -> "TwitterTweet":
-        tweet_id = self.get_id_by_url(url)
-        headers = {
-            "accept-language": "zh-CN,zh;q=0.9",
-            "authorization": self.authorization,
-            "content-type": "application/json",
-            "user-agent": GlobalConfig.ua,
-            "x-guest-token": await self.get_guest_token(url),
-            "x-twitter-active-user": "yes",
-            "x-twitter-client-language": "zh-cn",
-        }
-
-        cookie = None
-        if self.cookie and self.check_cookie():
-            headers["x-csrf-token"] = self.cookie.get("ct0")
-            cookie = self.cookie
-
-        params = {
-            "variables": f'{{"tweetId":"{tweet_id}","withComm'
-            f'unity":false,"includePromotedContent":false,"withVoice":false}}',
-            "features": '{"creator_subscriptions_tweet_preview_api_enabled":true,'
-            '"communities_web_enable_tweet_community_results_fetch":true,'
-            '"c9s_tweet_anatomy_moderator_badge_enabled":true,"tweetypie_unmention_optimization_enabled":true,'
-            '"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled"'
-            ":true,"
-            '"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,'
-            '"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,'
-            '"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled"'
-            ":true,"
-            '"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enable'
-            'd":true,'
-            '"tweet_with_visibility_results_prefer_gql_media_interstitial_enabled":false,"rweb_video_timestamps_enabled'
-            '":true,'
-            '"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,'
-            '"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_'
-            'phone_label_enabled":false,'
-            '"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline'
-            '_navigation_enabled":true,'
-            '"responsive_web_enhance_cards_enabled":false}',
-            "fieldToggles": '{"withArticleRichContentState":true,"withArticlePlainText":false}',
-        }
-
-        async with httpx.AsyncClient(proxy=self.proxy) as client:
-            response = await client.get(
-                "https://api.twitter.com/graphql/kPLTRmMnzbPTv70___D06w/TweetResultByRestId",
-                params=params,
-                headers=headers,
-                cookies=cookie,
-            )
-        response.raise_for_status()
-        return self.parse(response.json())
-
-    def parse(self, result: dict):
-        if e := result.get("errors"):
-            raise Exception(f"error -1: {e[0]['message']}")
-
-        result = result["data"]["tweetResult"].get("result")
-        if not result:
-            raise ParseError("error -4: 帖子或用户不存在")
-
-        if tweet := result.get("tweet"):
-            tweet_id = tweet.get("rest_id", {})
-            legacy: dict = tweet.get("legacy")
-        else:
-            tweet_id = result.get("rest_id", {})
-            legacy = result.get("legacy")
-
-        if not legacy:
-            if result.get("__typename") == "TweetTombstone":
-                raise Exception("error -2: 该推文开启了限制, 匿名用户无法查看")
-            raise Exception(f"error -3: {result.get('reason')}")
-
-        if note_tweet := result.get("note_tweet"):
-            full_text = note_tweet.get("note_tweet_results", {}).get("result", {}).get("text", None)
-            if not full_text:
-                full_text = legacy.get("full_text", "")
-        else:
-            full_text = legacy.get("full_text", "")
-
-        media = legacy["entities"].get("media", [])
-        medias = []
-        for i in media:
-            original_info = i.get("original_info", {})
-            height = original_info.get("height", 0)
-            width = original_info.get("width", 0)
-            media_url_https = i["media_url_https"]
-
-            match i["type"]:
-                case "photo":
-                    medias.append(
-                        TwitterPhoto(
-                            url=self._build_img_url(media_url_https, "orig"),
-                            width=width,
-                            height=height,
-                            thumb_url=self._build_img_url(media_url_https, "small"),
-                        )
-                    )
-                case "video":
-                    video_info = i.get("video_info", {})
-                    medias.append(
-                        TwitterVideo(
-                            url=video_info["variants"][-1]["url"],
-                            height=height,
-                            width=width,
-                            duration_millis=video_info.get("duration_millis", 0),
-                            thumb_url=self._build_img_url(media_url_https, "medium"),
-                        )
-                    )
-                case "animated_gif":
-                    medias.append(
-                        TwitterAni(
-                            url=i["video_info"]["variants"][-1]["url"],
-                            height=height,
-                            width=width,
-                            thumb_url=self._build_img_url(media_url_https, "small"),
-                        )
-                    )
-
-        return TwitterTweet(tweet_id=tweet_id, full_text=full_text, media=medias)
-
-    @staticmethod
-    def _build_img_url(url: str, size: Literal["orig", "large", "medium", "small", "thumb"]):
-        p = "&" if "?" in url else "?"
-        return f"{url}{p}name={size}"
-
-    @staticmethod
-    def get_id_by_url(url: str):
-        return re.search(r"status/(\d+)", url)[1]
-
-    async def get_guest_token(self, url: str):
-        async with httpx.AsyncClient(proxy=self.proxy) as client:
-            response = await client.post(url)
-            response.raise_for_status()
-        guest_token = re.search(r'cookie="gt=(\d+);', response.text)
-        if not guest_token:
-            raise Exception("error -5: 获取 guest_token 失败")
-        return guest_token[1]
-
-    def check_cookie(self):
-        if not self.cookie.get("ct0"):
-            logger.warning("cookie 缺少必要参数: ct0")
-            return False
-        if not self.cookie.get("auth_token"):
-            logger.warning("cookie 缺少必要参数: auth_token")
-            return False
-        return True
-
-
-class TwitterTweet:
-    def __init__(
-        self,
-        tweet_id: str,
-        full_text: str,
-        media: list[Union["TwitterVideo", "TwitterPhoto", "TwitterAni"]],
-    ):
-        self.tweet_id = tweet_id
-        self.full_text = re.sub(r"https://t\.co/[^\s,]+$", "", full_text) if media else full_text
-        self.media = media
-
-
-@dataclass
-class TwitterVideo:
-    url: str
-    height: int
-    width: int
-    duration_millis: int
-    thumb_url: str | None = None
-
-
-@dataclass
-class TwitterPhoto:
-    url: str
-    height: int
-    width: int
-    thumb_url: str | None = None
-
-
-@dataclass
-class TwitterAni:
-    url: str
-    height: int
-    width: int
-    thumb_url: str | None = None