parsehub 2.1.4__tar.gz → 2.1.6__tar.gz

{parsehub-2.1.4/src/parsehub.egg-info → parsehub-2.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: parsehub
-Version: 2.1.4
+Version: 2.1.6
 Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
 Author-email: 梓澪 <zilingmio@gmail.com>
 License: MIT
@@ -53,6 +53,8 @@ Dynamic: license-file
 
 轻量, 异步, 开箱即用的社交媒体解析与媒体下载库, 支持 17+ 平台
 
+简体中文 | [English](README.en.md)
+
 [安装](#-安装) · [快速开始](#-快速开始) · [高级用法](#-高级用法) · [TG Bot](https://github.com/z-mio/parse_hub_bot)
 
 </div>
@@ -205,6 +207,7 @@ print(result)
 
 - `Twitter / X`
 - `Instagram`
+- `Threads`
 - `YouTube`
 - `Bilibili`
 - `抖音`

{parsehub-2.1.4 → parsehub-2.1.6}/README.md

@@ -11,6 +11,8 @@
 
 轻量, 异步, 开箱即用的社交媒体解析与媒体下载库, 支持 17+ 平台
 
+简体中文 | [English](README.en.md)
+
 [安装](#-安装) · [快速开始](#-快速开始) · [高级用法](#-高级用法) · [TG Bot](https://github.com/z-mio/parse_hub_bot)
 
 </div>
@@ -163,6 +165,7 @@ print(result)
 
 - `Twitter / X`
 - `Instagram`
+- `Threads`
 - `YouTube`
 - `Bilibili`
 - `抖音`

{parsehub-2.1.4 → parsehub-2.1.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "parsehub"
-version = "2.1.4"
+version = "2.1.6"
 description = "轻量、异步、开箱即用的社交媒体聚合解析库"
 readme = "README.md"
 requires-python = ">=3.12.0"

parsehub-2.1.6/src/parsehub/parsers/parser/threads.py

@@ -0,0 +1,39 @@
+from ...provider_api.threads import ThreadsAPI, ThreadsAPIError, ThreadsMedia, ThreadsMediaType, ThreadsPost
+from ...types import AnyMediaRef, ImageRef, MultimediaParseResult, ParseError, Platform, VideoRef
+from ..base.base import BaseParser
+
+
+class ThreadsParser(BaseParser):
+    __platform__ = Platform.THREADS
+    __supported_type__ = ["视频", "图文"]
+    __match__ = r"^(http(s)?://)?.+threads.com/@[\w.]+/post/.*"
+
+    async def _do_parse(self, raw_url: str) -> "MultimediaParseResult":
+        post = await self._parse(raw_url)
+        media: list[AnyMediaRef] = []
+        if post.media:
+            pm: list[ThreadsMedia] = post.media if isinstance(post.media, list) else [post.media]
+            for m in pm:
+                match m.type:
+                    case ThreadsMediaType.VIDEO:
+                        media.append(VideoRef(url=m.url, thumb_url=m.thumb_url, width=m.width, height=m.height))
+                    case ThreadsMediaType.IMAGE:
+                        media.append(ImageRef(url=m.url, thumb_url=m.url, width=m.width, height=m.height))
+        return MultimediaParseResult(content=post.content, media=media)
+
+    async def _parse(self, url: str) -> ThreadsPost:
+        # 公开帖子无需登录即可解析; 登录墙内容 (私密/受限/年龄限制) 才需要 Cookie, 有则带上
+        try:
+            api = ThreadsAPI(proxy=self.proxy, cookie=self.cookie.get_value() if self.cookie else None)
+            return await api.parse(url)
+        except ThreadsAPIError as e:
+            if not self.cookie:
+                raise ParseError("无法获取帖子内容: 该帖子可能位于登录墙内, 请为 threads 平台配置 Cookie") from e
+            raise ParseError("无法获取帖子内容(可能为私人或受限内容, 或 Cookie 已失效)") from e
+        except ParseError:
+            raise
+        except Exception as e:
+            raise ParseError(f"无法获取帖子内容: {e}") from e
+
+
+__all__ = ["ThreadsParser"]

{parsehub-2.1.4 → parsehub-2.1.6}/src/parsehub/parsers/parser/xhs.py

@@ -20,8 +20,8 @@ from ..base import BaseParser
 class XHSParser(BaseParser):
     __platform__ = Platform.XHS
     __supported_type__ = ["视频", "图文"]
-    __match__ = r"^(http(s)?://)?.+(xiaohongshu|xhslink).com/.+"
-    __redirect_keywords__ = ["xhslink", "item"]
+    __match__ = r"^(http(s)?://)?.+(xiaohongshu|xhslink).(com|cn)/.+"
+    __redirect_keywords__ = ["xhslink"]
     __after_clean_parameters__ = ["xsec_token"]
 
     async def _do_parse(self, raw_url: str) -> Union["VideoParseResult", "ImageParseResult", "MultimediaParseResult"]:

parsehub-2.1.6/src/parsehub/provider_api/threads.py

@@ -0,0 +1,287 @@
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+import httpx
+
+from ..utils.helpers import UA
+
+
+class ThreadsAPIError(Exception):
+    """Threads API 相关错误"""
+
+
+class ThreadsAPI:
+    GRAPHQL_URL = "https://www.threads.com/graphql/query"
+    THREADS_URL = "https://www.threads.com/"
+    # BarcelonaPostPageDirectQuery, 通过帖子 ID 获取帖子内容
+    POST_DOC_ID = "27419285281047858"
+    X_IG_APP_ID = "238260118697367"
+    # shortcode <-> pk 使用与 Instagram 相同的 base64 字母表
+    SHORTCODE_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
+
+    # Threads 与 Instagram 共用 Meta 的登录后端, 因此使用同一套 Cookie 键
+    DEFAULT_COOKIES = {
+        "sessionid": "",
+        "ds_user_id": "",
+        "csrftoken": "",
+        "mid": "",
+        "ig_did": "",
+    }
+
+    # GraphQL 查询强制要求的 relay provider 变量: 必须全部传入 (值统一给 False 即可),
+    # 缺失或只传部分都会导致查询返回 execution error.
+    # 与 Instagram 的 doc_id 一样, 该列表会随 Threads 前端更新而变化, 需要时同步维护.
+    RELAY_PROVIDERS = (
+        "BarcelonaHasPermalinkIndentation",
+        "BarcelonaIsLoggedIn",
+        "BarcelonaHasPostAuthorNotifControls",
+        "BarcelonaShouldShowFediverseM1Features",
+        "BarcelonaHasPermalinkPodcastCard",
+        "BarcelonaHasDearAlgoConsumption",
+        "BarcelonaHasEventBadge",
+        "BarcelonaGenAIRepliesEnabled",
+        "BarcelonaIsSearchDiscoveryEnabled",
+        "BarcelonaHasCommunities",
+        "BarcelonaHasGameScoreShare",
+        "BarcelonaHasPublicViewCountCard",
+        "BarcelonaHasCommunityEntityCard",
+        "BarcelonaHasScorecardCommunity",
+        "BarcelonaHasSportTeamAllegianceCard",
+        "BarcelonaHasMusic",
+        "BarcelonaHasNewspaperLinkStyle",
+        "BarcelonaHasMessaging",
+        "BarcelonaHasPodcastTextFragments",
+        "BarcelonaShouldFulfillLightboxQuery",
+        "BarcelonaHasViewerReplied",
+        "BarcelonaHasPrivateRepliesDeprecation",
+        "BarcelonaHasGhostPostEmojiActivation",
+        "BarcelonaOptionalCookiesEnabled",
+        "BarcelonaHasDearAlgoWebProduction",
+        "BarcelonaHasWebFavicons",
+        "BarcelonaIsCrawler",
+        "BarcelonaHasCommunityTopContributors",
+        "BarcelonaCanSeeSponsoredContent",
+        "BarcelonaShouldShowFediverseM075Features",
+        "BarcelonaIsInternalUser",
+    )
+
+    def __init__(
+        self,
+        proxy: str | None = None,
+        cookie: dict[str, str] | None = None,
+        timeout: float = 30,
+    ):
+        self.proxy = proxy
+        self.cookie = cookie or {}
+        self.timeout = timeout
+
+    async def parse(self, url: str) -> ThreadsPost:
+        code = self.get_post_id_by_url(url)
+        payload = await self._post_graphql(
+            doc_id=self.POST_DOC_ID,
+            variables=self._build_variables(code),
+        )
+        post = self._extract_post(payload, code)
+        if post is None:
+            raise ThreadsAPIError("Fetching Post metadata failed.")
+        return ThreadsPost.from_graphql(post)
+
+    def _build_variables(self, code: str) -> dict[str, Any]:
+        variables: dict[str, Any] = {"postID": str(self.shortcode_to_pk(code))}
+        for name in self.RELAY_PROVIDERS:
+            variables[f"__relay_internal__pv__{name}relayprovider"] = False
+        return variables
+
+    async def _post_graphql(self, *, doc_id: str, variables: dict[str, Any]) -> dict[str, Any]:
+        async with self._new_client() as client:
+            await self._ensure_csrf_token(client)
+            data = {
+                "variables": json.dumps(variables, separators=(",", ":")),
+                "doc_id": doc_id,
+                "server_timestamps": "true",
+            }
+            try:
+                response = await client.post(self.GRAPHQL_URL, data=data, follow_redirects=False)
+            except httpx.HTTPError as exc:
+                raise ThreadsAPIError(f"请求 Threads GraphQL 失败: {exc}") from exc
+
+        if response.status_code != 200:
+            raise ThreadsAPIError(f"Threads GraphQL 返回 HTTP {response.status_code}: {response.text[:500]}")
+
+        try:
+            payload: dict[str, Any] = response.json()
+        except ValueError as exc:
+            # 未登录时会返回 HTML 登录页
+            raise ThreadsAPIError("Threads GraphQL 返回非 JSON 响应(可能需要登录)") from exc
+
+        if payload.get("errors"):
+            raise ThreadsAPIError(f"Threads GraphQL 返回错误: {payload['errors']}")
+        return payload
+
+    @staticmethod
+    def _extract_post(payload: dict[str, Any], code: str) -> dict[str, Any] | None:
+        data = ((payload.get("data") or {}).get("data")) or {}
+        edges = data.get("edges") or []
+        fallback: dict[str, Any] | None = None
+        for edge in edges:
+            for item in (edge.get("node") or {}).get("thread_items") or []:
+                post = item.get("post")
+                if not isinstance(post, dict):
+                    continue
+                if fallback is None:
+                    fallback = post
+                if post.get("code") == code:
+                    return post
+        # 找不到精确匹配时退回第一条 (通常即目标帖子本身)
+        return fallback
+
+    def _new_client(self) -> httpx.AsyncClient:
+        cookies = self.DEFAULT_COOKIES | self.cookie
+        headers = {
+            "Accept": "*/*",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Referer": self.THREADS_URL,
+            "User-Agent": UA,
+            "X-IG-App-ID": self.X_IG_APP_ID,
+        }
+        return httpx.AsyncClient(
+            cookies=cookies,
+            headers=headers,
+            proxy=self.proxy,
+            timeout=self.timeout,
+        )
+
+    async def _ensure_csrf_token(self, client: httpx.AsyncClient) -> None:
+        csrf_token = self._get_cookie_value(client, "csrftoken")
+        if not csrf_token:
+            try:
+                await client.get(self.THREADS_URL, follow_redirects=True)
+            except httpx.HTTPError as exc:
+                raise ThreadsAPIError(f"获取 Threads csrftoken 失败: {exc}") from exc
+            csrf_token = self._get_cookie_value(client, "csrftoken")
+        if csrf_token:
+            client.headers["x-csrftoken"] = csrf_token
+
+    @staticmethod
+    def _get_cookie_value(client: httpx.AsyncClient, name: str) -> str:
+        values = [cookie.value for cookie in client.cookies.jar if cookie.name == name and cookie.value]
+        return values[-1] if values else ""
+
+    @classmethod
+    def shortcode_to_pk(cls, code: str) -> int:
+        pk = 0
+        for ch in code:
+            try:
+                pk = pk * 64 + cls.SHORTCODE_ALPHABET.index(ch)
+            except ValueError as exc:
+                raise ValueError(f"无效的 Threads 帖子 ID: {code}") from exc
+        return pk
+
+    @staticmethod
+    def get_username_by_url(url: str) -> str:
+        u = re.search(r"/(@[\w.]+)/post/", url)
+        if not u:
+            raise ValueError("从 URL 中获取用户名失败")
+        return u[1]
+
+    @staticmethod
+    def get_post_id_by_url(url: str) -> str:
+        p = re.search(r"/post/([\w-]+)", url)
+        if not p:
+            raise ValueError("从 URL 中获取帖子 ID 失败")
+        return p[1]
+
+
+class ThreadsMediaType(Enum):
+    IMAGE = "image"
+    VIDEO = "video"
+
+
+@dataclass
+class ThreadsMedia:
+    type: ThreadsMediaType
+    url: str
+    thumb_url: str | None = None
+    width: int = 0
+    height: int = 0
+
+
+@dataclass
+class ThreadsPost:
+    content: str
+    media: ThreadsMedia | list[ThreadsMedia] | None = None
+
+    @classmethod
+    def from_graphql(cls, post: dict[str, Any]) -> ThreadsPost:
+        caption = post.get("caption")
+        content = caption.get("text") if isinstance(caption, dict) else caption
+        return cls(content=str(content or ""), media=cls._fetch_media(post))
+
+    @classmethod
+    def _fetch_media(cls, d: dict[str, Any]) -> ThreadsMedia | list[ThreadsMedia]:
+        media: ThreadsMedia | list[ThreadsMedia]
+        match d.get("media_type"):
+            case 1:  # 单张图片
+                image = d["image_versions2"]["candidates"][0]
+                media = ThreadsMedia(
+                    type=ThreadsMediaType.IMAGE,
+                    url=image["url"],
+                    thumb_url=image["url"],
+                    width=image.get("width", 0),
+                    height=image.get("height", 0),
+                )
+            case 2:  # 单个视频
+                thumb = d["image_versions2"]["candidates"][0]["url"]
+                video = d["video_versions"][0]["url"]
+                media = ThreadsMedia(
+                    type=ThreadsMediaType.VIDEO,
+                    url=video,
+                    thumb_url=thumb,
+                    width=d.get("original_width", 0),
+                    height=d.get("original_height", 0),
+                )
+            case 8:  # 多图/视频
+                media = []
+                for m in d.get("carousel_media") or []:
+                    if m.get("video_versions"):
+                        thumb = m["image_versions2"]["candidates"][0]["url"]
+                        media.append(
+                            ThreadsMedia(
+                                type=ThreadsMediaType.VIDEO,
+                                url=m["video_versions"][0]["url"],
+                                thumb_url=thumb,
+                                width=m.get("original_width", 0),
+                                height=m.get("original_height", 0),
+                            )
+                        )
+                    else:
+                        image = m["image_versions2"]["candidates"][0]
+                        media.append(
+                            ThreadsMedia(
+                                type=ThreadsMediaType.IMAGE,
+                                url=image["url"],
+                                thumb_url=image["url"],
+                                width=m.get("original_width", 0),
+                                height=m.get("original_height", 0),
+                            )
+                        )
+            case 19:  # 纯文本/外部链接
+                linked = (d.get("text_post_app_info") or {}).get("linked_inline_media")
+                media = cls._fetch_media(linked) if linked else []
+            case _:
+                media = []
+        return media
+
+
+__all__ = [
+    "ThreadsAPI",
+    "ThreadsAPIError",
+    "ThreadsMedia",
+    "ThreadsMediaType",
+    "ThreadsPost",
+]

{parsehub-2.1.4 → parsehub-2.1.6/src/parsehub.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: parsehub
-Version: 2.1.4
+Version: 2.1.6
 Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
 Author-email: 梓澪 <zilingmio@gmail.com>
 License: MIT
@@ -53,6 +53,8 @@ Dynamic: license-file
 
 轻量, 异步, 开箱即用的社交媒体解析与媒体下载库, 支持 17+ 平台
 
+简体中文 | [English](README.en.md)
+
 [安装](#-安装) · [快速开始](#-快速开始) · [高级用法](#-高级用法) · [TG Bot](https://github.com/z-mio/parse_hub_bot)
 
 </div>
@@ -205,6 +207,7 @@ print(result)
 
 - `Twitter / X`
 - `Instagram`
+- `Threads`
 - `YouTube`
 - `Bilibili`
 - `抖音`

parsehub-2.1.4/src/parsehub/parsers/parser/threads.py

@@ -1,25 +0,0 @@
-from ...provider_api.threads import ThreadsAPI, ThreadsMedia, ThreadsMediaType
-from ...types import AnyMediaRef, ImageRef, MultimediaParseResult, Platform, VideoRef
-from ..base.base import BaseParser
-
-
-class ThreadsParser(BaseParser):
-    __platform__ = Platform.THREADS
-    __supported_type__ = ["视频", "图文"]
-    __match__ = r"^(http(s)?://)?.+threads.com/@[\w.]+/post/.*"
-
-    async def _do_parse(self, raw_url: str) -> "MultimediaParseResult":
-        post = await ThreadsAPI(proxy=self.proxy).parse(raw_url)
-        media: list[AnyMediaRef] = []
-        if post.media:
-            pm: list[ThreadsMedia] = post.media if isinstance(post.media, list) else [post.media]
-            for m in pm:
-                match m.type:
-                    case ThreadsMediaType.VIDEO:
-                        media.append(VideoRef(url=m.url, thumb_url=m.thumb_url, width=m.width, height=m.height))
-                    case ThreadsMediaType.IMAGE:
-                        media.append(ImageRef(url=m.url, thumb_url=m.url, width=m.width, height=m.height))
-        return MultimediaParseResult(content=post.content, media=media)
-
-
-__all__ = ["ThreadsParser"]

parsehub-2.1.4/src/parsehub/provider_api/threads.py

@@ -1,174 +0,0 @@
-from __future__ import annotations
-
-import json
-import random
-import re
-import string
-from dataclasses import dataclass
-from enum import Enum
-
-import httpx
-
-from ..utils.helpers import UA
-
-
-class ThreadsAPI:
-    def __init__(self, proxy: str | None = None):
-        self.proxy = proxy
-
-    async def parse(self, url: str) -> ThreadsPost:
-        lsd = self.random_lsd()
-        headers = {
-            "content-type": "application/x-www-form-urlencoded",
-            "sec-fetch-site": "same-origin",
-            "user-agent": UA,
-            "x-fb-lsd": lsd,
-        }
-
-        data = {
-            "route_url": f"/{self.get_username_by_url(url)}/post/{self.get_post_id_by_url(url)}/media",
-            "routing_namespace": "barcelona_web",
-            "__user": "0",
-            "__a": "1",
-            "__req": "m",
-            "__comet_req": "29",
-            "lsd": lsd,
-        }
-
-        async with httpx.AsyncClient(proxy=self.proxy) as client:
-            response = await client.post("https://www.threads.com/ajax/route-definition", headers=headers, data=data)
-            response.raise_for_status()
-            jsonp = [json.loads(j.strip()) for j in response.text.strip().split("for (;;);") if j]
-            return ThreadsPost.parse(jsonp)
-
-    @staticmethod
-    def get_username_by_url(url: str) -> str:
-        u = re.search(r"/(@[\w.]+)/post/", url)
-        if not u:
-            raise ValueError("从 URL 中获取用户名失败")
-        return u[1]
-
-    @staticmethod
-    def get_post_id_by_url(url: str) -> str:
-        p = re.search(r"/post/([\w-]+)", url)
-        if not p:
-            raise ValueError("从 URL 中获取帖子 ID 失败")
-        return p[1]
-
-    @staticmethod
-    def random_lsd() -> str:
-        return "".join(random.sample(string.ascii_letters + string.digits, 11))
-
-
-class ThreadsMediaType(Enum):
-    IMAGE = "image"
-    VIDEO = "video"
-
-
-@dataclass
-class ThreadsMedia:
-    type: ThreadsMediaType
-    url: str
-    thumb_url: str | None = None
-    width: int = 0
-    height: int = 0
-
-
-@dataclass
-class ThreadsPost:
-    content: str
-    media: ThreadsMedia | list[ThreadsMedia] | None = None
-
-    @classmethod
-    def parse(cls, jsonp: list[dict]) -> ThreadsPost:
-        content = ""
-        media: ThreadsMedia | list[ThreadsMedia] | None = []
-        for j in jsonp:
-            match j["__type"]:
-                case "first_response":
-                    content = cls._fetch_content(j)
-                case "preloader":
-                    if "BarcelonaLightboxDialogRootQueryRelayPreloader" in (j.get("id") or ""):
-                        media = cls._fetch_media(j)
-                case "last_response":
-                    ...
-        return cls(content=content, media=media)
-
-    @staticmethod
-    def _fetch_content(data: dict) -> str:
-        payload = data["payload"]
-        result = payload.get("result", {})
-        # 尝试从 redirect_result 获取（用户更改名称后的情况）
-        meta = result.get("redirect_result", {}).get("exports", {}).get("meta")
-        # 如果没有 redirect_result，则从正常路径获取
-        if not meta:
-            meta = result.get("exports", {}).get("meta")
-        if not meta:
-            raise Exception("获取内容失败")
-        return str(meta["title"])
-
-    @staticmethod
-    def _fetch_media(data: dict) -> ThreadsMedia | list[ThreadsMedia]:
-        data = data.get("result", {}).get("result", {}).get("data", {}).get("data")
-        if not data:
-            return []
-
-        def fn(d: dict) -> ThreadsMedia | list[ThreadsMedia]:
-            media: ThreadsMedia | list[ThreadsMedia]
-            match d["media_type"]:
-                case 1:  # 单张图片
-                    image = d["image_versions2"]["candidates"][0]
-                    media = ThreadsMedia(
-                        type=ThreadsMediaType.IMAGE,
-                        url=image["url"],
-                        thumb_url=image["url"],
-                        width=image["width"],
-                        height=image["height"],
-                    )
-                case 2:  # 单个视频
-                    thumb = d["image_versions2"]["candidates"][0]["url"]
-                    video = d["video_versions"][0]["url"]
-                    media = ThreadsMedia(
-                        type=ThreadsMediaType.VIDEO,
-                        url=video,
-                        thumb_url=thumb,
-                        width=d["original_width"],
-                        height=d["original_height"],
-                    )
-                case 8:  # 多图/视频
-                    carousel_media = d["carousel_media"]
-                    media = []
-                    for m in carousel_media:
-                        if m["video_versions"]:
-                            thumb = m["image_versions2"]["candidates"][0]["url"]
-                            video = m["video_versions"][0]["url"]
-                            media.append(
-                                ThreadsMedia(
-                                    type=ThreadsMediaType.VIDEO,
-                                    url=video,
-                                    thumb_url=thumb,
-                                    width=m["original_width"],
-                                    height=m["original_height"],
-                                )
-                            )
-                        else:
-                            image = m["image_versions2"]["candidates"][0]["url"]
-                            media.append(
-                                ThreadsMedia(
-                                    type=ThreadsMediaType.IMAGE,
-                                    url=image,
-                                    thumb_url=image,
-                                    width=m["original_width"],
-                                    height=m["original_height"],
-                                )
-                            )
-                case 19:  # 纯文本/外部链接
-                    if linked_inline_media := d["text_post_app_info"]["linked_inline_media"]:
-                        media = fn(linked_inline_media)
-                    else:
-                        media = []
-                case _:
-                    media = []
-            return media
-
-        return fn(data)