parsehub 2.2.4__tar.gz → 2.3.1__tar.gz

{parsehub-2.2.4/src/parsehub.egg-info → parsehub-2.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: parsehub
-Version: 2.2.4
+Version: 2.3.1
 Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
 Author-email: 梓澪 <zilingmio@gmail.com>
 License: MIT
@@ -36,6 +36,7 @@ Requires-Dist: opencv-python-headless>=4.13.0.92
 Requires-Dist: cryptography>=49.0.0
 Requires-Dist: gmssl>=3.2.2
 Requires-Dist: signerpy>=0.12.0
+Requires-Dist: curl-cffi>=0.13
 Provides-Extra: cli
 Requires-Dist: argcomplete>=3.6.3; extra == "cli"
 Requires-Dist: platformdirs>=4.5.1; extra == "cli"
@@ -95,6 +96,9 @@ Dynamic: license-file
 | **Snapchat**    |  ✅  |      |                           |
 | **知乎**        |  ✅  |  ✅  | 🐶 问答, 专栏, 圈子, 日报 |
 | **豆瓣**        |  ✅  |  ✅  | 👥 小组话题               |
+| **Bangumi**     |      |  ✅  | 💬 条目讨论, 日志, 小组话题 |
+| **今日头条**    |  ✅  |  ✅  | 📰 文章, 微头条 |
+| **Medium**      |  ✅  |  ✅  | 📰 文章 |
 
 ## 📦 安装
 

{parsehub-2.2.4 → parsehub-2.3.1}/README.md

@@ -52,6 +52,9 @@
 | **Snapchat**    |  ✅  |      |                           |
 | **知乎**        |  ✅  |  ✅  | 🐶 问答, 专栏, 圈子, 日报 |
 | **豆瓣**        |  ✅  |  ✅  | 👥 小组话题               |
+| **Bangumi**     |      |  ✅  | 💬 条目讨论, 日志, 小组话题 |
+| **今日头条**    |  ✅  |  ✅  | 📰 文章, 微头条 |
+| **Medium**      |  ✅  |  ✅  | 📰 文章 |
 
 ## 📦 安装
 

{parsehub-2.2.4 → parsehub-2.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "parsehub"
-version = "2.2.4"
+version = "2.3.1"
 description = "轻量、异步、开箱即用的社交媒体聚合解析库"
 readme = "README.md"
 requires-python = ">=3.12.0"
@@ -39,6 +39,7 @@ dependencies = [
     "cryptography>=49.0.0",
     "gmssl>=3.2.2",
     "signerpy>=0.12.0",
+    "curl-cffi>=0.13",
 ]
 
 [project.scripts]

{parsehub-2.2.4 → parsehub-2.3.1}/src/parsehub/errors.py

@@ -11,6 +11,12 @@ class ParseError(ParseHubError):
     MSG = __doc__
 
 
+class ContentUnavailableError(ParseError):
+    """作品不可用"""
+
+    MSG = __doc__
+
+
 class DownloadError(ParseHubError):
     """下载错误"""
 

{parsehub-2.2.4 → parsehub-2.3.1}/src/parsehub/parsers/parser/__init__.py

@@ -1,12 +1,15 @@
 # ruff: noqa: F403
+from .bangumi import *
 from .bilibili import *
 from .coolapk import *
 from .douban import *
 from .douyin import *
 from .facebook import *
 from .instagram import *
+from .medium import *
 from .threads import *
 from .tieba import *
+from .toutiao import *
 from .twitter import *
 from .weibo import *
 from .weixin import *

parsehub-2.3.1/src/parsehub/parsers/parser/bangumi.py

@@ -0,0 +1,27 @@
+from ...provider_api.bangumi import Bangumi, BangumiError
+from ...types import ImageRef, ParseError, Platform, RichTextParseResult
+from ..base.base import BaseParser
+
+
+class BangumiParser(BaseParser):
+    __platform__ = Platform.BANGUMI
+    __supported_type__ = ["条目讨论", "日志", "小组话题"]
+    __match__ = (
+        r"^(http(s)?://)?(www\.)?(bgm\.tv|bangumi\.tv|chii\.in)/"
+        r"(subject/topic|group/topic|rakuen/topic/(subject|group)|blog)/\d+"
+    )
+
+    async def _do_parse(self, raw_url: str) -> RichTextParseResult:
+        try:
+            post = await Bangumi(proxy=self.proxy).parse(raw_url)
+        except BangumiError as e:
+            raise ParseError(f"Bangumi 解析失败: {e.msg}") from e
+
+        return RichTextParseResult(
+            title=post.title,
+            media=[ImageRef(url=url) for url in post.images],
+            markdown_content=post.markdown_content,
+        )
+
+
+__all__ = ["BangumiParser"]

{parsehub-2.2.4 → parsehub-2.3.1}/src/parsehub/parsers/parser/douyin.py

@@ -6,6 +6,7 @@ from typing import Self, Union
 from ... import ProgressCallback
 from ...provider_api.douyin import DouyinMobileCrawler, DouyinMobileDevice, DouyinWebCrawler
 from ...types import (
+    ContentUnavailableError,
     DownloadResult,
     ImageParseResult,
     ImageRef,
@@ -46,6 +47,8 @@ class DouyinParser(BaseParser):
                 web_crawler = DouyinWebCrawler(proxy=self.proxy, cookie=web_cookie)
                 response = await web_crawler.parse(raw_url)
                 return DouyinApiResult.parse(response)
+            except ContentUnavailableError:
+                raise
             except ParseError as e:
                 web_error = e
 
@@ -57,6 +60,8 @@ class DouyinParser(BaseParser):
         except ParseError as e:
             mobile_error = e
 
+        if isinstance(mobile_error, ContentUnavailableError):
+            raise mobile_error
         if web_error:
             raise ParseError(f"{web_error}；移动端解析也失败: {mobile_error}") from mobile_error
         raise mobile_error
@@ -188,6 +193,10 @@ class DouyinApiResult:
     def parse(cls, json_dict: dict) -> Self:
         data = json_dict.get("aweme_detail")
         if not data:
+            filter_detail = json_dict.get("filter_detail") or {}
+            if reason := filter_detail.get("filter_reason"):
+                detail_msg = filter_detail.get("detail_msg") or filter_detail.get("notice") or ""
+                raise ContentUnavailableError(f"抖音 {reason} {detail_msg}".strip())
             raise ParseError("抖音解析失败: 未获取到作品详情")
 
         desc = data.get("desc", "")

parsehub-2.3.1/src/parsehub/parsers/parser/medium.py

@@ -0,0 +1,24 @@
+from ...provider_api.medium import Medium, MediumError
+from ...types import ImageRef, ParseError, Platform, RichTextParseResult
+from ..base.base import BaseParser
+
+
+class MediumParser(BaseParser):
+    __platform__ = Platform.MEDIUM
+    __supported_type__ = ["文章"]
+    __match__ = r"^(http(s)?://)?([\w-]+\.)?medium\.com/(.+/)?([^/?#]*-)?[0-9a-f]{8,12}(/|\?|#|$)"
+
+    async def _do_parse(self, raw_url: str) -> RichTextParseResult:
+        try:
+            post = await Medium(proxy=self.proxy).parse(raw_url)
+        except MediumError as e:
+            raise ParseError(f"Medium 解析失败: {e}") from e
+
+        return RichTextParseResult(
+            title=post.title,
+            media=[ImageRef(url=image.url, width=image.width, height=image.height) for image in post.images],
+            markdown_content=post.markdown_content,
+        )
+
+
+__all__ = ["MediumParser"]

parsehub-2.3.1/src/parsehub/parsers/parser/toutiao.py

@@ -0,0 +1,69 @@
+from ...provider_api.toutiao import Toutiao, ToutiaoArticle, ToutiaoError, ToutiaoVideoPost
+from ...types import (
+    AnyMediaRef,
+    ImageParseResult,
+    ImageRef,
+    ParseError,
+    Platform,
+    RichTextParseResult,
+    VideoParseResult,
+    VideoRef,
+)
+from ..base.base import BaseParser
+
+
+class ToutiaoParser(BaseParser):
+    __platform__ = Platform.TOUTIAO
+    __supported_type__ = ["文章", "视频", "微头条"]
+    __match__ = (
+        r"^(http(s)?://)?(www|m)\.toutiao\.com/"
+        r"(article/\d+|video/\d+|w/\d+|group/\d+|item/\d+|a\d+|i\d+|is/\w+)"
+    )
+    __redirect_keywords__ = ["toutiao.com/is/"]
+
+    async def _do_parse(self, raw_url: str) -> ImageParseResult | RichTextParseResult | VideoParseResult:
+        try:
+            post = await Toutiao(proxy=self.proxy).parse(raw_url)
+        except ToutiaoError as e:
+            raise ParseError(f"今日头条解析失败: {e}") from e
+
+        if isinstance(post, ToutiaoVideoPost):
+            video = post.video
+            return VideoParseResult(
+                title=post.title,
+                video=VideoRef(
+                    url=video.url,
+                    thumb_url=video.thumb_url,
+                    duration=video.duration,
+                    width=video.width,
+                    height=video.height,
+                ),
+            )
+        if isinstance(post, ToutiaoArticle):
+            media: list[AnyMediaRef] = [
+                ImageRef(url=image.url, width=image.width, height=image.height) for image in post.images
+            ]
+            media.extend(
+                VideoRef(
+                    url=video.url,
+                    thumb_url=video.thumb_url,
+                    duration=video.duration,
+                    width=video.width,
+                    height=video.height,
+                )
+                for video in post.videos
+            )
+            return RichTextParseResult(
+                title=post.title,
+                media=media,
+                markdown_content=post.markdown_content,
+            )
+
+        return ImageParseResult(
+            title="",
+            content=post.content,
+            photo=[ImageRef(url=image.url, width=image.width, height=image.height) for image in post.images] or None,
+        )
+
+
+__all__ = ["ToutiaoParser"]

parsehub-2.3.1/src/parsehub/provider_api/bangumi.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+import httpx
+
+from ..utils.helpers import UA
+
+_TAG_RE = re.compile(
+    r"\[(/?)(img|url|b|i|s|u|mask|quote|code|user|size|color|align|left|right|center|photo)(?:=([^\]]*))?\]",
+    re.IGNORECASE,
+)
+_URL_RE = re.compile(r"/(?:(subject|group)/topic|rakuen/topic/(subject|group)|(blog))/(\d+)(?:[/?#]|$)")
+_API_BASE = "https://next.bgm.tv/p1"
+
+
+@dataclass
+class _BBCodeNode:
+    tag: str
+    value: str | None = None
+    children: list[str | _BBCodeNode] = field(default_factory=list)
+
+
+@dataclass
+class BangumiPost:
+    title: str
+    markdown_content: str
+    images: list[str]
+
+
+class Bangumi:
+    def __init__(self, proxy: str | None = None):
+        self.proxy = proxy
+
+    async def parse(self, url: str) -> BangumiPost:
+        match = _URL_RE.search(url)
+        if not match:
+            raise BangumiError("不支持的 Bangumi 链接")
+
+        kind = match.group(1) or match.group(2) or match.group(3)
+        topic_id = match.group(4)
+        assert kind is not None and topic_id is not None
+        async with httpx.AsyncClient(proxy=self.proxy, timeout=30, headers={"User-Agent": UA}) as client:
+            if kind == "blog":
+                blog = await self._get_json(client, f"{_API_BASE}/blogs/{topic_id}")
+                photos = await self._get_json(client, f"{_API_BASE}/blogs/{topic_id}/photos", params={"limit": 100})
+                markdown_content, images = convert_bbcode(blog.get("content") or "")
+                photo_urls = [
+                    f"https://lain.bgm.tv/pic/photo/l/{photo['target'].lstrip('/')}"
+                    for photo in photos.get("data", [])
+                    if photo.get("target")
+                ]
+                content_images = set(images)
+                extra_photo_urls = list(dict.fromkeys(url for url in photo_urls if url not in content_images))
+                images = list(dict.fromkeys([*images, *photo_urls]))
+                markdown_content += "".join(f"\n\n![]({url})" for url in extra_photo_urls)
+                return BangumiPost(
+                    title=blog.get("title") or "",
+                    markdown_content=markdown_content,
+                    images=images,
+                )
+
+            api_type = "subjects" if kind == "subject" else "groups"
+            topic = await self._get_json(client, f"{_API_BASE}/{api_type}/-/topics/{topic_id}")
+            replies = topic.get("replies") or []
+            content = replies[0].get("content", "") if replies else ""
+            markdown_content, images = convert_bbcode(content)
+            return BangumiPost(
+                title=topic.get("title") or "",
+                markdown_content=markdown_content,
+                images=images,
+            )
+
+    @staticmethod
+    async def _get_json(client: httpx.AsyncClient, url: str, *, params: dict[str, int] | None = None) -> dict:
+        response = await client.get(url, params=params)
+        if not response.is_success:
+            try:
+                payload = response.json()
+            except ValueError:
+                payload = {}
+            message = payload.get("message") if isinstance(payload, dict) else None
+            raise BangumiError(str(message or f"HTTP {response.status_code}"))
+        payload = response.json()
+        if not isinstance(payload, dict):
+            raise BangumiError(f"HTTP {response.status_code}: invalid JSON object")
+        return payload
+
+
+def convert_bbcode(content: str) -> tuple[str, list[str]]:
+    content = content.replace("\r\n", "\n")
+
+    root = _BBCodeNode("")
+    stack = [root]
+    position = 0
+    for match in _TAG_RE.finditer(content):
+        stack[-1].children.append(content[position : match.start()])
+        closing, tag, value = match.groups()
+        tag = tag.lower()
+        if closing:
+            for index in range(len(stack) - 1, 0, -1):
+                if stack[index].tag == tag:
+                    del stack[index:]
+                    break
+        else:
+            node = _BBCodeNode(tag=tag, value=value)
+            stack[-1].children.append(node)
+            stack.append(node)
+        position = match.end()
+    stack[-1].children.append(content[position:])
+
+    images: list[str] = []
+    markdown_content = _render_nodes(root.children, images)
+    return markdown_content.strip(), images
+
+
+def _render_nodes(nodes: list[str | _BBCodeNode], images: list[str]) -> str:
+    rendered: list[str] = []
+    for node in nodes:
+        if isinstance(node, str):
+            rendered.append(node)
+            continue
+
+        body = _render_nodes(node.children, images)
+        match node.tag:
+            case "img":
+                url = body.strip()
+                if url:
+                    images.append(url)
+                    rendered.append(f"![]({url})")
+            case "photo":
+                photo_path = body.strip().lstrip("/")
+                if photo_path:
+                    url = f"https://lain.bgm.tv/pic/photo/l/{photo_path}"
+                    images.append(url)
+                    rendered.append(f"![]({url})")
+            case "url" if node.value:
+                rendered.append(f"[{body}]({node.value})")
+            case "url":
+                rendered.append(body)
+            case "b":
+                rendered.append(f"**{body}**")
+            case "i":
+                rendered.append(f"*{body}*")
+            case "s":
+                rendered.append(f"~~{body}~~")
+            case "quote":
+                quote = body.strip("\n")
+                rendered.append("\n".join(f"> {line}" for line in quote.split("\n")) if quote else "")
+            case "code":
+                rendered.append(f"```\n{body.strip('\n')}\n```")
+            case "user":
+                rendered.append(f"@{body}")
+            case _:
+                rendered.append(body)
+    return "".join(rendered)
+
+
+class BangumiError(Exception):
+    def __init__(self, msg: str):
+        self.msg = msg
+        super().__init__(msg)
+
+
+__all__ = ["Bangumi", "BangumiError", "BangumiPost", "convert_bbcode"]

{parsehub-2.2.4 → parsehub-2.3.1}/src/parsehub/provider_api/douyin.py

@@ -27,6 +27,8 @@ DEFAULT_USER_AGENT = (
 )
 
 POST_DETAIL = "https://www.douyin.com/aweme/v1/web/aweme/detail/"
+WEB_SIGN_SALT = "A96D855A08C0A9707F8BEF0D9A527E4E"
+WEB_SIGN_UIFID_COOKIES = ("uifid", "uifid_temp", "uifidtemp", "UIFID", "UIFID_TEMP", "UIFIDTEMP")
 MOBILE_BROWSER_USER_AGENT = (
     "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) "
     "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1"
@@ -754,6 +756,24 @@ class DouyinWebCrawler:
             "Referer": "https://www.douyin.com/",
         }
 
+    def _signed_query(self, params: dict[str, str]) -> tuple[str, dict[str, str]]:
+        uifid = next((v for name in WEB_SIGN_UIFID_COOKIES if (v := self.cookie.get(name))), None)
+        if not uifid:
+            raise ParseError("抖音 cookie 缺少 UIFID_TEMP, 无法生成网页签名")
+        pairs = [*params.items(), ("a_bogus", ABogus().get_value(params))]
+        if verify_fp := self.cookie.get("s_v_web_id"):
+            pairs += [("verifyFp", verify_fp), ("fp", verify_fp)]
+        timestamp = str(int(time.time()))
+        pairs += [("uifid", uifid), ("timestamp", timestamp)]
+        query = "&".join(f"{quote(k, safe='*-._')}={quote(str(v), safe='*-._')}" for k, v in pairs)
+        signature = hashlib.md5(f"{uifid}_{timestamp}_{WEB_SIGN_SALT}_{query}".encode()).hexdigest()
+        headers = {
+            "uifid": uifid,
+            "x-secsdk-web-signature": signature,
+            "x-secsdk-web-expire": timestamp,
+        }
+        return f"{query}&x-secsdk-web-signature={signature}", headers
+
     @staticmethod
     async def get_aweme_id(raw_url: str) -> str:
         for pattern in [
@@ -799,17 +819,23 @@ class DouyinWebCrawler:
             }
             for attempt in range(3):
                 try:
-                    a_bogus = ABogus().get_value(params)
-                    endpoint = f"{POST_DETAIL}?{urlencode(params)}&a_bogus={quote(a_bogus, safe='')}"
-                    response = await client.get(endpoint)
-                    response.raise_for_status()
-                    return cast(dict[str, Any], response.json())
-                except Exception as e:
+                    query, sign_headers = self._signed_query(params)
+                    response = await client.get(f"{POST_DETAIL}?{query}", headers=sign_headers)
+                except httpx.HTTPError as e:
                     if attempt + 1 < 3:
                         await asyncio.sleep(1)
                     else:
-                        raise ParseError("获取抖音作品失败, 请检查 cookie") from e
-            raise ParseError("获取抖音作品失败, 请检查 cookie")
+                        raise ParseError(f"抖音网页端请求失败: {e}") from e
+                    continue
+                if response.status_code != 200:
+                    raise ParseError(f"抖音网页端请求被拒绝: HTTP {response.status_code} {response.text[:80]}")
+                if not response.content:
+                    raise ParseError("抖音网页端返回空响应, cookie 可能已被限流")
+                try:
+                    return cast(dict[str, Any], response.json())
+                except (ValueError, UnicodeDecodeError) as e:
+                    raise ParseError("抖音网页端返回非 JSON 响应") from e
+            raise ParseError("抖音网页端请求失败: 未知错误")
 
     async def parse(self, raw_url: str) -> dict:
         aweme_id = await self.get_aweme_id(raw_url)
@@ -1101,11 +1127,23 @@ class DouyinMobileCrawler:
         cls._device_pool_index = (cls._device_pool_index + 1) % len(cls._device_pool)
         return device
 
+    @classmethod
+    def _evict_device(cls, device: DouyinMobileDevice) -> None:
+        if device in cls._device_pool:
+            cls._device_pool.remove(device)
+            cls._device_pool_index = cls._device_pool_index % len(cls._device_pool) if cls._device_pool else 0
+
     async def _ensure_device_pool(self, client: httpx.AsyncClient) -> None:
-        if self.__class__._device_pool:
+        cls = self.__class__
+        if len(cls._device_pool) >= MOBILE_DEVICE_POOL_SIZE:
             return
-        self.__class__._device_pool = await self.register_device_pool(client)
-        self.__class__._device_pool_index = 0
+        missing = MOBILE_DEVICE_POOL_SIZE - len(cls._device_pool)
+        new_devices = await self.register_device_pool(client, count=missing)
+        current_pool = cls._device_pool
+        cls._device_pool = (current_pool + [device for device in new_devices if device not in current_pool])[
+            :MOBILE_DEVICE_POOL_SIZE
+        ]
+        cls._device_pool_index = cls._device_pool_index % len(cls._device_pool) if cls._device_pool else 0
 
     async def _select_device(self, client: httpx.AsyncClient) -> DouyinMobileDevice:
         if self._fixed_device:
@@ -1124,6 +1162,7 @@ class DouyinMobileCrawler:
                 await self._select_device(client)
                 params = self._mobile_query(aweme_id)
                 query = urlencode(params)
+                replace_device = False
                 for profile in MOBILE_SIGN_PROFILES:
                     headers = self._signed_headers(params, profile)
                     for host in MOBILE_DETAIL_HOSTS:
@@ -1136,6 +1175,11 @@ class DouyinMobileCrawler:
                             continue
                         if not content:
                             last_error = f"{host} returned empty body"
+                            if not self._fixed_device:
+                                if self.device is not None:
+                                    self.__class__._evict_device(self.device)
+                                replace_device = True
+                                break
                             continue
                         try:
                             payload = response.json()
@@ -1145,7 +1189,11 @@ class DouyinMobileCrawler:
                         if payload.get("aweme_detail"):
                             await self._attach_story_default_play(payload["aweme_detail"])
                             return cast(dict[str, Any], payload)
+                        if payload.get("filter_detail"):
+                            return cast(dict[str, Any], payload)
                         last_error = f"{host} missing aweme_detail: {payload.get('status_msg') or payload}"
+                    if replace_device:
+                        break
                 await asyncio.sleep(0.15)
         raise ParseError(f"获取抖音作品失败: {last_error}")