parsehub 2.2.3__tar.gz → 2.3.0__tar.gz

{parsehub-2.2.3/src/parsehub.egg-info → parsehub-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: parsehub
-Version: 2.2.3
+Version: 2.3.0
 Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
 Author-email: 梓澪 <zilingmio@gmail.com>
 License: MIT
@@ -36,6 +36,7 @@ Requires-Dist: opencv-python-headless>=4.13.0.92
 Requires-Dist: cryptography>=49.0.0
 Requires-Dist: gmssl>=3.2.2
 Requires-Dist: signerpy>=0.12.0
+Requires-Dist: curl-cffi>=0.13
 Provides-Extra: cli
 Requires-Dist: argcomplete>=3.6.3; extra == "cli"
 Requires-Dist: platformdirs>=4.5.1; extra == "cli"
@@ -95,6 +96,9 @@ Dynamic: license-file
 | **Snapchat**    |  ✅  |      |                           |
 | **知乎**        |  ✅  |  ✅  | 🐶 问答, 专栏, 圈子, 日报 |
 | **豆瓣**        |  ✅  |  ✅  | 👥 小组话题               |
+| **Bangumi**     |      |  ✅  | 💬 条目讨论, 日志, 小组话题 |
+| **今日头条**    |  ✅  |  ✅  | 📰 文章, 微头条 |
+| **Medium**      |  ✅  |  ✅  | 📰 文章 |
 
 ## 📦 安装
 

{parsehub-2.2.3 → parsehub-2.3.0}/README.md

@@ -52,6 +52,9 @@
 | **Snapchat**    |  ✅  |      |                           |
 | **知乎**        |  ✅  |  ✅  | 🐶 问答, 专栏, 圈子, 日报 |
 | **豆瓣**        |  ✅  |  ✅  | 👥 小组话题               |
+| **Bangumi**     |      |  ✅  | 💬 条目讨论, 日志, 小组话题 |
+| **今日头条**    |  ✅  |  ✅  | 📰 文章, 微头条 |
+| **Medium**      |  ✅  |  ✅  | 📰 文章 |
 
 ## 📦 安装
 

{parsehub-2.2.3 → parsehub-2.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "parsehub"
-version = "2.2.3"
+version = "2.3.0"
 description = "轻量、异步、开箱即用的社交媒体聚合解析库"
 readme = "README.md"
 requires-python = ">=3.12.0"
@@ -39,6 +39,7 @@ dependencies = [
     "cryptography>=49.0.0",
     "gmssl>=3.2.2",
     "signerpy>=0.12.0",
+    "curl-cffi>=0.13",
 ]
 
 [project.scripts]

{parsehub-2.2.3 → parsehub-2.3.0}/src/parsehub/parsers/parser/__init__.py

@@ -1,12 +1,15 @@
 # ruff: noqa: F403
+from .bangumi import *
 from .bilibili import *
 from .coolapk import *
 from .douban import *
 from .douyin import *
 from .facebook import *
 from .instagram import *
+from .medium import *
 from .threads import *
 from .tieba import *
+from .toutiao import *
 from .twitter import *
 from .weibo import *
 from .weixin import *

parsehub-2.3.0/src/parsehub/parsers/parser/bangumi.py

@@ -0,0 +1,27 @@
+from ...provider_api.bangumi import Bangumi, BangumiError
+from ...types import ImageRef, ParseError, Platform, RichTextParseResult
+from ..base.base import BaseParser
+
+
+class BangumiParser(BaseParser):
+    __platform__ = Platform.BANGUMI
+    __supported_type__ = ["条目讨论", "日志", "小组话题"]
+    __match__ = (
+        r"^(http(s)?://)?(www\.)?(bgm\.tv|bangumi\.tv|chii\.in)/"
+        r"(subject/topic|group/topic|rakuen/topic/(subject|group)|blog)/\d+"
+    )
+
+    async def _do_parse(self, raw_url: str) -> RichTextParseResult:
+        try:
+            post = await Bangumi(proxy=self.proxy).parse(raw_url)
+        except BangumiError as e:
+            raise ParseError(f"Bangumi 解析失败: {e.msg}") from e
+
+        return RichTextParseResult(
+            title=post.title,
+            media=[ImageRef(url=url) for url in post.images],
+            markdown_content=post.markdown_content,
+        )
+
+
+__all__ = ["BangumiParser"]

parsehub-2.3.0/src/parsehub/parsers/parser/medium.py

@@ -0,0 +1,24 @@
+from ...provider_api.medium import Medium, MediumError
+from ...types import ImageRef, ParseError, Platform, RichTextParseResult
+from ..base.base import BaseParser
+
+
+class MediumParser(BaseParser):
+    __platform__ = Platform.MEDIUM
+    __supported_type__ = ["文章"]
+    __match__ = r"^(http(s)?://)?([\w-]+\.)?medium\.com/(.+/)?([^/?#]*-)?[0-9a-f]{8,12}(/|\?|#|$)"
+
+    async def _do_parse(self, raw_url: str) -> RichTextParseResult:
+        try:
+            post = await Medium(proxy=self.proxy).parse(raw_url)
+        except MediumError as e:
+            raise ParseError(f"Medium 解析失败: {e}") from e
+
+        return RichTextParseResult(
+            title=post.title,
+            media=[ImageRef(url=image.url, width=image.width, height=image.height) for image in post.images],
+            markdown_content=post.markdown_content,
+        )
+
+
+__all__ = ["MediumParser"]

parsehub-2.3.0/src/parsehub/parsers/parser/toutiao.py

@@ -0,0 +1,69 @@
+from ...provider_api.toutiao import Toutiao, ToutiaoArticle, ToutiaoError, ToutiaoVideoPost
+from ...types import (
+    AnyMediaRef,
+    ImageParseResult,
+    ImageRef,
+    ParseError,
+    Platform,
+    RichTextParseResult,
+    VideoParseResult,
+    VideoRef,
+)
+from ..base.base import BaseParser
+
+
+class ToutiaoParser(BaseParser):
+    __platform__ = Platform.TOUTIAO
+    __supported_type__ = ["文章", "视频", "微头条"]
+    __match__ = (
+        r"^(http(s)?://)?(www|m)\.toutiao\.com/"
+        r"(article/\d+|video/\d+|w/\d+|group/\d+|item/\d+|a\d+|i\d+|is/\w+)"
+    )
+    __redirect_keywords__ = ["toutiao.com/is/"]
+
+    async def _do_parse(self, raw_url: str) -> ImageParseResult | RichTextParseResult | VideoParseResult:
+        try:
+            post = await Toutiao(proxy=self.proxy).parse(raw_url)
+        except ToutiaoError as e:
+            raise ParseError(f"今日头条解析失败: {e}") from e
+
+        if isinstance(post, ToutiaoVideoPost):
+            video = post.video
+            return VideoParseResult(
+                title=post.title,
+                video=VideoRef(
+                    url=video.url,
+                    thumb_url=video.thumb_url,
+                    duration=video.duration,
+                    width=video.width,
+                    height=video.height,
+                ),
+            )
+        if isinstance(post, ToutiaoArticle):
+            media: list[AnyMediaRef] = [
+                ImageRef(url=image.url, width=image.width, height=image.height) for image in post.images
+            ]
+            media.extend(
+                VideoRef(
+                    url=video.url,
+                    thumb_url=video.thumb_url,
+                    duration=video.duration,
+                    width=video.width,
+                    height=video.height,
+                )
+                for video in post.videos
+            )
+            return RichTextParseResult(
+                title=post.title,
+                media=media,
+                markdown_content=post.markdown_content,
+            )
+
+        return ImageParseResult(
+            title="",
+            content=post.content,
+            photo=[ImageRef(url=image.url, width=image.width, height=image.height) for image in post.images] or None,
+        )
+
+
+__all__ = ["ToutiaoParser"]

{parsehub-2.2.3 → parsehub-2.3.0}/src/parsehub/parsers/parser/xhs.py

@@ -14,6 +14,7 @@ from ...types import (
     VideoParseResult,
     VideoRef,
 )
+from ...utils.helpers import UA
 from ..base import BaseParser
 
 
@@ -25,6 +26,11 @@ class XHSParser(BaseParser):
     __after_clean_parameters__ = ["xsec_token"]
 
     async def get_raw_url(self, url: str, *, clean_all: bool = False, headers: dict | None = None) -> str:
+        headers = {
+            "User-Agent": UA,
+            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,"
+            "*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        }
         url = await super().get_raw_url(url, clean_all=clean_all, headers=headers)
         if "/login" in url:
             raise ParseError("该帖子需要登录后查看")

parsehub-2.3.0/src/parsehub/provider_api/bangumi.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+import httpx
+
+from ..utils.helpers import UA
+
+_TAG_RE = re.compile(
+    r"\[(/?)(img|url|b|i|s|u|mask|quote|code|user|size|color|align|left|right|center|photo)(?:=([^\]]*))?\]",
+    re.IGNORECASE,
+)
+_URL_RE = re.compile(r"/(?:(subject|group)/topic|rakuen/topic/(subject|group)|(blog))/(\d+)(?:[/?#]|$)")
+_API_BASE = "https://next.bgm.tv/p1"
+
+
+@dataclass
+class _BBCodeNode:
+    tag: str
+    value: str | None = None
+    children: list[str | _BBCodeNode] = field(default_factory=list)
+
+
+@dataclass
+class BangumiPost:
+    title: str
+    markdown_content: str
+    images: list[str]
+
+
+class Bangumi:
+    def __init__(self, proxy: str | None = None):
+        self.proxy = proxy
+
+    async def parse(self, url: str) -> BangumiPost:
+        match = _URL_RE.search(url)
+        if not match:
+            raise BangumiError("不支持的 Bangumi 链接")
+
+        kind = match.group(1) or match.group(2) or match.group(3)
+        topic_id = match.group(4)
+        assert kind is not None and topic_id is not None
+        async with httpx.AsyncClient(proxy=self.proxy, timeout=30, headers={"User-Agent": UA}) as client:
+            if kind == "blog":
+                blog = await self._get_json(client, f"{_API_BASE}/blogs/{topic_id}")
+                photos = await self._get_json(client, f"{_API_BASE}/blogs/{topic_id}/photos", params={"limit": 100})
+                markdown_content, images = convert_bbcode(blog.get("content") or "")
+                photo_urls = [
+                    f"https://lain.bgm.tv/pic/photo/l/{photo['target'].lstrip('/')}"
+                    for photo in photos.get("data", [])
+                    if photo.get("target")
+                ]
+                content_images = set(images)
+                extra_photo_urls = list(dict.fromkeys(url for url in photo_urls if url not in content_images))
+                images = list(dict.fromkeys([*images, *photo_urls]))
+                markdown_content += "".join(f"\n\n![]({url})" for url in extra_photo_urls)
+                return BangumiPost(
+                    title=blog.get("title") or "",
+                    markdown_content=markdown_content,
+                    images=images,
+                )
+
+            api_type = "subjects" if kind == "subject" else "groups"
+            topic = await self._get_json(client, f"{_API_BASE}/{api_type}/-/topics/{topic_id}")
+            replies = topic.get("replies") or []
+            content = replies[0].get("content", "") if replies else ""
+            markdown_content, images = convert_bbcode(content)
+            return BangumiPost(
+                title=topic.get("title") or "",
+                markdown_content=markdown_content,
+                images=images,
+            )
+
+    @staticmethod
+    async def _get_json(client: httpx.AsyncClient, url: str, *, params: dict[str, int] | None = None) -> dict:
+        response = await client.get(url, params=params)
+        if not response.is_success:
+            try:
+                payload = response.json()
+            except ValueError:
+                payload = {}
+            message = payload.get("message") if isinstance(payload, dict) else None
+            raise BangumiError(str(message or f"HTTP {response.status_code}"))
+        payload = response.json()
+        if not isinstance(payload, dict):
+            raise BangumiError(f"HTTP {response.status_code}: invalid JSON object")
+        return payload
+
+
+def convert_bbcode(content: str) -> tuple[str, list[str]]:
+    content = content.replace("\r\n", "\n")
+
+    root = _BBCodeNode("")
+    stack = [root]
+    position = 0
+    for match in _TAG_RE.finditer(content):
+        stack[-1].children.append(content[position : match.start()])
+        closing, tag, value = match.groups()
+        tag = tag.lower()
+        if closing:
+            for index in range(len(stack) - 1, 0, -1):
+                if stack[index].tag == tag:
+                    del stack[index:]
+                    break
+        else:
+            node = _BBCodeNode(tag=tag, value=value)
+            stack[-1].children.append(node)
+            stack.append(node)
+        position = match.end()
+    stack[-1].children.append(content[position:])
+
+    images: list[str] = []
+    markdown_content = _render_nodes(root.children, images)
+    return markdown_content.strip(), images
+
+
+def _render_nodes(nodes: list[str | _BBCodeNode], images: list[str]) -> str:
+    rendered: list[str] = []
+    for node in nodes:
+        if isinstance(node, str):
+            rendered.append(node)
+            continue
+
+        body = _render_nodes(node.children, images)
+        match node.tag:
+            case "img":
+                url = body.strip()
+                if url:
+                    images.append(url)
+                    rendered.append(f"![]({url})")
+            case "photo":
+                photo_path = body.strip().lstrip("/")
+                if photo_path:
+                    url = f"https://lain.bgm.tv/pic/photo/l/{photo_path}"
+                    images.append(url)
+                    rendered.append(f"![]({url})")
+            case "url" if node.value:
+                rendered.append(f"[{body}]({node.value})")
+            case "url":
+                rendered.append(body)
+            case "b":
+                rendered.append(f"**{body}**")
+            case "i":
+                rendered.append(f"*{body}*")
+            case "s":
+                rendered.append(f"~~{body}~~")
+            case "quote":
+                quote = body.strip("\n")
+                rendered.append("\n".join(f"> {line}" for line in quote.split("\n")) if quote else "")
+            case "code":
+                rendered.append(f"```\n{body.strip('\n')}\n```")
+            case "user":
+                rendered.append(f"@{body}")
+            case _:
+                rendered.append(body)
+    return "".join(rendered)
+
+
+class BangumiError(Exception):
+    def __init__(self, msg: str):
+        self.msg = msg
+        super().__init__(msg)
+
+
+__all__ = ["Bangumi", "BangumiError", "BangumiPost", "convert_bbcode"]

parsehub-2.3.0/src/parsehub/provider_api/medium.py

@@ -0,0 +1,256 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import urlparse
+
+from curl_cffi import requests
+
+_ID_RE = re.compile(r"(?:^|-)([0-9a-f]{8,12})$")
+_GRAPHQL_URL = "https://medium.com/_/graphql"
+_GRAPHQL_QUERY = """
+query P($id: ID!) {
+  post(id: $id) {
+    title
+    isLocked
+    content {
+      bodyModel {
+        paragraphs {
+          type
+          text
+          href
+          markups {
+            type
+            start
+            end
+            href
+          }
+          metadata {
+            id
+            originalWidth
+            originalHeight
+          }
+          codeBlockMetadata {
+            lang
+          }
+          iframe {
+            mediaResource {
+              href
+              iframeSrc
+              title
+            }
+          }
+          mixtapeMetadata {
+            href
+          }
+        }
+      }
+    }
+  }
+}
+"""
+
+
+@dataclass
+class MediumImage:
+    url: str
+    width: int = 0
+    height: int = 0
+
+
+@dataclass
+class MediumPost:
+    title: str
+    markdown_content: str
+    images: list[MediumImage]
+
+
+class Medium:
+    def __init__(self, proxy: str | None = None):
+        self.proxy = proxy
+
+    async def parse(self, url: str) -> MediumPost:
+        path = urlparse(url).path.rstrip("/")
+        path_segment = path.rsplit("/", 1)[-1]
+        match = _ID_RE.search(path_segment)
+        if not match:
+            raise MediumError("不支持的 Medium 链接")
+        post_id = match.group(1)
+
+        async with requests.AsyncSession(impersonate="chrome", proxy=self.proxy, timeout=30) as session:
+            response = await session.post(
+                _GRAPHQL_URL,
+                json=[
+                    {
+                        "operationName": "P",
+                        "variables": {"id": post_id},
+                        "query": _GRAPHQL_QUERY,
+                    }
+                ],
+            )
+            if not 200 <= response.status_code < 300:
+                raise MediumError(f"HTTP {response.status_code}")
+
+            result = response.json()[0]
+            errors = result.get("errors")
+            if errors:
+                raise MediumError(errors[0]["message"])
+
+            post = result["data"]["post"]
+            if post is None:
+                raise MediumError("内容不存在或已删除")
+
+            paragraphs = ((post.get("content") or {}).get("bodyModel") or {}).get("paragraphs") or []
+            markdown_content, images = _render_paragraphs(paragraphs, post.get("title") or "")
+            return MediumPost(
+                title=post.get("title") or "",
+                markdown_content=markdown_content,
+                images=images,
+            )
+
+
+def _render_paragraphs(paragraphs: list[dict[str, Any]], title: str) -> tuple[str, list[MediumImage]]:
+    images: list[MediumImage] = []
+    rendered_paragraphs: list[tuple[str, str]] = []
+    ordered_count = 0
+
+    for index, paragraph in enumerate(paragraphs):
+        paragraph_type = paragraph.get("type") or ""
+        text = paragraph.get("text") or ""
+        if index == 0 and paragraph_type in {"H2", "H3", "H4"} and text.strip() == title.strip():
+            continue
+
+        content = _render_paragraph(paragraph, images)
+        if content is None or not content.strip():
+            continue
+
+        if paragraph_type == "ULI":
+            content = f"- {content}"
+            ordered_count = 0
+        elif paragraph_type == "OLI":
+            ordered_count = ordered_count + 1 if rendered_paragraphs and rendered_paragraphs[-1][0] == "OLI" else 1
+            content = f"{ordered_count}. {content}"
+        else:
+            ordered_count = 0
+
+        rendered_paragraphs.append((paragraph_type, content))
+
+    markdown_parts: list[str] = []
+    previous_type = ""
+    for paragraph_type, content in rendered_paragraphs:
+        if markdown_parts:
+            is_consecutive_list = paragraph_type in {"ULI", "OLI"} and previous_type in {"ULI", "OLI"}
+            markdown_parts.append("\n" if is_consecutive_list else "\n\n")
+        markdown_parts.append(content)
+        previous_type = paragraph_type
+
+    return "".join(markdown_parts).strip(), images
+
+
+def _render_paragraph(paragraph: dict[str, Any], images: list[MediumImage]) -> str | None:
+    paragraph_type = paragraph.get("type") or ""
+    text = paragraph.get("text") or ""
+    if paragraph_type == "IMG":
+        metadata = paragraph.get("metadata") or {}
+        image_id = metadata.get("id")
+        if not image_id:
+            return None
+        image = MediumImage(
+            url=f"https://miro.medium.com/v2/{image_id}",
+            width=int(metadata.get("originalWidth") or 0),
+            height=int(metadata.get("originalHeight") or 0),
+        )
+        images.append(image)
+        markdown = f"![]({image.url})"
+        caption = text.strip()
+        if caption:
+            markdown += f"\n*{caption}*"
+        return markdown
+
+    if paragraph_type == "PRE":
+        language = (paragraph.get("codeBlockMetadata") or {}).get("lang") or ""
+        return f"```{language}\n{text}\n```"
+
+    if paragraph_type == "IFRAME":
+        media_resource = (paragraph.get("iframe") or {}).get("mediaResource") or {}
+        href = media_resource.get("href") or media_resource.get("iframeSrc")
+        if not href:
+            return None
+        return f"[{media_resource.get('title') or href}]({href})"
+
+    if paragraph_type == "MIXTAPE_EMBED":
+        href = (paragraph.get("mixtapeMetadata") or {}).get("href")
+        linked_text = _render_markup(text, paragraph.get("markups") or [])
+        return f"[{linked_text}]({href})" if href else linked_text
+
+    rendered_text = _render_markup(text, paragraph.get("markups") or [])
+    match paragraph_type:
+        case "H2":
+            return f"## {rendered_text}"
+        case "H3":
+            return f"### {rendered_text}"
+        case "H4":
+            return f"#### {rendered_text}"
+        case "BQ" | "PQ":
+            return "\n".join(f"> {line}" for line in rendered_text.split("\n"))
+        case _:
+            return rendered_text
+
+
+def _render_markup(text: str, markups: list[dict[str, Any]]) -> str:
+    encoded_text = text.encode("utf-16-le")
+    text_length = len(encoded_text) // 2
+    openings: dict[int, list[tuple[int, int, str]]] = {}
+    closings: dict[int, list[tuple[int, int, str]]] = {}
+
+    for index, markup in enumerate(markups):
+        markup_type = markup.get("type")
+        match markup_type:
+            case "STRONG":
+                opening, closing = "**", "**"
+            case "EM":
+                opening, closing = "*", "*"
+            case "CODE":
+                opening, closing = "`", "`"
+            case "A" if markup.get("href"):
+                opening, closing = "[", f"]({markup['href']})"
+            case _:
+                continue
+
+        start = max(0, min(int(markup.get("start") or 0), text_length))
+        end = max(0, min(int(markup.get("end") or 0), text_length))
+        if start >= end:
+            continue
+
+        selected_text = encoded_text[start * 2 : end * 2].decode("utf-16-le")
+        left_trim = selected_text[: len(selected_text) - len(selected_text.lstrip())]
+        right_trim = selected_text[len(selected_text.rstrip()) :]
+        start += len(left_trim.encode("utf-16-le")) // 2
+        end -= len(right_trim.encode("utf-16-le")) // 2
+        if start >= end:
+            continue
+
+        openings.setdefault(start, []).append((end, index, opening))
+        closings.setdefault(end, []).append((start, index, closing))
+
+    rendered: list[str] = []
+    position = 0
+    for boundary in sorted(openings.keys() | closings.keys()):
+        rendered.append(encoded_text[position * 2 : boundary * 2].decode("utf-16-le"))
+        rendered.extend(
+            closing for _, _, closing in sorted(closings.get(boundary, []), key=lambda item: (-item[0], -item[1]))
+        )
+        rendered.extend(
+            opening for _, _, opening in sorted(openings.get(boundary, []), key=lambda item: (-item[0], item[1]))
+        )
+        position = boundary
+    rendered.append(encoded_text[position * 2 :].decode("utf-16-le"))
+    return "".join(rendered)
+
+
+class MediumError(Exception):
+    pass
+
+
+__all__ = ["Medium", "MediumError", "MediumImage", "MediumPost"]

parsehub-2.3.0/src/parsehub/provider_api/toutiao.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+import base64
+import json
+import re
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+from bs4 import BeautifulSoup
+from markdownify import MarkdownConverter
+
+from ..utils.helpers import UA
+
+_ID_RE = re.compile(r"toutiao\.com/(?:(?:article|video|w|group|item)/|a|i)(\d+)")
+_INFO_API = "https://m.toutiao.com/i{}/info/v2/"
+_VOD_API = "https://vod.bytedanceapi.com/?{}"
+
+
+@dataclass
+class ToutiaoImage:
+    url: str
+    width: int = 0
+    height: int = 0
+
+
+@dataclass
+class ToutiaoVideo:
+    url: str
+    width: int = 0
+    height: int = 0
+    duration: int = 0
+    thumb_url: str | None = None
+
+
+@dataclass
+class ToutiaoArticle:
+    title: str
+    markdown_content: str
+    images: list[ToutiaoImage]
+    videos: list[ToutiaoVideo]
+
+
+@dataclass
+class ToutiaoVideoPost:
+    title: str
+    video: ToutiaoVideo
+
+
+@dataclass
+class ToutiaoMicroPost:
+    content: str
+    images: list[ToutiaoImage]
+
+
+class Toutiao:
+    def __init__(self, proxy: str | None = None):
+        self.proxy = proxy
+
+    async def parse(self, url: str) -> ToutiaoArticle | ToutiaoVideoPost | ToutiaoMicroPost:
+        match = _ID_RE.search(url)
+        if not match:
+            raise ToutiaoError("不支持的今日头条链接")
+
+        async with httpx.AsyncClient(proxy=self.proxy, timeout=30, headers={"User-Agent": UA}) as client:
+            response = await client.get(_INFO_API.format(match.group(1)))
+            response.raise_for_status()
+            data = response.json().get("data")
+            if data is None:
+                raise ToutiaoError("内容不存在或已删除")
+
+            if "thread" in data:
+                return self._parse_micro_post(data["thread"])
+            if data.get("biz_tag") == "图文":
+                return await self._parse_article(client, data)
+            if data.get("play_auth_token_v2"):
+                video = await self._parse_video(
+                    client,
+                    data["play_auth_token_v2"],
+                    duration=int(data.get("video_duration") or 0),
+                    thumb_url=data.get("poster_url"),
+                )
+                return ToutiaoVideoPost(title=data.get("title") or "", video=video)
+            return await self._parse_article(client, data)
+
+    async def _parse_article(self, client: httpx.AsyncClient, data: dict[str, Any]) -> ToutiaoArticle:
+        soup = BeautifulSoup(data.get("content") or "", "lxml")
+        videos = []
+        for video_box in soup.find_all("div", class_="tt-video-box"):
+            token = str(video_box.get("data-token") or "")
+            videos.append(await self._parse_video(client, token))
+            video_box.decompose()
+
+        converter = _ToutiaoMarkdownConverter()
+        markdown_content = converter.convert(str(soup)).strip()
+        return ToutiaoArticle(
+            title=data.get("title") or "",
+            markdown_content=markdown_content,
+            images=[ToutiaoImage(url=url) for url in converter.images],
+            videos=videos,
+        )
+
+    @staticmethod
+    def _parse_micro_post(thread: dict[str, Any]) -> ToutiaoMicroPost:
+        thread_base = thread.get("thread_base") or {}
+        images = [
+            ToutiaoImage(
+                url=image["url"],
+                width=int(image.get("width") or 0),
+                height=int(image.get("height") or 0),
+            )
+            for image in thread_base.get("large_image_list", [])
+            if image.get("url")
+        ]
+        return ToutiaoMicroPost(content=(thread_base.get("content") or "").strip(), images=images)
+
+    async def _parse_video(
+        self,
+        client: httpx.AsyncClient,
+        token: str,
+        *,
+        duration: int = 0,
+        thumb_url: str | None = None,
+    ) -> ToutiaoVideo:
+        token_data = json.loads(base64.urlsafe_b64decode(token + "=" * (-len(token) % 4)))
+        play_info_token = token_data["GetPlayInfoToken"]
+        response = await client.get(_VOD_API.format(play_info_token))
+        response.raise_for_status()
+        payload = response.json()
+
+        error = (payload.get("ResponseMetadata") or {}).get("Error")
+        if error:
+            if isinstance(error, dict):
+                message = error.get("Message") or error.get("message") or error.get("ErrorMessage")
+            else:
+                message = error
+            raise ToutiaoError(str(message or error))
+
+        play_infos = payload["Result"]["Data"]["PlayInfoList"]
+        best = max(
+            play_infos,
+            key=lambda item: (int(item.get("Height") or 0), int(item.get("Size") or 0)),
+        )
+        return ToutiaoVideo(
+            url=best["MainPlayUrl"],
+            width=int(best.get("Width") or 0),
+            height=int(best.get("Height") or 0),
+            duration=duration or int(best.get("Duration") or 0),
+            thumb_url=thumb_url,
+        )
+
+
+class _ToutiaoMarkdownConverter(MarkdownConverter):
+    def __init__(self) -> None:
+        super().__init__(heading_style="ATX")
+        self.images: list[str] = []
+
+    def convert_img(self, el: Any, text: Any, parent_tags: Any) -> str:
+        alt = el.attrs.get("alt") or ""
+        src = el.attrs.get("src") or ""
+        title = el.attrs.get("title") or ""
+        if src:
+            self.images.append(src)
+        title_part = ' "{}"'.format(title.replace('"', r"\"")) if title else ""
+        return f"![{alt}]({src}{title_part})"
+
+
+class ToutiaoError(Exception):
+    pass
+
+
+__all__ = [
+    "Toutiao",
+    "ToutiaoArticle",
+    "ToutiaoError",
+    "ToutiaoImage",
+    "ToutiaoMicroPost",
+    "ToutiaoVideo",
+    "ToutiaoVideoPost",
+]

{parsehub-2.2.3 → parsehub-2.3.0}/src/parsehub/provider_api/xhs.py

@@ -10,9 +10,8 @@ from typing import Any, cast
 import httpx
 from bs4 import BeautifulSoup
 
-from parsehub.utils.helpers import UA
-
 from ..errors import ParseError
+from ..utils.helpers import UA
 
 
 class XHSAPI:
@@ -24,7 +23,11 @@ class XHSAPI:
         async with httpx.AsyncClient(proxy=self.proxy, cookies=self.cookie, follow_redirects=True) as client:
             result = await client.get(
                 url,
-                headers={"User-Agent": UA},
+                headers={
+                    "User-Agent": UA,
+                    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp"
+                    ",image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+                },
                 timeout=30,
             )
             if "/login" in str(result.url):

{parsehub-2.2.3 → parsehub-2.3.0}/src/parsehub/provider_api/zhihu.py

@@ -172,6 +172,13 @@ class ZhihuAPI:
         self.proxy = proxy
         self.cookie = cookie
 
+    @staticmethod
+    def _json(r: httpx.Response) -> dict:
+        data = dict(r.json())
+        if err := data.get("error"):
+            raise ValueError(f"知乎接口错误 {err.get('code')}: {err.get('message')}")
+        return data
+
     @property
     def d_c0(self) -> str:
         v = self.cookie.get("d_c0")
@@ -258,7 +265,7 @@ class ZhihuAPI:
 
         async with httpx.AsyncClient() as client:
             r = await client.get(url, headers=headers, params=query, cookies=self.cookie)
-            return dict(r.json())
+            return self._json(r)
 
     async def _questions_answers(self, question_id: int | str) -> dict:
         """获取问题的回答"""
@@ -274,7 +281,7 @@ class ZhihuAPI:
 
         async with httpx.AsyncClient() as client:
             r = await client.get(url, headers=headers, params=query, cookies=self.cookie)
-            return dict(r.json())
+            return self._json(r)
 
     async def _answers(self, answers_id: int | str) -> dict:
         """获取问题的指定回答"""
@@ -287,7 +294,7 @@ class ZhihuAPI:
 
         async with httpx.AsyncClient() as client:
             r = await client.get(url, headers=headers, params=query, cookies=self.cookie)
-            return dict(r.json())
+            return self._json(r)
 
     async def _zl(self, zl_id: int | str) -> dict:
         url = f"https://zhuanlan.zhihu.com/api/articles/{zl_id}"
@@ -297,7 +304,7 @@ class ZhihuAPI:
 
         async with httpx.AsyncClient() as client:
             r = await client.get(url, headers=headers, params=query, cookies=self.cookie)
-            return dict(r.json())
+            return self._json(r)
 
     async def _pin(self, pin_id: int | str) -> dict:
         url = f"https://www.zhihu.com/api/v4/pins/{pin_id}"
@@ -307,13 +314,13 @@ class ZhihuAPI:
 
         async with httpx.AsyncClient() as client:
             r = await client.get(url, headers=headers, params=query, cookies=self.cookie)
-            return dict(r.json())
+            return self._json(r)
 
     async def _daily(self, story_id: int | str) -> dict:
         url = f"https://daily.zhihu.com/api/7/story/{story_id}"
         async with httpx.AsyncClient(proxy=self.proxy) as client:
             r = await client.get(url)
-            return dict(r.json())
+            return self._json(r)
 
     @staticmethod
     def _get_qa_id(raw_url: str) -> tuple[str, str | None]:

{parsehub-2.2.3 → parsehub-2.3.0}/src/parsehub/types/platform.py

@@ -4,11 +4,14 @@ from enum import Enum
 class Platform(Enum):
     """支持的平台"""
 
+    BANGUMI = ("bangumi", "Bangumi")
     BILIBILI = ("bilibili", "Bilibili")
     COOLAPK = ("coolapk", "酷安")
     DOUBAN = ("douban", "豆瓣")
     DOUYIN = ("douyin", "抖音")
     TIKTOK = ("tiktok", "TikTok")
+    TOUTIAO = ("toutiao", "今日头条")
+    MEDIUM = ("medium", "Medium")
     FACEBOOK = ("facebook", "Facebook")
     INSTAGRAM = ("instagram", "Instagram")
     KUAISHOU = ("kuaishou", "快手")

{parsehub-2.2.3 → parsehub-2.3.0/src/parsehub.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: parsehub
-Version: 2.2.3
+Version: 2.3.0
 Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
 Author-email: 梓澪 <zilingmio@gmail.com>
 License: MIT
@@ -36,6 +36,7 @@ Requires-Dist: opencv-python-headless>=4.13.0.92
 Requires-Dist: cryptography>=49.0.0
 Requires-Dist: gmssl>=3.2.2
 Requires-Dist: signerpy>=0.12.0
+Requires-Dist: curl-cffi>=0.13
 Provides-Extra: cli
 Requires-Dist: argcomplete>=3.6.3; extra == "cli"
 Requires-Dist: platformdirs>=4.5.1; extra == "cli"
@@ -95,6 +96,9 @@ Dynamic: license-file
 | **Snapchat**    |  ✅  |      |                           |
 | **知乎**        |  ✅  |  ✅  | 🐶 问答, 专栏, 圈子, 日报 |
 | **豆瓣**        |  ✅  |  ✅  | 👥 小组话题               |
+| **Bangumi**     |      |  ✅  | 💬 条目讨论, 日志, 小组话题 |
+| **今日头条**    |  ✅  |  ✅  | 📰 文章, 微头条 |
+| **Medium**      |  ✅  |  ✅  | 📰 文章 |
 
 ## 📦 安装
 

{parsehub-2.2.3 → parsehub-2.3.0}/src/parsehub.egg-info/SOURCES.txt

@@ -19,6 +19,7 @@ src/parsehub/parsers/base/__init__.py
 src/parsehub/parsers/base/base.py
 src/parsehub/parsers/base/ytdlp.py
 src/parsehub/parsers/parser/__init__.py
+src/parsehub/parsers/parser/bangumi.py
 src/parsehub/parsers/parser/bilibili.py
 src/parsehub/parsers/parser/coolapk.py
 src/parsehub/parsers/parser/douban.py
@@ -26,11 +27,13 @@ src/parsehub/parsers/parser/douyin.py
 src/parsehub/parsers/parser/facebook.py
 src/parsehub/parsers/parser/instagram.py
 src/parsehub/parsers/parser/kuaishou.py
+src/parsehub/parsers/parser/medium.py
 src/parsehub/parsers/parser/pipix.py
 src/parsehub/parsers/parser/snapchat.py
 src/parsehub/parsers/parser/threads.py
 src/parsehub/parsers/parser/tieba.py
 src/parsehub/parsers/parser/tiktok.py
+src/parsehub/parsers/parser/toutiao.py
 src/parsehub/parsers/parser/twitter.py
 src/parsehub/parsers/parser/weibo.py
 src/parsehub/parsers/parser/weixin.py
@@ -40,16 +43,19 @@ src/parsehub/parsers/parser/youtube.py
 src/parsehub/parsers/parser/zhihu.py
 src/parsehub/parsers/parser/zuiyou.py
 src/parsehub/provider_api/__init__.py
+src/parsehub/provider_api/bangumi.py
 src/parsehub/provider_api/bilibili.py
 src/parsehub/provider_api/coolapk.py
 src/parsehub/provider_api/douban.py
 src/parsehub/provider_api/douyin.py
 src/parsehub/provider_api/instagram.py
 src/parsehub/provider_api/kuaishou.py
+src/parsehub/provider_api/medium.py
 src/parsehub/provider_api/pipix.py
 src/parsehub/provider_api/threads.py
 src/parsehub/provider_api/tieba.py
 src/parsehub/provider_api/tiktok.py
+src/parsehub/provider_api/toutiao.py
 src/parsehub/provider_api/twitter.py
 src/parsehub/provider_api/weibo.py
 src/parsehub/provider_api/weixin.py

{parsehub-2.2.3 → parsehub-2.3.0}/src/parsehub.egg-info/requires.txt

@@ -19,6 +19,7 @@ opencv-python-headless>=4.13.0.92
 cryptography>=49.0.0
 gmssl>=3.2.2
 signerpy>=0.12.0
+curl-cffi>=0.13
 
 [cli]
 argcomplete>=3.6.3