parsehub 2.1.10__tar.gz → 2.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {parsehub-2.1.10/src/parsehub.egg-info → parsehub-2.2.0}/PKG-INFO +14 -2
  2. {parsehub-2.1.10 → parsehub-2.2.0}/README.md +12 -0
  3. {parsehub-2.1.10 → parsehub-2.2.0}/pyproject.toml +2 -2
  4. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/__init__.py +1 -0
  5. parsehub-2.2.0/src/parsehub/parsers/parser/douban.py +86 -0
  6. parsehub-2.2.0/src/parsehub/provider_api/douban.py +159 -0
  7. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/types/platform.py +1 -0
  8. {parsehub-2.1.10 → parsehub-2.2.0/src/parsehub.egg-info}/PKG-INFO +14 -2
  9. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub.egg-info/SOURCES.txt +2 -0
  10. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub.egg-info/requires.txt +1 -1
  11. {parsehub-2.1.10 → parsehub-2.2.0}/test/test_core_offline.py +258 -2
  12. {parsehub-2.1.10 → parsehub-2.2.0}/LICENSE +0 -0
  13. {parsehub-2.1.10 → parsehub-2.2.0}/setup.cfg +0 -0
  14. {parsehub-2.1.10 → parsehub-2.2.0}/src/__init__.py +0 -0
  15. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/__init__.py +0 -0
  16. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/cli.py +0 -0
  17. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/cli_config.py +0 -0
  18. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/config/__init__.py +0 -0
  19. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/config/config.py +0 -0
  20. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/errors.py +0 -0
  21. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/__init__.py +0 -0
  22. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/base/__init__.py +0 -0
  23. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/base/base.py +0 -0
  24. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/base/ytdlp.py +0 -0
  25. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/bilibili.py +0 -0
  26. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/coolapk.py +0 -0
  27. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/douyin.py +0 -0
  28. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/facebook.py +0 -0
  29. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/instagram.py +0 -0
  30. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/kuaishou.py +0 -0
  31. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/pipix.py +0 -0
  32. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/snapchat.py +0 -0
  33. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/threads.py +0 -0
  34. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/tieba.py +0 -0
  35. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/tiktok.py +0 -0
  36. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/twitter.py +0 -0
  37. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/weibo.py +0 -0
  38. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/weixin.py +0 -0
  39. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/xhs.py +0 -0
  40. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/xiaoheihe.py +0 -0
  41. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/youtube.py +0 -0
  42. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/zhihu.py +0 -0
  43. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/parsers/parser/zuiyou.py +0 -0
  44. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/__init__.py +0 -0
  45. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/bilibili.py +0 -0
  46. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/coolapk.py +0 -0
  47. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/douyin.py +0 -0
  48. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/instagram.py +0 -0
  49. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/kuaishou.py +0 -0
  50. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/pipix.py +0 -0
  51. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/threads.py +0 -0
  52. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/tieba.py +0 -0
  53. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/tiktok.py +0 -0
  54. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/twitter.py +0 -0
  55. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/weibo.py +0 -0
  56. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/weixin.py +0 -0
  57. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/xhs.py +0 -0
  58. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/xiaoheihe.py +0 -0
  59. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/zhihu.py +0 -0
  60. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/provider_api/zuiyou.py +0 -0
  61. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/types/__init__.py +0 -0
  62. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/types/callback.py +0 -0
  63. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/types/media_file.py +0 -0
  64. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/types/media_ref.py +0 -0
  65. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/types/post.py +0 -0
  66. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/types/result.py +0 -0
  67. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/utils/downloader.py +0 -0
  68. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/utils/helpers.py +0 -0
  69. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub/utils/media_info.py +0 -0
  70. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub.egg-info/dependency_links.txt +0 -0
  71. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub.egg-info/entry_points.txt +0 -0
  72. {parsehub-2.1.10 → parsehub-2.2.0}/src/parsehub.egg-info/top_level.txt +0 -0
  73. {parsehub-2.1.10 → parsehub-2.2.0}/test/test_cli.py +0 -0
  74. {parsehub-2.1.10 → parsehub-2.2.0}/test/test_cli_config.py +0 -0
  75. {parsehub-2.1.10 → parsehub-2.2.0}/test/test_downloader.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: parsehub
3
- Version: 2.1.10
3
+ Version: 2.2.0
4
4
  Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
5
5
  Author-email: 梓澪 <zilingmio@gmail.com>
6
6
  License: MIT
@@ -22,7 +22,7 @@ Requires-Dist: pydub>=0.25.1
22
22
  Requires-Dist: python-dotenv>=1.0.1
23
23
  Requires-Dist: tenacity>=8.5.0
24
24
  Requires-Dist: urlextract>=1.9.0
25
- Requires-Dist: yt-dlp[default]
25
+ Requires-Dist: yt-dlp[default]>=2026.8.18.122307.dev0
26
26
  Requires-Dist: lxml>=5.3.0
27
27
  Requires-Dist: pydantic>=1.10.19
28
28
  Requires-Dist: markdownify>=1.1.0
@@ -93,6 +93,7 @@ Dynamic: license-file
93
93
  | **小黑盒** | ✅ | ✅ | |
94
94
  | **Snapchat** | ✅ | | |
95
95
  | **知乎** | ✅ | ✅ | 🐶 问答, 专栏, 圈子 |
96
+ | **豆瓣** | ✅ | ✅ | 👥 小组话题 |
96
97
 
97
98
  ## 📦 安装
98
99
 
@@ -215,6 +216,7 @@ print(result)
215
216
  - `快手`
216
217
  - `小红书`
217
218
  - `知乎`
219
+ - `豆瓣`
218
220
 
219
221
  ```python
220
222
  from parsehub import ParseHub
@@ -313,6 +315,16 @@ except ParseError as exc:
313
315
 
314
316
  ---
315
317
 
318
+ ### 启用日志
319
+
320
+ ```python
321
+ from loguru import logger
322
+
323
+ logger.enable('parsehub')
324
+ ```
325
+
326
+ ---
327
+
316
328
  ## 🤝 参与贡献
317
329
 
318
330
  欢迎提交 Pull Request 或 Issue!
@@ -51,6 +51,7 @@
51
51
  | **小黑盒** | ✅ | ✅ | |
52
52
  | **Snapchat** | ✅ | | |
53
53
  | **知乎** | ✅ | ✅ | 🐶 问答, 专栏, 圈子 |
54
+ | **豆瓣** | ✅ | ✅ | 👥 小组话题 |
54
55
 
55
56
  ## 📦 安装
56
57
 
@@ -173,6 +174,7 @@ print(result)
173
174
  - `快手`
174
175
  - `小红书`
175
176
  - `知乎`
177
+ - `豆瓣`
176
178
 
177
179
  ```python
178
180
  from parsehub import ParseHub
@@ -271,6 +273,16 @@ except ParseError as exc:
271
273
 
272
274
  ---
273
275
 
276
+ ### 启用日志
277
+
278
+ ```python
279
+ from loguru import logger
280
+
281
+ logger.enable('parsehub')
282
+ ```
283
+
284
+ ---
285
+
274
286
  ## 🤝 参与贡献
275
287
 
276
288
  欢迎提交 Pull Request 或 Issue!
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "parsehub"
3
- version = "2.1.10"
3
+ version = "2.2.0"
4
4
  description = "轻量、异步、开箱即用的社交媒体聚合解析库"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.12.0"
@@ -25,7 +25,7 @@ dependencies = [
25
25
  "python-dotenv>=1.0.1",
26
26
  "tenacity>=8.5.0",
27
27
  "urlextract>=1.9.0",
28
- "yt-dlp[default]",
28
+ "yt-dlp[default]>=2026.8.18.122307.dev0",
29
29
  "lxml>=5.3.0",
30
30
  "pydantic>=1.10.19",
31
31
  "markdownify>=1.1.0",
@@ -1,6 +1,7 @@
1
1
  # ruff: noqa: F403
2
2
  from .bilibili import *
3
3
  from .coolapk import *
4
+ from .douban import *
4
5
  from .douyin import *
5
6
  from .facebook import *
6
7
  from .instagram import *
@@ -0,0 +1,86 @@
1
+ from pathlib import Path
2
+
3
+ from ...provider_api.douban import IMAGE_REFERER, Douban, DoubanError, DoubanPhoto, DoubanVideo
4
+ from ...types import (
5
+ AniRef,
6
+ AnyMediaRef,
7
+ DownloadResult,
8
+ ImageRef,
9
+ ParseError,
10
+ Platform,
11
+ ProgressCallback,
12
+ RichTextParseResult,
13
+ VideoRef,
14
+ )
15
+ from ...utils.helpers import UA
16
+ from ..base.base import BaseParser
17
+
18
+
19
+ class DoubanParser(BaseParser):
20
+ __platform__ = Platform.DOUBAN
21
+ __supported_type__ = ["视频", "图文"]
22
+ __match__ = r"^(http(s)?://)?(((www|m)\.)?douban\.com/((group/)?topic/\d+|doubanapp/dispatch)|douc\.cc/.+)"
23
+ __redirect_keywords__ = ["douc.cc", "doubanapp/dispatch"]
24
+
25
+ async def _do_parse(self, raw_url: str) -> "DoubanRichTextParseResult":
26
+ try:
27
+ topic = await Douban(proxy=self.proxy, cookie=self.cookie.get_value()).parse(raw_url)
28
+ except DoubanError as e:
29
+ raise ParseError(f"豆瓣解析失败: {e.msg}") from e
30
+ except Exception as e:
31
+ raise ParseError("豆瓣解析失败: 未知错误") from e
32
+
33
+ # 图片和视频在正文里有位置关系, 交给 RichText 由 markdown 保留顺序
34
+ media: list[AnyMediaRef] = [
35
+ *([self.to_video_ref(topic.video)] if topic.video else []),
36
+ *(self.to_media_ref(p) for p in topic.photos),
37
+ ]
38
+ return DoubanRichTextParseResult(title=topic.title, media=media, markdown_content=topic.markdown_content)
39
+
40
+ @staticmethod
41
+ def to_video_ref(video: DoubanVideo) -> VideoRef:
42
+ return VideoRef(
43
+ url=video.url,
44
+ thumb_url=video.thumb_url,
45
+ width=video.width,
46
+ height=video.height,
47
+ duration=video.duration,
48
+ )
49
+
50
+ @staticmethod
51
+ def to_media_ref(photo: DoubanPhoto) -> ImageRef | AniRef:
52
+ ref_type = AniRef if photo.is_animated else ImageRef
53
+ return ref_type(
54
+ url=photo.url,
55
+ ext=photo.ext,
56
+ thumb_url=photo.thumb_url,
57
+ width=photo.width,
58
+ height=photo.height,
59
+ )
60
+
61
+
62
+ class DoubanRichTextParseResult(RichTextParseResult):
63
+ async def _do_download(
64
+ self,
65
+ *,
66
+ output_dir: Path,
67
+ callback: ProgressCallback | None = None,
68
+ callback_args: tuple = (),
69
+ callback_kwargs: dict | None = None,
70
+ proxy: str | None = None,
71
+ headers: dict | None = None,
72
+ connections: int = 4,
73
+ ) -> "DownloadResult":
74
+ headers = {"User-Agent": UA, "Referer": IMAGE_REFERER}
75
+ return await super()._do_download(
76
+ output_dir=output_dir,
77
+ callback=callback,
78
+ callback_args=callback_args,
79
+ callback_kwargs=callback_kwargs,
80
+ proxy=proxy,
81
+ headers=headers,
82
+ connections=connections,
83
+ )
84
+
85
+
86
+ __all__ = ["DoubanParser", "DoubanRichTextParseResult"]
@@ -0,0 +1,159 @@
1
+ import re
2
+ from dataclasses import dataclass, field
3
+ from typing import Any, cast
4
+
5
+ import httpx
6
+ from bs4 import BeautifulSoup
7
+ from markdown import markdown
8
+ from markdownify import MarkdownConverter
9
+
10
+ TOPIC_API = "https://m.douban.com/rexxar/api/v2/group/topic/{}"
11
+ IMAGE_REFERER = "https://www.douban.com/"
12
+ """豆瓣图床有防盗链, 下载图片时必须带上 Referer"""
13
+
14
+ TOPIC_URL_RE = r"douban\.com/(?:group/)?topic/(\d+)"
15
+ """/group/topic/<id> 与 /topic/<id> 共用同一套 ID. 锚定域名避免误匹配 /gallery/topic/<id> (另一套 ID)"""
16
+
17
+ # rexxar 是移动版网页的内部接口, 无 UA 时返回 418
18
+ MOBILE_UA = (
19
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 "
20
+ "(KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"
21
+ )
22
+
23
+
24
+ class Douban:
25
+ def __init__(self, proxy: str | None = None, cookie: dict[str, str] | None = None):
26
+ self.proxy = proxy
27
+ self.cookie = cookie
28
+
29
+ async def parse(self, url: str) -> "DoubanTopic":
30
+ return DoubanTopic.parse(await self.fetch_topic_data(url))
31
+
32
+ @staticmethod
33
+ def get_topic_id(url: str) -> str:
34
+ if match := re.search(TOPIC_URL_RE, url):
35
+ return match.group(1)
36
+ raise DoubanError("暂不支持该豆瓣链接, 目前仅支持小组话题")
37
+
38
+ async def fetch_topic_data(self, url: str) -> dict[str, Any]:
39
+ headers = {"User-Agent": MOBILE_UA, "Referer": "https://m.douban.com/"}
40
+ async with httpx.AsyncClient(proxy=self.proxy, cookies=self.cookie, timeout=30) as cli:
41
+ result = await cli.get(TOPIC_API.format(self.get_topic_id(url)), headers=headers)
42
+
43
+ if result.status_code != 200:
44
+ fallback = f"获取话题内容失败: HTTP {result.status_code}"
45
+ try:
46
+ error = result.json()
47
+ except Exception as e:
48
+ raise DoubanError(fallback) from e
49
+ # 缺少 localized_message 时 msg 只是英文错误码 (如 need_permission), 需补中文上下文
50
+ if localized := error.get("localized_message"):
51
+ raise DoubanError(localized)
52
+ if msg := error.get("msg"):
53
+ raise DoubanError(f"获取话题内容失败: {msg}")
54
+ raise DoubanError(fallback)
55
+
56
+ return cast(dict[str, Any], result.json())
57
+
58
+
59
+ @dataclass
60
+ class DoubanVideo:
61
+ url: str
62
+ thumb_url: str | None = None
63
+ width: int = 0
64
+ height: int = 0
65
+ duration: int = 0
66
+
67
+
68
+ @dataclass
69
+ class DoubanPhoto:
70
+ url: str
71
+ ext: str = "jpg"
72
+ thumb_url: str | None = None
73
+ width: int = 0
74
+ height: int = 0
75
+ is_animated: bool = False
76
+
77
+
78
+ @dataclass
79
+ class DoubanTopic:
80
+ title: str
81
+ markdown_content: str
82
+ text_content: str
83
+ video: DoubanVideo | None = None
84
+ photos: list[DoubanPhoto] = field(default_factory=list)
85
+
86
+ @classmethod
87
+ def parse(cls, data: dict) -> "DoubanTopic":
88
+ content = data.get("content") or ""
89
+ markdown_content = MarkdownConverter(heading_style="ATX").convert(content).strip() if content else ""
90
+ text_content = "".join(BeautifulSoup(markdown(markdown_content), "lxml").find_all(string=True)).strip()
91
+
92
+ video_info = data.get("video_info") or {}
93
+ return cls(
94
+ title=data.get("title") or "",
95
+ markdown_content=markdown_content,
96
+ text_content=text_content,
97
+ video=parse_video(video_info) if video_info else None,
98
+ photos=[p for photo in data.get("photos") or [] if (p := parse_photo(photo))],
99
+ )
100
+
101
+
102
+ def parse_photo(photo: dict) -> DoubanPhoto | None:
103
+ image = photo.get("image") or {}
104
+ large = image.get("large") or {}
105
+ normal = image.get("normal") or {}
106
+
107
+ # 动图的 large 是体积极大的原始 GIF, 豆瓣同时提供了等效的 mp4
108
+ if image.get("is_animated") and (video := image.get("video")) and video.get("url"):
109
+ return DoubanPhoto(
110
+ url=video["url"],
111
+ ext="mp4",
112
+ thumb_url=normal.get("url"),
113
+ width=video.get("width") or 0,
114
+ height=video.get("height") or 0,
115
+ is_animated=True,
116
+ )
117
+
118
+ source = large or normal
119
+ if not (url := source.get("url")):
120
+ return None
121
+ thumb_url = normal.get("url")
122
+ return DoubanPhoto(
123
+ url=url,
124
+ thumb_url=thumb_url if thumb_url != url else None,
125
+ width=source.get("width") or 0,
126
+ height=source.get("height") or 0,
127
+ )
128
+
129
+
130
+ def parse_video(video_info: dict) -> DoubanVideo | None:
131
+ if not (url := video_info.get("video_url")):
132
+ return None
133
+ return DoubanVideo(
134
+ url=url,
135
+ thumb_url=video_info.get("cover_url"),
136
+ width=video_info.get("video_width") or 0,
137
+ height=video_info.get("video_height") or 0,
138
+ duration=parse_duration(video_info.get("duration")),
139
+ )
140
+
141
+
142
+ def parse_duration(duration: Any) -> int:
143
+ """把 ``HH:MM:SS`` / ``MM:SS`` 形式的时长转换为秒"""
144
+ if isinstance(duration, int):
145
+ return duration
146
+ if not isinstance(duration, str):
147
+ return 0
148
+ seconds = 0
149
+ for part in duration.split(":"):
150
+ if not part.isdigit():
151
+ return 0
152
+ seconds = seconds * 60 + int(part)
153
+ return seconds
154
+
155
+
156
+ class DoubanError(Exception):
157
+ def __init__(self, msg: str):
158
+ self.msg = msg
159
+ super().__init__(msg)
@@ -6,6 +6,7 @@ class Platform(Enum):
6
6
 
7
7
  BILIBILI = ("bilibili", "Bilibili")
8
8
  COOLAPK = ("coolapk", "酷安")
9
+ DOUBAN = ("douban", "豆瓣")
9
10
  DOUYIN = ("douyin", "抖音")
10
11
  TIKTOK = ("tiktok", "TikTok")
11
12
  FACEBOOK = ("facebook", "Facebook")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: parsehub
3
- Version: 2.1.10
3
+ Version: 2.2.0
4
4
  Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
5
5
  Author-email: 梓澪 <zilingmio@gmail.com>
6
6
  License: MIT
@@ -22,7 +22,7 @@ Requires-Dist: pydub>=0.25.1
22
22
  Requires-Dist: python-dotenv>=1.0.1
23
23
  Requires-Dist: tenacity>=8.5.0
24
24
  Requires-Dist: urlextract>=1.9.0
25
- Requires-Dist: yt-dlp[default]
25
+ Requires-Dist: yt-dlp[default]>=2026.8.18.122307.dev0
26
26
  Requires-Dist: lxml>=5.3.0
27
27
  Requires-Dist: pydantic>=1.10.19
28
28
  Requires-Dist: markdownify>=1.1.0
@@ -93,6 +93,7 @@ Dynamic: license-file
93
93
  | **小黑盒** | ✅ | ✅ | |
94
94
  | **Snapchat** | ✅ | | |
95
95
  | **知乎** | ✅ | ✅ | 🐶 问答, 专栏, 圈子 |
96
+ | **豆瓣** | ✅ | ✅ | 👥 小组话题 |
96
97
 
97
98
  ## 📦 安装
98
99
 
@@ -215,6 +216,7 @@ print(result)
215
216
  - `快手`
216
217
  - `小红书`
217
218
  - `知乎`
219
+ - `豆瓣`
218
220
 
219
221
  ```python
220
222
  from parsehub import ParseHub
@@ -313,6 +315,16 @@ except ParseError as exc:
313
315
 
314
316
  ---
315
317
 
318
+ ### 启用日志
319
+
320
+ ```python
321
+ from loguru import logger
322
+
323
+ logger.enable('parsehub')
324
+ ```
325
+
326
+ ---
327
+
316
328
  ## 🤝 参与贡献
317
329
 
318
330
  欢迎提交 Pull Request 或 Issue!
@@ -21,6 +21,7 @@ src/parsehub/parsers/base/ytdlp.py
21
21
  src/parsehub/parsers/parser/__init__.py
22
22
  src/parsehub/parsers/parser/bilibili.py
23
23
  src/parsehub/parsers/parser/coolapk.py
24
+ src/parsehub/parsers/parser/douban.py
24
25
  src/parsehub/parsers/parser/douyin.py
25
26
  src/parsehub/parsers/parser/facebook.py
26
27
  src/parsehub/parsers/parser/instagram.py
@@ -41,6 +42,7 @@ src/parsehub/parsers/parser/zuiyou.py
41
42
  src/parsehub/provider_api/__init__.py
42
43
  src/parsehub/provider_api/bilibili.py
43
44
  src/parsehub/provider_api/coolapk.py
45
+ src/parsehub/provider_api/douban.py
44
46
  src/parsehub/provider_api/douyin.py
45
47
  src/parsehub/provider_api/instagram.py
46
48
  src/parsehub/provider_api/kuaishou.py
@@ -5,7 +5,7 @@ pydub>=0.25.1
5
5
  python-dotenv>=1.0.1
6
6
  tenacity>=8.5.0
7
7
  urlextract>=1.9.0
8
- yt-dlp[default]
8
+ yt-dlp[default]>=2026.8.18.122307.dev0
9
9
  lxml>=5.3.0
10
10
  pydantic>=1.10.19
11
11
  markdownify>=1.1.0
@@ -1,13 +1,25 @@
1
1
  import unittest
2
- from unittest.mock import patch
2
+ from pathlib import Path
3
+ from unittest.mock import AsyncMock, patch
3
4
  from urllib.parse import parse_qs, urlparse
4
5
 
6
+ import httpx
7
+
5
8
  from parsehub import ParseHub
6
9
  from parsehub.errors import ParseError, UnknownPlatform
7
10
  from parsehub.parsers.base import BaseParser
11
+ from parsehub.parsers.parser.douban import DoubanParser, DoubanRichTextParseResult
8
12
  from parsehub.parsers.parser.douyin import parse_video_info
13
+ from parsehub.provider_api.douban import (
14
+ IMAGE_REFERER,
15
+ Douban,
16
+ DoubanError,
17
+ DoubanPhoto,
18
+ DoubanTopic,
19
+ DoubanVideo,
20
+ )
9
21
  from parsehub.provider_api.douyin import DouyinMobileCrawler, DouyinMobileDevice
10
- from parsehub.types import ImageParseResult, ImageRef, Platform, VideoParseResult, VideoRef
22
+ from parsehub.types import AniRef, ImageParseResult, ImageRef, ParseResult, Platform, VideoParseResult, VideoRef
11
23
  from parsehub.utils.helpers import SecretCookie, match_url, run_sync
12
24
 
13
25
 
@@ -296,6 +308,237 @@ class TestDouyinStorySupport(unittest.TestCase):
296
308
  self.assertEqual(info["duration"], 9682)
297
309
 
298
310
 
311
+ class TestDoubanTopicParsing(unittest.TestCase):
312
+ @staticmethod
313
+ def _photo(image: dict, width: int = 500, height: int = 400) -> dict:
314
+ return {"id": "1", "image": image, "size": {"width": width, "height": height}}
315
+
316
+ def test_error_messages_prefer_chinese_and_contextualise_raw_codes(self):
317
+ cases = [
318
+ # 豆瓣给了中文提示时直接透出
319
+ ({"localized_message": "这篇内容不存在了", "msg": "topic not found"}, "这篇内容不存在了"),
320
+ # 只有英文错误码时补上中文上下文, 避免用户只看到 need_permission
321
+ ({"msg": "need_permission"}, "获取话题内容失败: need_permission"),
322
+ ({}, "获取话题内容失败: HTTP 403"),
323
+ ]
324
+ for payload, expected in cases:
325
+ with self.subTest(payload=payload):
326
+ response = httpx.Response(403, json=payload)
327
+ with patch.object(httpx.AsyncClient, "get", new=AsyncMock(return_value=response)):
328
+ with self.assertRaises(DoubanError) as ctx:
329
+ run_sync(Douban().fetch_topic_data("https://www.douban.com/group/topic/1/"))
330
+ self.assertEqual(ctx.exception.msg, expected)
331
+
332
+ def test_get_topic_id_accepts_both_topic_url_forms(self):
333
+ # /group/topic/<id> 与 /topic/<id> 共用同一套 ID 与接口
334
+ self.assertEqual(Douban.get_topic_id("https://www.douban.com/group/topic/495373106/"), "495373106")
335
+ self.assertEqual(Douban.get_topic_id("https://www.douban.com/topic/492821052/"), "492821052")
336
+
337
+ def test_get_topic_id_rejects_non_topic_url(self):
338
+ for url in (
339
+ "https://movie.douban.com/subject/1292052/",
340
+ # 话题广场是另一套 ID, 不能被 /topic/ 规则误匹配
341
+ "https://www.douban.com/gallery/topic/125573/",
342
+ # 广播是另一种内容类型, 接口与数据结构均不同
343
+ "https://m.douban.com/people/182691094/status/9372433345/",
344
+ ):
345
+ with self.subTest(url=url), self.assertRaises(DoubanError):
346
+ Douban.get_topic_id(url)
347
+
348
+ def test_photo_topic_prefers_large_and_keeps_inline_markdown(self):
349
+ large = "https://img3.doubanio.com/view/group_topic/l/public/p1.jpg"
350
+ small = "https://img3.doubanio.com/view/group_topic/m/public/p1.jpg"
351
+ topic = DoubanTopic.parse(
352
+ {
353
+ "title": "标题",
354
+ "content": f"<div id='content'><p>正文</p><img src=\"{large}\"/></div>",
355
+ "photos": [
356
+ self._photo(
357
+ {
358
+ "is_animated": False,
359
+ "large": {"url": large, "width": 500, "height": 482},
360
+ "normal": {"url": small, "width": 200, "height": 193},
361
+ }
362
+ )
363
+ ],
364
+ }
365
+ )
366
+
367
+ self.assertEqual(topic.text_content, "正文")
368
+ self.assertIn(large, topic.markdown_content)
369
+ # 取 large 作正片, normal 退为缩略图
370
+ self.assertEqual(topic.photos, [DoubanPhoto(url=large, ext="jpg", thumb_url=small, width=500, height=482)])
371
+
372
+ def test_animated_photo_uses_mp4_variant(self):
373
+ topic = DoubanTopic.parse(
374
+ {
375
+ "title": "动图",
376
+ "content": "",
377
+ "photos": [
378
+ self._photo(
379
+ {
380
+ "is_animated": True,
381
+ # large 是体积极大的原始 GIF, 应优先取 video
382
+ "large": {"url": "https://img3.doubanio.com/view/group_topic/raw/public/p2.jpg"},
383
+ "normal": {"url": "https://img3.doubanio.com/view/group_topic/l/public/p2.jpg"},
384
+ "video": {
385
+ "url": "https://img3.doubanio.com/view/group_topic/l/public/p2.mp4",
386
+ "width": 500,
387
+ "height": 291,
388
+ },
389
+ }
390
+ )
391
+ ],
392
+ }
393
+ )
394
+
395
+ self.assertEqual(
396
+ topic.photos,
397
+ [
398
+ DoubanPhoto(
399
+ url="https://img3.doubanio.com/view/group_topic/l/public/p2.mp4",
400
+ ext="mp4",
401
+ thumb_url="https://img3.doubanio.com/view/group_topic/l/public/p2.jpg",
402
+ width=500,
403
+ height=291,
404
+ is_animated=True,
405
+ )
406
+ ],
407
+ )
408
+
409
+ def test_video_topic_converts_duration_to_seconds(self):
410
+ topic = DoubanTopic.parse(
411
+ {
412
+ "title": "视频",
413
+ "content": "<div id='content'><p>说明</p></div>",
414
+ "photos": [],
415
+ "video_info": {
416
+ "video_url": "https://sv1.doubanio.com/example.mp4",
417
+ "cover_url": "https://sv1.doubanio.com/example_cover.png",
418
+ "duration": "01:05",
419
+ "video_width": 720,
420
+ "video_height": 1280,
421
+ },
422
+ }
423
+ )
424
+
425
+ self.assertEqual(
426
+ topic.video,
427
+ DoubanVideo(
428
+ url="https://sv1.doubanio.com/example.mp4",
429
+ thumb_url="https://sv1.doubanio.com/example_cover.png",
430
+ width=720,
431
+ height=1280,
432
+ duration=65,
433
+ ),
434
+ )
435
+
436
+ def test_video_info_without_url_is_ignored(self):
437
+ """video_info 缺 video_url 时不该当成视频话题"""
438
+ topic = DoubanTopic.parse({"title": "t", "content": "", "photos": [], "video_info": {"duration": "00:12"}})
439
+
440
+ self.assertEqual(topic.photos, [])
441
+
442
+
443
+ class TestDoubanParserResultTypes(unittest.IsolatedAsyncioTestCase):
444
+ @staticmethod
445
+ def _topic(**kwargs) -> DoubanTopic:
446
+ defaults: dict = {
447
+ "title": "标题",
448
+ "markdown_content": "",
449
+ "text_content": "",
450
+ "video": None,
451
+ "photos": [],
452
+ }
453
+ return DoubanTopic(**{**defaults, **kwargs})
454
+
455
+ @staticmethod
456
+ async def _parse(topic: DoubanTopic):
457
+ with patch.object(Douban, "parse", new=AsyncMock(return_value=topic)):
458
+ return await DoubanParser()._do_parse("https://www.douban.com/group/topic/1/")
459
+
460
+ async def test_topic_always_returns_richtext(self):
461
+ """图 / 视频 / 纯文字 任意组合都走 RichText, 不再区分纯图"""
462
+ photo = DoubanPhoto(url="https://img3.doubanio.com/p1.jpg", width=500, height=482)
463
+ video = DoubanVideo(url="https://sv1.doubanio.com/a.mp4", duration=39)
464
+ img_md = "正文\n\n![](https://img3.doubanio.com/p1.jpg)"
465
+ cases = {
466
+ # markdown 必须原样带过去: 图片内嵌的位置信息只存在于 markdown 里
467
+ "图 + 正文": (self._topic(markdown_content=img_md, text_content="正文", photos=[photo]), 1, img_md),
468
+ "纯图无正文": (self._topic(markdown_content="![](x)", photos=[photo]), 1, "![](x)"),
469
+ "纯文字": (self._topic(markdown_content="只有文字", text_content="只有文字"), 0, "只有文字"),
470
+ "纯视频": (self._topic(video=video), 1, ""),
471
+ }
472
+ for label, (topic, media_count, expected_md) in cases.items():
473
+ with self.subTest(case=label):
474
+ result = await self._parse(topic)
475
+ self.assertIsInstance(result, DoubanRichTextParseResult)
476
+ self.assertEqual(len(result.media or []), media_count)
477
+ self.assertEqual(result.markdown_content, expected_md)
478
+
479
+ async def test_video_with_photos_keeps_video_first(self):
480
+ result = await self._parse(
481
+ self._topic(
482
+ text_content="说明",
483
+ markdown_content="说明",
484
+ video=DoubanVideo(url="https://sv1.doubanio.com/a.mp4", duration=39),
485
+ photos=[DoubanPhoto(url="https://img3.doubanio.com/p1.jpg", width=500, height=482)],
486
+ )
487
+ )
488
+
489
+ self.assertIsInstance(result, DoubanRichTextParseResult)
490
+ self.assertEqual(
491
+ result.media,
492
+ [
493
+ VideoRef(url="https://sv1.doubanio.com/a.mp4", duration=39),
494
+ ImageRef(url="https://img3.doubanio.com/p1.jpg", width=500, height=482),
495
+ ],
496
+ )
497
+
498
+ async def test_animated_photo_becomes_ani_ref(self):
499
+ result = await self._parse(
500
+ self._topic(
501
+ photos=[
502
+ DoubanPhoto(
503
+ url="https://img3.doubanio.com/p2.mp4",
504
+ ext="mp4",
505
+ thumb_url="https://img3.doubanio.com/p2.jpg",
506
+ width=500,
507
+ height=291,
508
+ is_animated=True,
509
+ )
510
+ ]
511
+ )
512
+ )
513
+
514
+ self.assertEqual(
515
+ result.media,
516
+ [
517
+ AniRef(
518
+ url="https://img3.doubanio.com/p2.mp4",
519
+ ext="mp4",
520
+ thumb_url="https://img3.doubanio.com/p2.jpg",
521
+ width=500,
522
+ height=291,
523
+ )
524
+ ],
525
+ )
526
+
527
+ async def test_provider_error_is_wrapped_in_parse_error(self):
528
+ with patch.object(Douban, "parse", new=AsyncMock(side_effect=DoubanError("这篇内容不存在了"))):
529
+ with self.assertRaisesRegex(ParseError, "豆瓣解析失败: 这篇内容不存在了"):
530
+ await DoubanParser()._do_parse("https://www.douban.com/group/topic/1/")
531
+
532
+ async def test_download_injects_douban_referer(self):
533
+ result = DoubanRichTextParseResult(title="标题", media=[ImageRef(url="https://img3.doubanio.com/p1.jpg")])
534
+
535
+ with patch.object(ParseResult, "_do_download", new=AsyncMock()) as mocked:
536
+ await result._do_download(output_dir=Path("/tmp/does-not-matter"))
537
+
538
+ # 豆瓣图床无 Referer 时返回 418
539
+ self.assertEqual(mocked.await_args.kwargs["headers"]["Referer"], IMAGE_REFERER)
540
+
541
+
299
542
  class TestPlatformUrlMatching(unittest.TestCase):
300
543
  def test_supported_platform_url_formats(self):
301
544
  parsehub = ParseHub()
@@ -314,6 +557,14 @@ class TestPlatformUrlMatching(unittest.TestCase):
314
557
  "https://www.coolapk.com/feed/70163953",
315
558
  "https://www.coolapk.com/picture/123456",
316
559
  ],
560
+ Platform.DOUBAN: [
561
+ "https://www.douban.com/group/topic/495373106/",
562
+ "https://m.douban.com/group/topic/495373106/",
563
+ "douban.com/group/topic/495373106/?_spm_id=MTU0MzM0ODY2&_i=5304694LhuE3jh",
564
+ "https://www.douban.com/topic/492821052/?_spm_id=MTQ3NTcyNQ&dt_dapp=1",
565
+ "https://www.douban.com/doubanapp/dispatch?uri=/group/topic/495373106/",
566
+ "https://douc.cc/2Yx4Ol",
567
+ ],
317
568
  Platform.DOUYIN: [
318
569
  "https://www.douyin.com/video/7615533976798727464",
319
570
  "https://www.douyin.com/note/7615533976798727464",
@@ -419,6 +670,11 @@ class TestPlatformUrlMatching(unittest.TestCase):
419
670
  def test_known_unsupported_url_formats_are_not_matched(self):
420
671
  parsehub = ParseHub()
421
672
  urls = [
673
+ "https://movie.douban.com/subject/1292052/",
674
+ "https://book.douban.com/subject/1084336/",
675
+ "https://www.douban.com/people/154334866/",
676
+ "https://www.douban.com/group/657759/",
677
+ "https://www.douban.com/gallery/topic/125573/",
422
678
  "https://www.douyin.com/share/user/MS4wLjABAAAA",
423
679
  "https://www.douyin.com/qishui/share/video/123456",
424
680
  "https://www.tiktok.com/share/user/123456",
File without changes
File without changes
File without changes
File without changes
File without changes