parsehub 2.4.0__tar.gz → 2.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. {parsehub-2.4.0/src/parsehub.egg-info → parsehub-2.4.1}/PKG-INFO +22 -22
  2. {parsehub-2.4.0 → parsehub-2.4.1}/pyproject.toml +25 -25
  3. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/instagram.py +9 -7
  4. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/kuaishou.py +1 -0
  5. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/instagram.py +1 -2
  6. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/wxchannels.py +1 -3
  7. {parsehub-2.4.0 → parsehub-2.4.1/src/parsehub.egg-info}/PKG-INFO +22 -22
  8. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub.egg-info/SOURCES.txt +1 -5
  9. parsehub-2.4.1/src/parsehub.egg-info/requires.txt +27 -0
  10. {parsehub-2.4.0 → parsehub-2.4.1}/test/test_core_offline.py +1 -335
  11. parsehub-2.4.0/src/parsehub.egg-info/requires.txt +0 -27
  12. parsehub-2.4.0/test/test_bilibili_music_offline.py +0 -127
  13. parsehub-2.4.0/test/test_bilibili_offline.py +0 -63
  14. parsehub-2.4.0/test/test_instagram_duration_offline.py +0 -134
  15. parsehub-2.4.0/test/test_zuiyou_duration_offline.py +0 -69
  16. {parsehub-2.4.0 → parsehub-2.4.1}/LICENSE +0 -0
  17. {parsehub-2.4.0 → parsehub-2.4.1}/README.md +0 -0
  18. {parsehub-2.4.0 → parsehub-2.4.1}/setup.cfg +0 -0
  19. {parsehub-2.4.0 → parsehub-2.4.1}/src/__init__.py +0 -0
  20. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/__init__.py +0 -0
  21. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/cli.py +0 -0
  22. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/cli_config.py +0 -0
  23. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/config/__init__.py +0 -0
  24. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/config/config.py +0 -0
  25. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/errors.py +0 -0
  26. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/__init__.py +0 -0
  27. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/base/__init__.py +0 -0
  28. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/base/base.py +0 -0
  29. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/base/ytdlp.py +0 -0
  30. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/__init__.py +0 -0
  31. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/bangumi.py +0 -0
  32. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/bilibili.py +0 -0
  33. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/coolapk.py +0 -0
  34. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/douban.py +0 -0
  35. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/douyin.py +0 -0
  36. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/facebook.py +0 -0
  37. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/medium.py +0 -0
  38. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/pipix.py +0 -0
  39. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/snapchat.py +0 -0
  40. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/threads.py +0 -0
  41. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/tieba.py +0 -0
  42. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/tiktok.py +0 -0
  43. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/toutiao.py +0 -0
  44. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/twitter.py +0 -0
  45. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/weibo.py +0 -0
  46. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/weixin.py +0 -0
  47. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/wxchannels.py +0 -0
  48. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/xhs.py +0 -0
  49. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/xiaoheihe.py +0 -0
  50. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/youtube.py +0 -0
  51. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/zhihu.py +0 -0
  52. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/parsers/parser/zuiyou.py +0 -0
  53. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/__init__.py +0 -0
  54. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/bangumi.py +0 -0
  55. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/bilibili.py +0 -0
  56. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/coolapk.py +0 -0
  57. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/douban.py +0 -0
  58. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/douyin.py +0 -0
  59. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/kuaishou.py +0 -0
  60. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/medium.py +0 -0
  61. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/pipix.py +0 -0
  62. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/threads.py +0 -0
  63. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/tieba.py +0 -0
  64. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/tiktok.py +0 -0
  65. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/toutiao.py +0 -0
  66. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/twitter.py +0 -0
  67. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/weibo.py +0 -0
  68. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/weixin.py +0 -0
  69. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/xhs.py +0 -0
  70. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/xiaoheihe.py +0 -0
  71. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/zhihu.py +0 -0
  72. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/provider_api/zuiyou.py +0 -0
  73. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/types/__init__.py +0 -0
  74. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/types/callback.py +0 -0
  75. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/types/media_file.py +0 -0
  76. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/types/media_ref.py +0 -0
  77. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/types/platform.py +0 -0
  78. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/types/post.py +0 -0
  79. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/types/result.py +0 -0
  80. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/utils/downloader.py +0 -0
  81. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/utils/fallback.py +0 -0
  82. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/utils/helpers.py +0 -0
  83. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub/utils/media_info.py +0 -0
  84. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub.egg-info/dependency_links.txt +0 -0
  85. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub.egg-info/entry_points.txt +0 -0
  86. {parsehub-2.4.0 → parsehub-2.4.1}/src/parsehub.egg-info/top_level.txt +0 -0
  87. {parsehub-2.4.0 → parsehub-2.4.1}/test/test_cli.py +0 -0
  88. {parsehub-2.4.0 → parsehub-2.4.1}/test/test_cli_config.py +0 -0
  89. {parsehub-2.4.0 → parsehub-2.4.1}/test/test_downloader.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: parsehub
3
- Version: 2.4.0
3
+ Version: 2.4.1
4
4
  Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
5
5
  Author-email: 梓澪 <zilingmio@gmail.com>
6
6
  License: MIT
@@ -15,32 +15,32 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
15
15
  Requires-Python: >=3.12.0
16
16
  Description-Content-Type: text/markdown
17
17
  License-File: LICENSE
18
- Requires-Dist: aiofiles>=23.2
19
- Requires-Dist: beautifulsoup4>=4.12.3
20
- Requires-Dist: loguru>=0.6.0
18
+ Requires-Dist: aiofiles>=25.1.0
19
+ Requires-Dist: beautifulsoup4>=4.15.0
20
+ Requires-Dist: loguru>=0.7.3
21
21
  Requires-Dist: pydub>=0.25.1
22
- Requires-Dist: python-dotenv>=1.0.1
23
- Requires-Dist: tenacity>=8.5.0
24
- Requires-Dist: urlextract>=1.9.0
25
- Requires-Dist: yt-dlp[default]>=2026.10.26.220202.dev0
26
- Requires-Dist: lxml>=5.3.0
27
- Requires-Dist: pydantic>=1.10.19
28
- Requires-Dist: markdownify>=1.1.0
29
- Requires-Dist: markdown>=3.7
30
- Requires-Dist: json5>=0.12
31
- Requires-Dist: requests
32
- Requires-Dist: httpx[socks]>=0.24.1
33
- Requires-Dist: pillow>=12.1.0
34
- Requires-Dist: python-slugify[unidecode]>=8.0.4
35
- Requires-Dist: opencv-python-headless>=4.13.0.92
36
- Requires-Dist: cryptography>=49.0.0
22
+ Requires-Dist: python-dotenv>=1.2.4
23
+ Requires-Dist: tenacity>=9.2.1
24
+ Requires-Dist: urlextract>=1.10.0
25
+ Requires-Dist: yt-dlp[default]>=2026.11.4.232659.dev0
26
+ Requires-Dist: lxml>=6.1.3
27
+ Requires-Dist: pydantic>=2.14.3
28
+ Requires-Dist: markdownify>=1.2.3
29
+ Requires-Dist: markdown>=3.11.3
30
+ Requires-Dist: json5>=0.17.3
31
+ Requires-Dist: requests>=2.34.2
32
+ Requires-Dist: httpx[socks]>=0.28.1
33
+ Requires-Dist: pillow>=13.0.0
34
+ Requires-Dist: python-slugify[unidecode]>=9.1.3
35
+ Requires-Dist: opencv-python-headless>=5.0.0.93
36
+ Requires-Dist: cryptography>=50.0.2
37
37
  Requires-Dist: gmssl>=3.2.2
38
38
  Requires-Dist: signerpy>=0.12.0
39
- Requires-Dist: curl-cffi>=0.13
39
+ Requires-Dist: curl-cffi>=0.17.0
40
40
  Requires-Dist: xhshow>=0.2.0
41
41
  Provides-Extra: cli
42
- Requires-Dist: argcomplete>=3.6.3; extra == "cli"
43
- Requires-Dist: platformdirs>=4.5.1; extra == "cli"
42
+ Requires-Dist: argcomplete>=3.7.2; extra == "cli"
43
+ Requires-Dist: platformdirs>=4.14.5; extra == "cli"
44
44
  Dynamic: license-file
45
45
 
46
46
  <div align="center">
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "parsehub"
3
- version = "2.4.0"
3
+ version = "2.4.1"
4
4
  description = "轻量、异步、开箱即用的社交媒体聚合解析库"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.12.0"
@@ -18,28 +18,28 @@ classifiers = [
18
18
  ]
19
19
 
20
20
  dependencies = [
21
- "aiofiles>=23.2",
22
- "beautifulsoup4>=4.12.3",
23
- "loguru>=0.6.0",
21
+ "aiofiles>=25.1.0",
22
+ "beautifulsoup4>=4.15.0",
23
+ "loguru>=0.7.3",
24
24
  "pydub>=0.25.1",
25
- "python-dotenv>=1.0.1",
26
- "tenacity>=8.5.0",
27
- "urlextract>=1.9.0",
28
- "yt-dlp[default]>=2026.10.26.220202.dev0",
29
- "lxml>=5.3.0",
30
- "pydantic>=1.10.19",
31
- "markdownify>=1.1.0",
32
- "markdown>=3.7",
33
- "json5>=0.12",
34
- "requests",
35
- "httpx[socks]>=0.24.1",
36
- "pillow>=12.1.0",
37
- "python-slugify[unidecode]>=8.0.4",
38
- "opencv-python-headless>=4.13.0.92",
39
- "cryptography>=49.0.0",
25
+ "python-dotenv>=1.2.4",
26
+ "tenacity>=9.2.1",
27
+ "urlextract>=1.10.0",
28
+ "yt-dlp[default]>=2026.11.4.232659.dev0",
29
+ "lxml>=6.1.3",
30
+ "pydantic>=2.14.3",
31
+ "markdownify>=1.2.3",
32
+ "markdown>=3.11.3",
33
+ "json5>=0.17.3",
34
+ "requests>=2.34.2",
35
+ "httpx[socks]>=0.28.1",
36
+ "pillow>=13.0.0",
37
+ "python-slugify[unidecode]>=9.1.3",
38
+ "opencv-python-headless>=5.0.0.93",
39
+ "cryptography>=50.0.2",
40
40
  "gmssl>=3.2.2",
41
41
  "signerpy>=0.12.0",
42
- "curl-cffi>=0.13",
42
+ "curl-cffi>=0.17.0",
43
43
  "xhshow>=0.2.0",
44
44
  ]
45
45
 
@@ -49,15 +49,15 @@ ph = "parsehub.cli:main"
49
49
 
50
50
  [project.optional-dependencies]
51
51
  cli = [
52
- "argcomplete>=3.6.3",
53
- "platformdirs>=4.5.1",
52
+ "argcomplete>=3.7.2",
53
+ "platformdirs>=4.14.5",
54
54
  ]
55
55
 
56
56
  [dependency-groups]
57
57
  dev = [
58
- "mypy>=2.1.0",
59
- "pytest>=9.0.3",
60
- "ruff>=0.14.14",
58
+ "mypy>=2.4.1",
59
+ "pytest>=9.2.0",
60
+ "ruff>=0.17.4",
61
61
  ]
62
62
 
63
63
  [tool.hatch.build.targets.wheel]
@@ -66,7 +66,7 @@ class InstagramParser(BaseParser):
66
66
  except ParseError as e:
67
67
  logger.warning(f"Instagram yt-dlp 解析失败, 使用直链视频: {e}")
68
68
  else:
69
- return YtVideoParseResult(
69
+ return InstagramYtVideoParseResult(
70
70
  dl=info,
71
71
  title=post.title,
72
72
  content=post.caption,
@@ -98,12 +98,8 @@ class InstagramParser(BaseParser):
98
98
  async def _parse(self, shortcode: str) -> InstagramPost:
99
99
  stages: list[FallbackStage[InstagramPost]] = [FallbackStage("匿名", lambda: self._fetch(shortcode))]
100
100
  if self.cookie.get_value():
101
- stages.append(
102
- FallbackStage("cookie", lambda: self._fetch(shortcode, self.cookie), cookie=True)
103
- )
104
- return await self._fallback(
105
- *stages, stop=lambda e: not isinstance(e.__cause__, InstagramRestrictedError)
106
- )
101
+ stages.append(FallbackStage("cookie", lambda: self._fetch(shortcode, self.cookie), cookie=True))
102
+ return await self._fallback(*stages, stop=lambda e: not isinstance(e.__cause__, InstagramRestrictedError))
107
103
 
108
104
  async def _fetch(self, shortcode: str, cookie: SecretCookie | None = None) -> InstagramPost:
109
105
  api = InstagramAPI(proxy=self.proxy, cookie=cookie.get_value() if cookie else None, timeout=30)
@@ -127,4 +123,10 @@ class InstagramYtParse(YtParser, register=False):
127
123
  pass
128
124
 
129
125
 
126
+ class InstagramYtVideoParseResult(YtVideoParseResult):
127
+ @property
128
+ def cli_args(self) -> list[str]:
129
+ return [*super().cli_args, "-S", "vcodec:h264"]
130
+
131
+
130
132
  __all__ = ["InstagramParser"]
@@ -46,6 +46,7 @@ class KuaiShouParser(BaseParser):
46
46
  if (
47
47
  ksp.page_type == "VIDEO" or "/short-video/" in raw_url
48
48
  ): # 地区风控导致获取到的 page_type 可能为 ATLAS 但实际为视频
49
+
49
50
  async def parse_html() -> VideoParseResult:
50
51
  content = ksp.get_title_content()
51
52
  video = ksp.get_real_video_url()
@@ -131,8 +131,7 @@ class InstagramAPI:
131
131
  ) -> InstagramPost:
132
132
  """用 embed 补齐 v1 / GraphQL 缺失的视频直链和时长。"""
133
133
  if not any(
134
- (post.type is InstagramMediaType.VIDEO or item.is_video)
135
- and (not item.video_url or item.duration <= 0)
134
+ (post.type is InstagramMediaType.VIDEO or item.is_video) and (not item.video_url or item.duration <= 0)
136
135
  for item in post.media
137
136
  ):
138
137
  return post
@@ -95,9 +95,7 @@ class WXChannels:
95
95
 
96
96
  feed_data = feed_result.get("data") or {}
97
97
  feed_error = feed_data.get("errMsg") or {}
98
- error_text = " ".join(
99
- _strip_html(str(feed_error.get(key) or "")) for key in ("title", "content")
100
- ).strip()
98
+ error_text = " ".join(_strip_html(str(feed_error.get(key) or "")) for key in ("title", "content")).strip()
101
99
  if feed_result.get("errCode") != 0:
102
100
  message = feed_result.get("errMsg") or "微信视频号解析失败"
103
101
  raise WXChannelsError(str(message))
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: parsehub
3
- Version: 2.4.0
3
+ Version: 2.4.1
4
4
  Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
5
5
  Author-email: 梓澪 <zilingmio@gmail.com>
6
6
  License: MIT
@@ -15,32 +15,32 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
15
15
  Requires-Python: >=3.12.0
16
16
  Description-Content-Type: text/markdown
17
17
  License-File: LICENSE
18
- Requires-Dist: aiofiles>=23.2
19
- Requires-Dist: beautifulsoup4>=4.12.3
20
- Requires-Dist: loguru>=0.6.0
18
+ Requires-Dist: aiofiles>=25.1.0
19
+ Requires-Dist: beautifulsoup4>=4.15.0
20
+ Requires-Dist: loguru>=0.7.3
21
21
  Requires-Dist: pydub>=0.25.1
22
- Requires-Dist: python-dotenv>=1.0.1
23
- Requires-Dist: tenacity>=8.5.0
24
- Requires-Dist: urlextract>=1.9.0
25
- Requires-Dist: yt-dlp[default]>=2026.10.26.220202.dev0
26
- Requires-Dist: lxml>=5.3.0
27
- Requires-Dist: pydantic>=1.10.19
28
- Requires-Dist: markdownify>=1.1.0
29
- Requires-Dist: markdown>=3.7
30
- Requires-Dist: json5>=0.12
31
- Requires-Dist: requests
32
- Requires-Dist: httpx[socks]>=0.24.1
33
- Requires-Dist: pillow>=12.1.0
34
- Requires-Dist: python-slugify[unidecode]>=8.0.4
35
- Requires-Dist: opencv-python-headless>=4.13.0.92
36
- Requires-Dist: cryptography>=49.0.0
22
+ Requires-Dist: python-dotenv>=1.2.4
23
+ Requires-Dist: tenacity>=9.2.1
24
+ Requires-Dist: urlextract>=1.10.0
25
+ Requires-Dist: yt-dlp[default]>=2026.11.4.232659.dev0
26
+ Requires-Dist: lxml>=6.1.3
27
+ Requires-Dist: pydantic>=2.14.3
28
+ Requires-Dist: markdownify>=1.2.3
29
+ Requires-Dist: markdown>=3.11.3
30
+ Requires-Dist: json5>=0.17.3
31
+ Requires-Dist: requests>=2.34.2
32
+ Requires-Dist: httpx[socks]>=0.28.1
33
+ Requires-Dist: pillow>=13.0.0
34
+ Requires-Dist: python-slugify[unidecode]>=9.1.3
35
+ Requires-Dist: opencv-python-headless>=5.0.0.93
36
+ Requires-Dist: cryptography>=50.0.2
37
37
  Requires-Dist: gmssl>=3.2.2
38
38
  Requires-Dist: signerpy>=0.12.0
39
- Requires-Dist: curl-cffi>=0.13
39
+ Requires-Dist: curl-cffi>=0.17.0
40
40
  Requires-Dist: xhshow>=0.2.0
41
41
  Provides-Extra: cli
42
- Requires-Dist: argcomplete>=3.6.3; extra == "cli"
43
- Requires-Dist: platformdirs>=4.5.1; extra == "cli"
42
+ Requires-Dist: argcomplete>=3.7.2; extra == "cli"
43
+ Requires-Dist: platformdirs>=4.14.5; extra == "cli"
44
44
  Dynamic: license-file
45
45
 
46
46
  <div align="center">
@@ -76,11 +76,7 @@ src/parsehub/utils/downloader.py
76
76
  src/parsehub/utils/fallback.py
77
77
  src/parsehub/utils/helpers.py
78
78
  src/parsehub/utils/media_info.py
79
- test/test_bilibili_music_offline.py
80
- test/test_bilibili_offline.py
81
79
  test/test_cli.py
82
80
  test/test_cli_config.py
83
81
  test/test_core_offline.py
84
- test/test_downloader.py
85
- test/test_instagram_duration_offline.py
86
- test/test_zuiyou_duration_offline.py
82
+ test/test_downloader.py
@@ -0,0 +1,27 @@
1
+ aiofiles>=25.1.0
2
+ beautifulsoup4>=4.15.0
3
+ loguru>=0.7.3
4
+ pydub>=0.25.1
5
+ python-dotenv>=1.2.4
6
+ tenacity>=9.2.1
7
+ urlextract>=1.10.0
8
+ yt-dlp[default]>=2026.11.4.232659.dev0
9
+ lxml>=6.1.3
10
+ pydantic>=2.14.3
11
+ markdownify>=1.2.3
12
+ markdown>=3.11.3
13
+ json5>=0.17.3
14
+ requests>=2.34.2
15
+ httpx[socks]>=0.28.1
16
+ pillow>=13.0.0
17
+ python-slugify[unidecode]>=9.1.3
18
+ opencv-python-headless>=5.0.0.93
19
+ cryptography>=50.0.2
20
+ gmssl>=3.2.2
21
+ signerpy>=0.12.0
22
+ curl-cffi>=0.17.0
23
+ xhshow>=0.2.0
24
+
25
+ [cli]
26
+ argcomplete>=3.7.2
27
+ platformdirs>=4.14.5
@@ -1,25 +1,10 @@
1
1
  import unittest
2
- from pathlib import Path
3
- from unittest.mock import AsyncMock, patch
4
2
  from urllib.parse import parse_qs, urlparse
5
3
 
6
- import httpx
7
-
8
4
  from parsehub import ParseHub
9
5
  from parsehub.errors import ParseError, UnknownPlatform
10
6
  from parsehub.parsers.base import BaseParser
11
- from parsehub.parsers.parser.douban import DoubanParser, DoubanRichTextParseResult
12
- from parsehub.parsers.parser.douyin import parse_video_info
13
- from parsehub.provider_api.douban import (
14
- IMAGE_REFERER,
15
- Douban,
16
- DoubanError,
17
- DoubanPhoto,
18
- DoubanTopic,
19
- DoubanVideo,
20
- )
21
- from parsehub.provider_api.douyin import DouyinMobileCrawler, DouyinMobileDevice
22
- from parsehub.types import AniRef, ImageParseResult, ImageRef, ParseResult, Platform, VideoParseResult, VideoRef
7
+ from parsehub.types import ImageParseResult, ImageRef, Platform, VideoParseResult, VideoRef
23
8
  from parsehub.utils.helpers import SecretCookie, match_url, run_sync
24
9
 
25
10
 
@@ -225,325 +210,6 @@ class TestParseResultToDict(unittest.TestCase):
225
210
  )
226
211
 
227
212
 
228
- class TestDouyinStorySupport(unittest.TestCase):
229
- def test_mobile_device_can_be_loaded_from_env(self):
230
- with patch.dict(
231
- "os.environ",
232
- {
233
- "PARSEHUB_DOUYIN_DEVICE_ID": "1325343490970016",
234
- "PARSEHUB_DOUYIN_IID": "1325343490974112",
235
- },
236
- clear=True,
237
- ):
238
- device = DouyinMobileDevice.from_env()
239
-
240
- self.assertIsNotNone(device)
241
- assert device is not None
242
- self.assertEqual(device.device_id, "1325343490970016")
243
- self.assertEqual(device.iid, "1325343490974112")
244
-
245
- def test_mobile_device_can_be_loaded_from_register_response(self):
246
- device = DouyinMobileDevice.from_register_response(
247
- {
248
- "device_id": 1219790537803243,
249
- "install_id": 1219790537807339,
250
- "device_id_str": "1219790537803243",
251
- "install_id_str": "1219790537807339",
252
- },
253
- cdid="demo-cdid",
254
- openudid="demo-openudid",
255
- )
256
-
257
- self.assertIsNotNone(device)
258
- assert device is not None
259
- self.assertEqual(device.device_id, "1219790537803243")
260
- self.assertEqual(device.iid, "1219790537807339")
261
- self.assertEqual(device.cdid, "demo-cdid")
262
- self.assertEqual(device.openudid, "demo-openudid")
263
-
264
- def test_mobile_device_pool_round_robin(self):
265
- old_pool = DouyinMobileCrawler._device_pool
266
- old_index = DouyinMobileCrawler._device_pool_index
267
- try:
268
- DouyinMobileCrawler._device_pool = [
269
- DouyinMobileDevice(device_id="1", iid="11"),
270
- DouyinMobileDevice(device_id="2", iid="22"),
271
- DouyinMobileDevice(device_id="3", iid="33"),
272
- ]
273
- DouyinMobileCrawler._device_pool_index = 0
274
-
275
- picked = [DouyinMobileCrawler._next_pooled_device().device_id for _ in range(5)]
276
-
277
- self.assertEqual(picked, ["1", "2", "3", "1", "2"])
278
- finally:
279
- DouyinMobileCrawler._device_pool = old_pool
280
- DouyinMobileCrawler._device_pool_index = old_index
281
-
282
- def test_parse_video_info_prefers_story_default_play_url_by_data_size(self):
283
- video_data = {
284
- "duration": 9682,
285
- "cover": {"url_list": ["https://cdn.example/thumb.jpg"]},
286
- "bit_rate": [
287
- {
288
- "bit_rate": 0,
289
- "play_addr": {
290
- "url_list": ["https://cdn.example/story-default.mp4"],
291
- "width": 720,
292
- "height": 1280,
293
- "data_size": 7567631,
294
- },
295
- },
296
- {
297
- "bit_rate": 632,
298
- "play_addr": {
299
- "url_list": ["https://cdn.example/720p.mp4"],
300
- "width": 720,
301
- "height": 1280,
302
- "data_size": 783567,
303
- },
304
- },
305
- ],
306
- }
307
-
308
- info = parse_video_info(video_data)
309
-
310
- self.assertEqual(info["video_url"], "https://cdn.example/story-default.mp4")
311
- self.assertEqual(info["thumb_url"], "https://cdn.example/thumb.jpg")
312
- self.assertEqual(info["duration"], 9682)
313
-
314
-
315
- class TestDoubanTopicParsing(unittest.TestCase):
316
- @staticmethod
317
- def _photo(image: dict, width: int = 500, height: int = 400) -> dict:
318
- return {"id": "1", "image": image, "size": {"width": width, "height": height}}
319
-
320
- def test_error_messages_prefer_chinese_and_contextualise_raw_codes(self):
321
- cases = [
322
- # 豆瓣给了中文提示时直接透出
323
- ({"localized_message": "这篇内容不存在了", "msg": "topic not found"}, "这篇内容不存在了"),
324
- # 只有英文错误码时补上中文上下文, 避免用户只看到 need_permission
325
- ({"msg": "need_permission"}, "获取话题内容失败: need_permission"),
326
- ({}, "获取话题内容失败: HTTP 403"),
327
- ]
328
- for payload, expected in cases:
329
- with self.subTest(payload=payload):
330
- response = httpx.Response(403, json=payload)
331
- with patch.object(httpx.AsyncClient, "get", new=AsyncMock(return_value=response)):
332
- with self.assertRaises(DoubanError) as ctx:
333
- run_sync(Douban().fetch_topic_data("https://www.douban.com/group/topic/1/"))
334
- self.assertEqual(ctx.exception.msg, expected)
335
-
336
- def test_get_topic_id_accepts_both_topic_url_forms(self):
337
- # /group/topic/<id> 与 /topic/<id> 共用同一套 ID 与接口
338
- self.assertEqual(Douban.get_topic_id("https://www.douban.com/group/topic/495373106/"), "495373106")
339
- self.assertEqual(Douban.get_topic_id("https://www.douban.com/topic/492821052/"), "492821052")
340
-
341
- def test_get_topic_id_rejects_non_topic_url(self):
342
- for url in (
343
- "https://movie.douban.com/subject/1292052/",
344
- # 话题广场是另一套 ID, 不能被 /topic/ 规则误匹配
345
- "https://www.douban.com/gallery/topic/125573/",
346
- # 广播是另一种内容类型, 接口与数据结构均不同
347
- "https://m.douban.com/people/182691094/status/9372433345/",
348
- ):
349
- with self.subTest(url=url), self.assertRaises(DoubanError):
350
- Douban.get_topic_id(url)
351
-
352
- def test_photo_topic_prefers_large_and_keeps_inline_markdown(self):
353
- large = "https://img3.doubanio.com/view/group_topic/l/public/p1.jpg"
354
- small = "https://img3.doubanio.com/view/group_topic/m/public/p1.jpg"
355
- topic = DoubanTopic.parse(
356
- {
357
- "title": "标题",
358
- "content": f"<div id='content'><p>正文</p><img src=\"{large}\"/></div>",
359
- "photos": [
360
- self._photo(
361
- {
362
- "is_animated": False,
363
- "large": {"url": large, "width": 500, "height": 482},
364
- "normal": {"url": small, "width": 200, "height": 193},
365
- }
366
- )
367
- ],
368
- }
369
- )
370
-
371
- self.assertEqual(topic.text_content, "正文")
372
- self.assertIn(large, topic.markdown_content)
373
- # 取 large 作正片, normal 退为缩略图
374
- self.assertEqual(topic.photos, [DoubanPhoto(url=large, ext="jpg", thumb_url=small, width=500, height=482)])
375
-
376
- def test_animated_photo_uses_mp4_variant(self):
377
- topic = DoubanTopic.parse(
378
- {
379
- "title": "动图",
380
- "content": "",
381
- "photos": [
382
- self._photo(
383
- {
384
- "is_animated": True,
385
- # large 是体积极大的原始 GIF, 应优先取 video
386
- "large": {"url": "https://img3.doubanio.com/view/group_topic/raw/public/p2.jpg"},
387
- "normal": {"url": "https://img3.doubanio.com/view/group_topic/l/public/p2.jpg"},
388
- "video": {
389
- "url": "https://img3.doubanio.com/view/group_topic/l/public/p2.mp4",
390
- "width": 500,
391
- "height": 291,
392
- },
393
- }
394
- )
395
- ],
396
- }
397
- )
398
-
399
- self.assertEqual(
400
- topic.photos,
401
- [
402
- DoubanPhoto(
403
- url="https://img3.doubanio.com/view/group_topic/l/public/p2.mp4",
404
- ext="mp4",
405
- thumb_url="https://img3.doubanio.com/view/group_topic/l/public/p2.jpg",
406
- width=500,
407
- height=291,
408
- is_animated=True,
409
- )
410
- ],
411
- )
412
-
413
- def test_video_topic_converts_duration_to_seconds(self):
414
- topic = DoubanTopic.parse(
415
- {
416
- "title": "视频",
417
- "content": "<div id='content'><p>说明</p></div>",
418
- "photos": [],
419
- "video_info": {
420
- "video_url": "https://sv1.doubanio.com/example.mp4",
421
- "cover_url": "https://sv1.doubanio.com/example_cover.png",
422
- "duration": "01:05",
423
- "video_width": 720,
424
- "video_height": 1280,
425
- },
426
- }
427
- )
428
-
429
- self.assertEqual(
430
- topic.video,
431
- DoubanVideo(
432
- url="https://sv1.doubanio.com/example.mp4",
433
- thumb_url="https://sv1.doubanio.com/example_cover.png",
434
- width=720,
435
- height=1280,
436
- duration=65,
437
- ),
438
- )
439
-
440
- def test_video_info_without_url_is_ignored(self):
441
- """video_info 缺 video_url 时不该当成视频话题"""
442
- topic = DoubanTopic.parse({"title": "t", "content": "", "photos": [], "video_info": {"duration": "00:12"}})
443
-
444
- self.assertEqual(topic.photos, [])
445
-
446
-
447
- class TestDoubanParserResultTypes(unittest.IsolatedAsyncioTestCase):
448
- @staticmethod
449
- def _topic(**kwargs) -> DoubanTopic:
450
- defaults: dict = {
451
- "title": "标题",
452
- "markdown_content": "",
453
- "text_content": "",
454
- "image_layout": "vertical", # 默认图文混排, 走 RichText 分支
455
- "video": None,
456
- "photos": [],
457
- }
458
- return DoubanTopic(**{**defaults, **kwargs})
459
-
460
- @staticmethod
461
- async def _parse(topic: DoubanTopic):
462
- with patch.object(Douban, "parse", new=AsyncMock(return_value=topic)):
463
- return await DoubanParser()._do_parse("https://www.douban.com/group/topic/1/")
464
-
465
- async def test_topic_always_returns_richtext(self):
466
- """图 / 视频 / 纯文字 任意组合都走 RichText, 不再区分纯图"""
467
- photo = DoubanPhoto(url="https://img3.doubanio.com/p1.jpg", width=500, height=482)
468
- video = DoubanVideo(url="https://sv1.doubanio.com/a.mp4", duration=39)
469
- img_md = "正文\n\n![](https://img3.doubanio.com/p1.jpg)"
470
- cases = {
471
- # markdown 必须原样带过去: 图片内嵌的位置信息只存在于 markdown 里
472
- "图 + 正文": (self._topic(markdown_content=img_md, text_content="正文", photos=[photo]), 1, img_md),
473
- "纯图无正文": (self._topic(markdown_content="![](x)", photos=[photo]), 1, "![](x)"),
474
- "纯文字": (self._topic(markdown_content="只有文字", text_content="只有文字"), 0, "只有文字"),
475
- "纯视频": (self._topic(video=video), 1, ""),
476
- }
477
- for label, (topic, media_count, expected_md) in cases.items():
478
- with self.subTest(case=label):
479
- result = await self._parse(topic)
480
- self.assertIsInstance(result, DoubanRichTextParseResult)
481
- self.assertEqual(len(result.media or []), media_count)
482
- self.assertEqual(result.markdown_content, expected_md)
483
-
484
- async def test_video_with_photos_keeps_video_first(self):
485
- result = await self._parse(
486
- self._topic(
487
- text_content="说明",
488
- markdown_content="说明",
489
- video=DoubanVideo(url="https://sv1.doubanio.com/a.mp4", duration=39),
490
- photos=[DoubanPhoto(url="https://img3.doubanio.com/p1.jpg", width=500, height=482)],
491
- )
492
- )
493
-
494
- self.assertIsInstance(result, DoubanRichTextParseResult)
495
- self.assertEqual(
496
- result.media,
497
- [
498
- VideoRef(url="https://sv1.doubanio.com/a.mp4", duration=39),
499
- ImageRef(url="https://img3.doubanio.com/p1.jpg", width=500, height=482),
500
- ],
501
- )
502
-
503
- async def test_animated_photo_becomes_ani_ref(self):
504
- result = await self._parse(
505
- self._topic(
506
- photos=[
507
- DoubanPhoto(
508
- url="https://img3.doubanio.com/p2.mp4",
509
- ext="mp4",
510
- thumb_url="https://img3.doubanio.com/p2.jpg",
511
- width=500,
512
- height=291,
513
- is_animated=True,
514
- )
515
- ]
516
- )
517
- )
518
-
519
- self.assertEqual(
520
- result.media,
521
- [
522
- AniRef(
523
- url="https://img3.doubanio.com/p2.mp4",
524
- ext="mp4",
525
- thumb_url="https://img3.doubanio.com/p2.jpg",
526
- width=500,
527
- height=291,
528
- )
529
- ],
530
- )
531
-
532
- async def test_provider_error_is_wrapped_in_parse_error(self):
533
- with patch.object(Douban, "parse", new=AsyncMock(side_effect=DoubanError("这篇内容不存在了"))):
534
- with self.assertRaisesRegex(ParseError, "豆瓣解析失败: 这篇内容不存在了"):
535
- await DoubanParser()._do_parse("https://www.douban.com/group/topic/1/")
536
-
537
- async def test_download_injects_douban_referer(self):
538
- result = DoubanRichTextParseResult(title="标题", media=[ImageRef(url="https://img3.doubanio.com/p1.jpg")])
539
-
540
- with patch.object(ParseResult, "_do_download", new=AsyncMock()) as mocked:
541
- await result._do_download(output_dir=Path("/tmp/does-not-matter"))
542
-
543
- # 豆瓣图床无 Referer 时返回 418
544
- self.assertEqual(mocked.await_args.kwargs["headers"]["Referer"], IMAGE_REFERER)
545
-
546
-
547
213
  class TestPlatformUrlMatching(unittest.TestCase):
548
214
  def test_supported_platform_url_formats(self):
549
215
  parsehub = ParseHub()