parsehub 2.3.0__tar.gz → 2.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. {parsehub-2.3.0/src/parsehub.egg-info → parsehub-2.3.2}/PKG-INFO +3 -1
  2. {parsehub-2.3.0 → parsehub-2.3.2}/README.md +2 -0
  3. {parsehub-2.3.0 → parsehub-2.3.2}/pyproject.toml +1 -1
  4. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/errors.py +6 -0
  5. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/__init__.py +1 -0
  6. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/bilibili.py +3 -0
  7. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/douyin.py +9 -0
  8. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/twitter.py +5 -0
  9. parsehub-2.3.2/src/parsehub/parsers/parser/wxchannels.py +29 -0
  10. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/xhs.py +2 -1
  11. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/bilibili.py +3 -1
  12. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/douyin.py +59 -11
  13. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/medium.py +3 -1
  14. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/toutiao.py +2 -1
  15. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/twitter.py +2 -2
  16. parsehub-2.3.2/src/parsehub/provider_api/wxchannels.py +129 -0
  17. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/xhs.py +2 -2
  18. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/types/__init__.py +2 -1
  19. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/types/platform.py +1 -0
  20. {parsehub-2.3.0 → parsehub-2.3.2/src/parsehub.egg-info}/PKG-INFO +3 -1
  21. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub.egg-info/SOURCES.txt +2 -0
  22. {parsehub-2.3.0 → parsehub-2.3.2}/LICENSE +0 -0
  23. {parsehub-2.3.0 → parsehub-2.3.2}/setup.cfg +0 -0
  24. {parsehub-2.3.0 → parsehub-2.3.2}/src/__init__.py +0 -0
  25. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/__init__.py +0 -0
  26. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/cli.py +0 -0
  27. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/cli_config.py +0 -0
  28. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/config/__init__.py +0 -0
  29. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/config/config.py +0 -0
  30. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/__init__.py +0 -0
  31. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/base/__init__.py +0 -0
  32. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/base/base.py +0 -0
  33. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/base/ytdlp.py +0 -0
  34. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/bangumi.py +0 -0
  35. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/coolapk.py +0 -0
  36. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/douban.py +0 -0
  37. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/facebook.py +0 -0
  38. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/instagram.py +0 -0
  39. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/kuaishou.py +0 -0
  40. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/medium.py +0 -0
  41. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/pipix.py +0 -0
  42. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/snapchat.py +0 -0
  43. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/threads.py +0 -0
  44. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/tieba.py +0 -0
  45. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/tiktok.py +0 -0
  46. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/toutiao.py +0 -0
  47. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/weibo.py +0 -0
  48. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/weixin.py +0 -0
  49. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/xiaoheihe.py +0 -0
  50. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/youtube.py +0 -0
  51. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/zhihu.py +0 -0
  52. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/zuiyou.py +0 -0
  53. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/__init__.py +0 -0
  54. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/bangumi.py +0 -0
  55. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/coolapk.py +0 -0
  56. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/douban.py +0 -0
  57. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/instagram.py +0 -0
  58. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/kuaishou.py +0 -0
  59. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/pipix.py +0 -0
  60. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/threads.py +0 -0
  61. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/tieba.py +0 -0
  62. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/tiktok.py +0 -0
  63. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/weibo.py +0 -0
  64. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/weixin.py +0 -0
  65. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/xiaoheihe.py +0 -0
  66. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/zhihu.py +0 -0
  67. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/zuiyou.py +0 -0
  68. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/types/callback.py +0 -0
  69. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/types/media_file.py +0 -0
  70. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/types/media_ref.py +0 -0
  71. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/types/post.py +0 -0
  72. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/types/result.py +0 -0
  73. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/utils/downloader.py +0 -0
  74. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/utils/helpers.py +0 -0
  75. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/utils/media_info.py +0 -0
  76. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub.egg-info/dependency_links.txt +0 -0
  77. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub.egg-info/entry_points.txt +0 -0
  78. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub.egg-info/requires.txt +0 -0
  79. {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub.egg-info/top_level.txt +0 -0
  80. {parsehub-2.3.0 → parsehub-2.3.2}/test/test_cli.py +0 -0
  81. {parsehub-2.3.0 → parsehub-2.3.2}/test/test_cli_config.py +0 -0
  82. {parsehub-2.3.0 → parsehub-2.3.2}/test/test_core_offline.py +0 -0
  83. {parsehub-2.3.0 → parsehub-2.3.2}/test/test_downloader.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: parsehub
3
- Version: 2.3.0
3
+ Version: 2.3.2
4
4
  Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
5
5
  Author-email: 梓澪 <zilingmio@gmail.com>
6
6
  License: MIT
@@ -88,6 +88,7 @@ Dynamic: license-file
88
88
  | **小红书** | ✅ | ✅ | |
89
89
  | **贴吧** | ✅ | ✅ | |
90
90
  | **微信公众号** | | ✅ | |
91
+ | **微信视频号** | ✅ | | |
91
92
  | **快手** | ✅ | ✅ | |
92
93
  | **酷安** | | ✅ | |
93
94
  | **皮皮虾** | ✅ | ✅ | |
@@ -222,6 +223,7 @@ print(result)
222
223
  - `小红书`
223
224
  - `知乎`
224
225
  - `豆瓣`
226
+ - `微信视频号`(需要填腾讯元宝 yuanbao.tencent.com 登录后的 Cookie)
225
227
 
226
228
  ```python
227
229
  from parsehub import ParseHub
@@ -44,6 +44,7 @@
44
44
  | **小红书** | ✅ | ✅ | |
45
45
  | **贴吧** | ✅ | ✅ | |
46
46
  | **微信公众号** | | ✅ | |
47
+ | **微信视频号** | ✅ | | |
47
48
  | **快手** | ✅ | ✅ | |
48
49
  | **酷安** | | ✅ | |
49
50
  | **皮皮虾** | ✅ | ✅ | |
@@ -178,6 +179,7 @@ print(result)
178
179
  - `小红书`
179
180
  - `知乎`
180
181
  - `豆瓣`
182
+ - `微信视频号`(需要填腾讯元宝 yuanbao.tencent.com 登录后的 Cookie)
181
183
 
182
184
  ```python
183
185
  from parsehub import ParseHub
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "parsehub"
3
- version = "2.3.0"
3
+ version = "2.3.2"
4
4
  description = "轻量、异步、开箱即用的社交媒体聚合解析库"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.12.0"
@@ -11,6 +11,12 @@ class ParseError(ParseHubError):
11
11
  MSG = __doc__
12
12
 
13
13
 
14
+ class ContentUnavailableError(ParseError):
15
+ """作品不可用"""
16
+
17
+ MSG = __doc__
18
+
19
+
14
20
  class DownloadError(ParseHubError):
15
21
  """下载错误"""
16
22
 
@@ -13,6 +13,7 @@ from .toutiao import *
13
13
  from .twitter import *
14
14
  from .weibo import *
15
15
  from .weixin import *
16
+ from .wxchannels import *
16
17
  from .xhs import *
17
18
  from .xiaoheihe import *
18
19
  from .youtube import *
@@ -9,6 +9,7 @@ from loguru import logger
9
9
 
10
10
  from ...provider_api.bilibili import BiliAPI, BiliDynamic
11
11
  from ...types import (
12
+ ContentUnavailableError,
12
13
  DownloadResult,
13
14
  ImageParseResult,
14
15
  ImageRef,
@@ -92,6 +93,8 @@ class BiliParse(BaseParser):
92
93
  try:
93
94
  async with BiliAPI(proxy=self.proxy) as bili:
94
95
  dynamic_info = await bili.get_dynamic_info(url, cookie=self.cookie.get_value())
96
+ except ContentUnavailableError:
97
+ raise
95
98
  except Exception as e:
96
99
  if "风控" in str(e):
97
100
  raise ParseError(f"账号风控\n使用的cookie: {self.cookie}") from e
@@ -6,6 +6,7 @@ from typing import Self, Union
6
6
  from ... import ProgressCallback
7
7
  from ...provider_api.douyin import DouyinMobileCrawler, DouyinMobileDevice, DouyinWebCrawler
8
8
  from ...types import (
9
+ ContentUnavailableError,
9
10
  DownloadResult,
10
11
  ImageParseResult,
11
12
  ImageRef,
@@ -46,6 +47,8 @@ class DouyinParser(BaseParser):
46
47
  web_crawler = DouyinWebCrawler(proxy=self.proxy, cookie=web_cookie)
47
48
  response = await web_crawler.parse(raw_url)
48
49
  return DouyinApiResult.parse(response)
50
+ except ContentUnavailableError:
51
+ raise
49
52
  except ParseError as e:
50
53
  web_error = e
51
54
 
@@ -57,6 +60,8 @@ class DouyinParser(BaseParser):
57
60
  except ParseError as e:
58
61
  mobile_error = e
59
62
 
63
+ if isinstance(mobile_error, ContentUnavailableError):
64
+ raise mobile_error
60
65
  if web_error:
61
66
  raise ParseError(f"{web_error};移动端解析也失败: {mobile_error}") from mobile_error
62
67
  raise mobile_error
@@ -188,6 +193,10 @@ class DouyinApiResult:
188
193
  def parse(cls, json_dict: dict) -> Self:
189
194
  data = json_dict.get("aweme_detail")
190
195
  if not data:
196
+ filter_detail = json_dict.get("filter_detail") or {}
197
+ if reason := filter_detail.get("filter_reason"):
198
+ detail_msg = filter_detail.get("detail_msg") or filter_detail.get("notice") or ""
199
+ raise ContentUnavailableError(f"抖音 {reason} {detail_msg}".strip())
191
200
  raise ParseError("抖音解析失败: 未获取到作品详情")
192
201
 
193
202
  desc = data.get("desc", "")
@@ -11,6 +11,7 @@ from ...provider_api.twitter import (
11
11
  from ...types import (
12
12
  AniRef,
13
13
  AnyMediaRef,
14
+ ContentUnavailableError,
14
15
  ImageRef,
15
16
  MultimediaParseResult,
16
17
  ParseError,
@@ -38,12 +39,16 @@ class TwitterParser(BaseParser):
38
39
  x = Twitter(self.proxy, cookie=None)
39
40
  try:
40
41
  tweet = await x.fetch_tweet(url)
42
+ except ContentUnavailableError:
43
+ raise
41
44
  except Exception as e:
42
45
  if any(s in str(e) for s in ("error -2", "error -3")):
43
46
  if cookie := self.cookie.get_value():
44
47
  x2 = Twitter(self.proxy, cookie=cookie)
45
48
  try:
46
49
  tweet = await x2.fetch_tweet(url)
50
+ except ContentUnavailableError:
51
+ raise
47
52
  except Exception as e2:
48
53
  raise ParseError(f"Twitter 账号无权限或已被封禁\n\n使用的 Cookie: {self.cookie}") from e2
49
54
  else:
@@ -0,0 +1,29 @@
1
+ from ...provider_api.wxchannels import WXChannels, WXChannelsError
2
+ from ...types import ParseError, Platform, VideoParseResult, VideoRef
3
+ from ..base.base import BaseParser
4
+
5
+
6
+ class WXChannelsParser(BaseParser):
7
+ __platform__ = Platform.WEIXIN_CHANNELS
8
+ __supported_type__ = ["视频"]
9
+ __match__ = (
10
+ r"^(http(s)?://)?(weixin\.qq\.com/sph/\w+|"
11
+ r"channels\.weixin\.qq\.com/finder-preview/pages/sph\?.*\bid=\w+)"
12
+ )
13
+ __reserved_parameters__ = ["id"]
14
+
15
+ async def _do_parse(self, raw_url: str) -> VideoParseResult:
16
+ cookie = self.cookie.get_value()
17
+ if not cookie:
18
+ raise ValueError("微信视频号需要配置腾讯元宝 (yuanbao.tencent.com) 登录 Cookie")
19
+
20
+ cookie_text = "; ".join(f"{key}={value}" for key, value in cookie.items())
21
+ try:
22
+ video = await WXChannels(cookie=cookie_text, proxy=self.proxy).parse(raw_url)
23
+ except WXChannelsError as e:
24
+ raise ParseError(f"微信视频号解析失败: {e}") from e
25
+
26
+ return VideoParseResult(title=video.title, video=VideoRef(url=video.video_url, thumb_url=video.thumb_url))
27
+
28
+
29
+ __all__ = ["WXChannelsParser"]
@@ -5,6 +5,7 @@ import httpx
5
5
 
6
6
  from ...provider_api.xhs import XHSAPI, XHSMedia, XHSMediaType, XHSPostType
7
7
  from ...types import (
8
+ ContentUnavailableError,
8
9
  ImageParseResult,
9
10
  ImageRef,
10
11
  LivePhotoRef,
@@ -35,7 +36,7 @@ class XHSParser(BaseParser):
35
36
  if "/login" in url:
36
37
  raise ParseError("该帖子需要登录后查看")
37
38
  elif "/404" in url:
38
- raise ParseError("帖子不存在")
39
+ raise ContentUnavailableError("帖子不存在")
39
40
  return url
40
41
 
41
42
  async def _do_parse(self, raw_url: str) -> Union["VideoParseResult", "ImageParseResult", "MultimediaParseResult"]:
@@ -12,6 +12,8 @@ from typing import Any, Self, cast
12
12
 
13
13
  import httpx
14
14
 
15
+ from ..errors import ContentUnavailableError
16
+
15
17
  USER_AGENT = (
16
18
  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36"
17
19
  )
@@ -68,7 +70,7 @@ class BiliAPI:
68
70
  case -352:
69
71
  raise Exception("获取动态信息失败: -352 风控限制")
70
72
  case 4101152:
71
- raise Exception("动态不可见")
73
+ raise ContentUnavailableError("动态不可见")
72
74
  case _:
73
75
  raise Exception(f"获取动态信息失败: {mj}")
74
76
  return BiliDynamic.parse(cast(dict[str, Any], data))
@@ -27,6 +27,8 @@ DEFAULT_USER_AGENT = (
27
27
  )
28
28
 
29
29
  POST_DETAIL = "https://www.douyin.com/aweme/v1/web/aweme/detail/"
30
+ WEB_SIGN_SALT = "A96D855A08C0A9707F8BEF0D9A527E4E"
31
+ WEB_SIGN_UIFID_COOKIES = ("uifid", "uifid_temp", "uifidtemp", "UIFID", "UIFID_TEMP", "UIFIDTEMP")
30
32
  MOBILE_BROWSER_USER_AGENT = (
31
33
  "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) "
32
34
  "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1"
@@ -754,6 +756,24 @@ class DouyinWebCrawler:
754
756
  "Referer": "https://www.douyin.com/",
755
757
  }
756
758
 
759
+ def _signed_query(self, params: dict[str, str]) -> tuple[str, dict[str, str]]:
760
+ uifid = next((v for name in WEB_SIGN_UIFID_COOKIES if (v := self.cookie.get(name))), None)
761
+ if not uifid:
762
+ raise ParseError("抖音 cookie 缺少 UIFID_TEMP, 无法生成网页签名")
763
+ pairs = [*params.items(), ("a_bogus", ABogus().get_value(params))]
764
+ if verify_fp := self.cookie.get("s_v_web_id"):
765
+ pairs += [("verifyFp", verify_fp), ("fp", verify_fp)]
766
+ timestamp = str(int(time.time()))
767
+ pairs += [("uifid", uifid), ("timestamp", timestamp)]
768
+ query = "&".join(f"{quote(k, safe='*-._')}={quote(str(v), safe='*-._')}" for k, v in pairs)
769
+ signature = hashlib.md5(f"{uifid}_{timestamp}_{WEB_SIGN_SALT}_{query}".encode()).hexdigest()
770
+ headers = {
771
+ "uifid": uifid,
772
+ "x-secsdk-web-signature": signature,
773
+ "x-secsdk-web-expire": timestamp,
774
+ }
775
+ return f"{query}&x-secsdk-web-signature={signature}", headers
776
+
757
777
  @staticmethod
758
778
  async def get_aweme_id(raw_url: str) -> str:
759
779
  for pattern in [
@@ -799,17 +819,23 @@ class DouyinWebCrawler:
799
819
  }
800
820
  for attempt in range(3):
801
821
  try:
802
- a_bogus = ABogus().get_value(params)
803
- endpoint = f"{POST_DETAIL}?{urlencode(params)}&a_bogus={quote(a_bogus, safe='')}"
804
- response = await client.get(endpoint)
805
- response.raise_for_status()
806
- return cast(dict[str, Any], response.json())
807
- except Exception as e:
822
+ query, sign_headers = self._signed_query(params)
823
+ response = await client.get(f"{POST_DETAIL}?{query}", headers=sign_headers)
824
+ except httpx.HTTPError as e:
808
825
  if attempt + 1 < 3:
809
826
  await asyncio.sleep(1)
810
827
  else:
811
- raise ParseError("获取抖音作品失败, 请检查 cookie") from e
812
- raise ParseError("获取抖音作品失败, 请检查 cookie")
828
+ raise ParseError(f"抖音网页端请求失败: {e}") from e
829
+ continue
830
+ if response.status_code != 200:
831
+ raise ParseError(f"抖音网页端请求被拒绝: HTTP {response.status_code} {response.text[:80]}")
832
+ if not response.content:
833
+ raise ParseError("抖音网页端返回空响应, cookie 可能已被限流")
834
+ try:
835
+ return cast(dict[str, Any], response.json())
836
+ except (ValueError, UnicodeDecodeError) as e:
837
+ raise ParseError("抖音网页端返回非 JSON 响应") from e
838
+ raise ParseError("抖音网页端请求失败: 未知错误")
813
839
 
814
840
  async def parse(self, raw_url: str) -> dict:
815
841
  aweme_id = await self.get_aweme_id(raw_url)
@@ -1101,11 +1127,23 @@ class DouyinMobileCrawler:
1101
1127
  cls._device_pool_index = (cls._device_pool_index + 1) % len(cls._device_pool)
1102
1128
  return device
1103
1129
 
1130
+ @classmethod
1131
+ def _evict_device(cls, device: DouyinMobileDevice) -> None:
1132
+ if device in cls._device_pool:
1133
+ cls._device_pool.remove(device)
1134
+ cls._device_pool_index = cls._device_pool_index % len(cls._device_pool) if cls._device_pool else 0
1135
+
1104
1136
  async def _ensure_device_pool(self, client: httpx.AsyncClient) -> None:
1105
- if self.__class__._device_pool:
1137
+ cls = self.__class__
1138
+ if len(cls._device_pool) >= MOBILE_DEVICE_POOL_SIZE:
1106
1139
  return
1107
- self.__class__._device_pool = await self.register_device_pool(client)
1108
- self.__class__._device_pool_index = 0
1140
+ missing = MOBILE_DEVICE_POOL_SIZE - len(cls._device_pool)
1141
+ new_devices = await self.register_device_pool(client, count=missing)
1142
+ current_pool = cls._device_pool
1143
+ cls._device_pool = (current_pool + [device for device in new_devices if device not in current_pool])[
1144
+ :MOBILE_DEVICE_POOL_SIZE
1145
+ ]
1146
+ cls._device_pool_index = cls._device_pool_index % len(cls._device_pool) if cls._device_pool else 0
1109
1147
 
1110
1148
  async def _select_device(self, client: httpx.AsyncClient) -> DouyinMobileDevice:
1111
1149
  if self._fixed_device:
@@ -1124,6 +1162,7 @@ class DouyinMobileCrawler:
1124
1162
  await self._select_device(client)
1125
1163
  params = self._mobile_query(aweme_id)
1126
1164
  query = urlencode(params)
1165
+ replace_device = False
1127
1166
  for profile in MOBILE_SIGN_PROFILES:
1128
1167
  headers = self._signed_headers(params, profile)
1129
1168
  for host in MOBILE_DETAIL_HOSTS:
@@ -1136,6 +1175,11 @@ class DouyinMobileCrawler:
1136
1175
  continue
1137
1176
  if not content:
1138
1177
  last_error = f"{host} returned empty body"
1178
+ if not self._fixed_device:
1179
+ if self.device is not None:
1180
+ self.__class__._evict_device(self.device)
1181
+ replace_device = True
1182
+ break
1139
1183
  continue
1140
1184
  try:
1141
1185
  payload = response.json()
@@ -1145,7 +1189,11 @@ class DouyinMobileCrawler:
1145
1189
  if payload.get("aweme_detail"):
1146
1190
  await self._attach_story_default_play(payload["aweme_detail"])
1147
1191
  return cast(dict[str, Any], payload)
1192
+ if payload.get("filter_detail"):
1193
+ return cast(dict[str, Any], payload)
1148
1194
  last_error = f"{host} missing aweme_detail: {payload.get('status_msg') or payload}"
1195
+ if replace_device:
1196
+ break
1149
1197
  await asyncio.sleep(0.15)
1150
1198
  raise ParseError(f"获取抖音作品失败: {last_error}")
1151
1199
 
@@ -7,6 +7,8 @@ from urllib.parse import urlparse
7
7
 
8
8
  from curl_cffi import requests
9
9
 
10
+ from ..errors import ContentUnavailableError
11
+
10
12
  _ID_RE = re.compile(r"(?:^|-)([0-9a-f]{8,12})$")
11
13
  _GRAPHQL_URL = "https://medium.com/_/graphql"
12
14
  _GRAPHQL_QUERY = """
@@ -99,7 +101,7 @@ class Medium:
99
101
 
100
102
  post = result["data"]["post"]
101
103
  if post is None:
102
- raise MediumError("内容不存在或已删除")
104
+ raise ContentUnavailableError("Medium 内容不存在或已删除")
103
105
 
104
106
  paragraphs = ((post.get("content") or {}).get("bodyModel") or {}).get("paragraphs") or []
105
107
  markdown_content, images = _render_paragraphs(paragraphs, post.get("title") or "")
@@ -10,6 +10,7 @@ import httpx
10
10
  from bs4 import BeautifulSoup
11
11
  from markdownify import MarkdownConverter
12
12
 
13
+ from ..errors import ContentUnavailableError
13
14
  from ..utils.helpers import UA
14
15
 
15
16
  _ID_RE = re.compile(r"toutiao\.com/(?:(?:article|video|w|group|item)/|a|i)(\d+)")
@@ -67,7 +68,7 @@ class Toutiao:
67
68
  response.raise_for_status()
68
69
  data = response.json().get("data")
69
70
  if data is None:
70
- raise ToutiaoError("内容不存在或已删除")
71
+ raise ContentUnavailableError("今日头条内容不存在或已删除")
71
72
 
72
73
  if "thread" in data:
73
74
  return self._parse_micro_post(data["thread"])
@@ -9,7 +9,7 @@ from typing import Literal, NamedTuple
9
9
  import httpx
10
10
  from loguru import logger
11
11
 
12
- from ..types import ParseError
12
+ from ..types import ContentUnavailableError
13
13
  from ..utils.helpers import UA
14
14
 
15
15
 
@@ -79,7 +79,7 @@ class Twitter:
79
79
 
80
80
  result = result["data"]["tweetResult"].get("result")
81
81
  if not result:
82
- raise ParseError("error -4: 帖子或用户不存在")
82
+ raise ContentUnavailableError("error -4: 帖子或用户不存在")
83
83
 
84
84
  if tweet := result.get("tweet"):
85
85
  tweet_id = tweet.get("rest_id", {})
@@ -0,0 +1,129 @@
1
+ from __future__ import annotations
2
+
3
+ import html
4
+ import re
5
+ import secrets
6
+ import time
7
+ from dataclasses import dataclass
8
+ from urllib.parse import parse_qs, urlparse
9
+
10
+ import httpx
11
+
12
+ from ..errors import ContentUnavailableError
13
+ from ..utils.helpers import UA
14
+
15
+ _SHARE_URL = "https://weixin.qq.com/sph/{}"
16
+ _YUANBAO_API = "https://yuanbao.tencent.com/api/weixin/get_parse_result"
17
+ _FEED_API = "https://channels.weixin.qq.com/finder-preview/api/feed/get_feed_info"
18
+ _FEED_PAGE = "https://channels.weixin.qq.com/finder-preview/pages/feed"
19
+ _ID_RE = re.compile(
20
+ r"(?:weixin\.qq\.com/sph/(\w+)|channels\.weixin\.qq\.com/finder-preview/pages/sph\?[^#]*?\bid=(\w+))"
21
+ )
22
+
23
+
24
+ @dataclass
25
+ class WXChannelsVideo:
26
+ title: str
27
+ author: str
28
+ video_url: str
29
+ thumb_url: str | None
30
+
31
+
32
+ class WXChannelsError(Exception):
33
+ pass
34
+
35
+
36
+ class WXChannels:
37
+ def __init__(self, cookie: str, proxy: str | None = None):
38
+ self.cookie = cookie
39
+ self.proxy = proxy
40
+
41
+ async def parse(self, url: str) -> WXChannelsVideo:
42
+ match = _ID_RE.search(url)
43
+ if not match:
44
+ raise WXChannelsError("不支持的微信视频号链接")
45
+ share_id = match.group(1) or match.group(2)
46
+
47
+ async with httpx.AsyncClient(proxy=self.proxy, timeout=30) as client:
48
+ response = await client.post(
49
+ _YUANBAO_API,
50
+ json={"type": "video_channel_url", "url": _SHARE_URL.format(share_id), "scene": 1},
51
+ headers={
52
+ "content-type": "application/json",
53
+ "User-Agent": UA,
54
+ "origin": "https://yuanbao.tencent.com",
55
+ "referer": "https://yuanbao.tencent.com/",
56
+ "cookie": self.cookie,
57
+ },
58
+ )
59
+ if response.status_code == 401:
60
+ raise WXChannelsError("Cookie 无效或已过期")
61
+ response.raise_for_status()
62
+ result = response.json()
63
+ if error := result.get("error"):
64
+ if str(error.get("code")) == "20000":
65
+ raise WXChannelsError("Cookie 无效或已过期")
66
+ raise WXChannelsError(str(error.get("message") or error))
67
+ if result.get("code") != 0:
68
+ raise WXChannelsError(str(result.get("message") or "微信视频号解析失败"))
69
+
70
+ data = result.get("data") or {}
71
+ playable_url = data.get("playable_url") or ""
72
+ if not data.get("wx_export_id") or not playable_url:
73
+ raise ContentUnavailableError("微信视频号视频不存在或已删除")
74
+
75
+ playable_query = parse_qs(urlparse(playable_url).query)
76
+ token = (playable_query.get("token") or [""])[0]
77
+ export_id = (playable_query.get("eid") or [""])[0]
78
+ if not token or not export_id:
79
+ raise WXChannelsError("无法获取视频信息")
80
+
81
+ rid = f"{int(time.time()):x}-{secrets.token_hex(4)}"
82
+ response = await client.post(
83
+ _FEED_API,
84
+ params={"_rid": rid, "_pageUrl": _FEED_PAGE},
85
+ json={"baseReq": {"generalToken": token}, "exportId": export_id},
86
+ headers={
87
+ "content-type": "application/json",
88
+ "User-Agent": UA,
89
+ "origin": "https://channels.weixin.qq.com",
90
+ "referer": playable_url,
91
+ },
92
+ )
93
+ response.raise_for_status()
94
+ feed_result = response.json()
95
+
96
+ feed_data = feed_result.get("data") or {}
97
+ feed_error = feed_data.get("errMsg") or {}
98
+ error_text = " ".join(
99
+ _strip_html(str(feed_error.get(key) or "")) for key in ("title", "content")
100
+ ).strip()
101
+ if feed_result.get("errCode") != 0:
102
+ message = feed_result.get("errMsg") or "微信视频号解析失败"
103
+ raise WXChannelsError(str(message))
104
+ if error_text:
105
+ raise WXChannelsError(error_text)
106
+
107
+ feed_info = feed_data.get("feedInfo") or {}
108
+ video_url = (
109
+ ((feed_info.get("h264VideoInfo") or {}).get("videoUrl"))
110
+ or feed_info.get("videoUrl")
111
+ or ((feed_info.get("h265VideoInfo") or {}).get("videoUrl"))
112
+ )
113
+ if not video_url:
114
+ raise WXChannelsError("暂不支持该类型")
115
+
116
+ author_info = feed_data.get("authorInfo") or {}
117
+ return WXChannelsVideo(
118
+ title=feed_info.get("description") or data.get("desc") or "",
119
+ author=author_info.get("nickname") or data.get("author") or "",
120
+ video_url=video_url,
121
+ thumb_url=feed_info.get("coverUrl") or data.get("cover_url") or None,
122
+ )
123
+
124
+
125
+ def _strip_html(value: str) -> str:
126
+ return html.unescape(re.sub(r"<[^>]*>", "", value)).strip()
127
+
128
+
129
+ __all__ = ["WXChannels", "WXChannelsError", "WXChannelsVideo"]
@@ -10,7 +10,7 @@ from typing import Any, cast
10
10
  import httpx
11
11
  from bs4 import BeautifulSoup
12
12
 
13
- from ..errors import ParseError
13
+ from ..errors import ContentUnavailableError, ParseError
14
14
  from ..utils.helpers import UA
15
15
 
16
16
 
@@ -33,7 +33,7 @@ class XHSAPI:
33
33
  if "/login" in str(result.url):
34
34
  raise ParseError("该帖子需要登录后查看")
35
35
  elif "/404" in str(result.url):
36
- raise ParseError("帖子不存在")
36
+ raise ContentUnavailableError("帖子不存在")
37
37
  return result.text
38
38
 
39
39
  @staticmethod
@@ -1,4 +1,4 @@
1
- from ..errors import DownloadError, ParseError
1
+ from ..errors import ContentUnavailableError, DownloadError, ParseError
2
2
  from .callback import ProgressCallback, ProgressUnit
3
3
  from .media_file import AniFile, AnyMediaFile, ImageFile, LivePhotoFile, MediaFile, VideoFile
4
4
  from .media_ref import AniRef, AnyMediaRef, ImageRef, LivePhotoRef, MediaRef, VideoRef
@@ -17,6 +17,7 @@ from .result import (
17
17
  __all__ = [
18
18
  "DownloadError",
19
19
  "ParseError",
20
+ "ContentUnavailableError",
20
21
  "MediaRef",
21
22
  "AniRef",
22
23
  "VideoRef",
@@ -21,6 +21,7 @@ class Platform(Enum):
21
21
  TWITTER = ("twitter", "Twitter")
22
22
  WEIBO = ("weibo", "微博")
23
23
  WEIXIN = ("weixin", "微信公众号")
24
+ WEIXIN_CHANNELS = ("weixin_channels", "微信视频号")
24
25
  XHS = ("xhs", "小红书")
25
26
  XIAOHEIHE = ("xiaoheihe", "小黑盒")
26
27
  YOUTUBE = ("youtube", "Youtube")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: parsehub
3
- Version: 2.3.0
3
+ Version: 2.3.2
4
4
  Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
5
5
  Author-email: 梓澪 <zilingmio@gmail.com>
6
6
  License: MIT
@@ -88,6 +88,7 @@ Dynamic: license-file
88
88
  | **小红书** | ✅ | ✅ | |
89
89
  | **贴吧** | ✅ | ✅ | |
90
90
  | **微信公众号** | | ✅ | |
91
+ | **微信视频号** | ✅ | | |
91
92
  | **快手** | ✅ | ✅ | |
92
93
  | **酷安** | | ✅ | |
93
94
  | **皮皮虾** | ✅ | ✅ | |
@@ -222,6 +223,7 @@ print(result)
222
223
  - `小红书`
223
224
  - `知乎`
224
225
  - `豆瓣`
226
+ - `微信视频号`(需要填腾讯元宝 yuanbao.tencent.com 登录后的 Cookie)
225
227
 
226
228
  ```python
227
229
  from parsehub import ParseHub
@@ -37,6 +37,7 @@ src/parsehub/parsers/parser/toutiao.py
37
37
  src/parsehub/parsers/parser/twitter.py
38
38
  src/parsehub/parsers/parser/weibo.py
39
39
  src/parsehub/parsers/parser/weixin.py
40
+ src/parsehub/parsers/parser/wxchannels.py
40
41
  src/parsehub/parsers/parser/xhs.py
41
42
  src/parsehub/parsers/parser/xiaoheihe.py
42
43
  src/parsehub/parsers/parser/youtube.py
@@ -59,6 +60,7 @@ src/parsehub/provider_api/toutiao.py
59
60
  src/parsehub/provider_api/twitter.py
60
61
  src/parsehub/provider_api/weibo.py
61
62
  src/parsehub/provider_api/weixin.py
63
+ src/parsehub/provider_api/wxchannels.py
62
64
  src/parsehub/provider_api/xhs.py
63
65
  src/parsehub/provider_api/xiaoheihe.py
64
66
  src/parsehub/provider_api/zhihu.py
File without changes
File without changes
File without changes
File without changes
File without changes