parsehub 2.3.0__tar.gz → 2.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {parsehub-2.3.0/src/parsehub.egg-info → parsehub-2.3.2}/PKG-INFO +3 -1
- {parsehub-2.3.0 → parsehub-2.3.2}/README.md +2 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/pyproject.toml +1 -1
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/errors.py +6 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/__init__.py +1 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/bilibili.py +3 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/douyin.py +9 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/twitter.py +5 -0
- parsehub-2.3.2/src/parsehub/parsers/parser/wxchannels.py +29 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/xhs.py +2 -1
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/bilibili.py +3 -1
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/douyin.py +59 -11
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/medium.py +3 -1
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/toutiao.py +2 -1
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/twitter.py +2 -2
- parsehub-2.3.2/src/parsehub/provider_api/wxchannels.py +129 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/xhs.py +2 -2
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/types/__init__.py +2 -1
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/types/platform.py +1 -0
- {parsehub-2.3.0 → parsehub-2.3.2/src/parsehub.egg-info}/PKG-INFO +3 -1
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub.egg-info/SOURCES.txt +2 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/LICENSE +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/setup.cfg +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/__init__.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/__init__.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/cli.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/cli_config.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/config/__init__.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/config/config.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/__init__.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/base/__init__.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/base/base.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/base/ytdlp.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/bangumi.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/coolapk.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/douban.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/facebook.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/instagram.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/kuaishou.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/medium.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/pipix.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/snapchat.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/threads.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/tieba.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/tiktok.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/toutiao.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/weibo.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/weixin.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/xiaoheihe.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/youtube.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/zhihu.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/parsers/parser/zuiyou.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/__init__.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/bangumi.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/coolapk.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/douban.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/instagram.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/kuaishou.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/pipix.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/threads.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/tieba.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/tiktok.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/weibo.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/weixin.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/xiaoheihe.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/zhihu.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/provider_api/zuiyou.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/types/callback.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/types/media_file.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/types/media_ref.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/types/post.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/types/result.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/utils/downloader.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/utils/helpers.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub/utils/media_info.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub.egg-info/dependency_links.txt +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub.egg-info/entry_points.txt +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub.egg-info/requires.txt +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/src/parsehub.egg-info/top_level.txt +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/test/test_cli.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/test/test_cli_config.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/test/test_core_offline.py +0 -0
- {parsehub-2.3.0 → parsehub-2.3.2}/test/test_downloader.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: parsehub
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.2
|
|
4
4
|
Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
|
|
5
5
|
Author-email: 梓澪 <zilingmio@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -88,6 +88,7 @@ Dynamic: license-file
|
|
|
88
88
|
| **小红书** | ✅ | ✅ | |
|
|
89
89
|
| **贴吧** | ✅ | ✅ | |
|
|
90
90
|
| **微信公众号** | | ✅ | |
|
|
91
|
+
| **微信视频号** | ✅ | | |
|
|
91
92
|
| **快手** | ✅ | ✅ | |
|
|
92
93
|
| **酷安** | | ✅ | |
|
|
93
94
|
| **皮皮虾** | ✅ | ✅ | |
|
|
@@ -222,6 +223,7 @@ print(result)
|
|
|
222
223
|
- `小红书`
|
|
223
224
|
- `知乎`
|
|
224
225
|
- `豆瓣`
|
|
226
|
+
- `微信视频号`(需要填腾讯元宝 yuanbao.tencent.com 登录后的 Cookie)
|
|
225
227
|
|
|
226
228
|
```python
|
|
227
229
|
from parsehub import ParseHub
|
|
@@ -44,6 +44,7 @@
|
|
|
44
44
|
| **小红书** | ✅ | ✅ | |
|
|
45
45
|
| **贴吧** | ✅ | ✅ | |
|
|
46
46
|
| **微信公众号** | | ✅ | |
|
|
47
|
+
| **微信视频号** | ✅ | | |
|
|
47
48
|
| **快手** | ✅ | ✅ | |
|
|
48
49
|
| **酷安** | | ✅ | |
|
|
49
50
|
| **皮皮虾** | ✅ | ✅ | |
|
|
@@ -178,6 +179,7 @@ print(result)
|
|
|
178
179
|
- `小红书`
|
|
179
180
|
- `知乎`
|
|
180
181
|
- `豆瓣`
|
|
182
|
+
- `微信视频号`(需要填腾讯元宝 yuanbao.tencent.com 登录后的 Cookie)
|
|
181
183
|
|
|
182
184
|
```python
|
|
183
185
|
from parsehub import ParseHub
|
|
@@ -9,6 +9,7 @@ from loguru import logger
|
|
|
9
9
|
|
|
10
10
|
from ...provider_api.bilibili import BiliAPI, BiliDynamic
|
|
11
11
|
from ...types import (
|
|
12
|
+
ContentUnavailableError,
|
|
12
13
|
DownloadResult,
|
|
13
14
|
ImageParseResult,
|
|
14
15
|
ImageRef,
|
|
@@ -92,6 +93,8 @@ class BiliParse(BaseParser):
|
|
|
92
93
|
try:
|
|
93
94
|
async with BiliAPI(proxy=self.proxy) as bili:
|
|
94
95
|
dynamic_info = await bili.get_dynamic_info(url, cookie=self.cookie.get_value())
|
|
96
|
+
except ContentUnavailableError:
|
|
97
|
+
raise
|
|
95
98
|
except Exception as e:
|
|
96
99
|
if "风控" in str(e):
|
|
97
100
|
raise ParseError(f"账号风控\n使用的cookie: {self.cookie}") from e
|
|
@@ -6,6 +6,7 @@ from typing import Self, Union
|
|
|
6
6
|
from ... import ProgressCallback
|
|
7
7
|
from ...provider_api.douyin import DouyinMobileCrawler, DouyinMobileDevice, DouyinWebCrawler
|
|
8
8
|
from ...types import (
|
|
9
|
+
ContentUnavailableError,
|
|
9
10
|
DownloadResult,
|
|
10
11
|
ImageParseResult,
|
|
11
12
|
ImageRef,
|
|
@@ -46,6 +47,8 @@ class DouyinParser(BaseParser):
|
|
|
46
47
|
web_crawler = DouyinWebCrawler(proxy=self.proxy, cookie=web_cookie)
|
|
47
48
|
response = await web_crawler.parse(raw_url)
|
|
48
49
|
return DouyinApiResult.parse(response)
|
|
50
|
+
except ContentUnavailableError:
|
|
51
|
+
raise
|
|
49
52
|
except ParseError as e:
|
|
50
53
|
web_error = e
|
|
51
54
|
|
|
@@ -57,6 +60,8 @@ class DouyinParser(BaseParser):
|
|
|
57
60
|
except ParseError as e:
|
|
58
61
|
mobile_error = e
|
|
59
62
|
|
|
63
|
+
if isinstance(mobile_error, ContentUnavailableError):
|
|
64
|
+
raise mobile_error
|
|
60
65
|
if web_error:
|
|
61
66
|
raise ParseError(f"{web_error};移动端解析也失败: {mobile_error}") from mobile_error
|
|
62
67
|
raise mobile_error
|
|
@@ -188,6 +193,10 @@ class DouyinApiResult:
|
|
|
188
193
|
def parse(cls, json_dict: dict) -> Self:
|
|
189
194
|
data = json_dict.get("aweme_detail")
|
|
190
195
|
if not data:
|
|
196
|
+
filter_detail = json_dict.get("filter_detail") or {}
|
|
197
|
+
if reason := filter_detail.get("filter_reason"):
|
|
198
|
+
detail_msg = filter_detail.get("detail_msg") or filter_detail.get("notice") or ""
|
|
199
|
+
raise ContentUnavailableError(f"抖音 {reason} {detail_msg}".strip())
|
|
191
200
|
raise ParseError("抖音解析失败: 未获取到作品详情")
|
|
192
201
|
|
|
193
202
|
desc = data.get("desc", "")
|
|
@@ -11,6 +11,7 @@ from ...provider_api.twitter import (
|
|
|
11
11
|
from ...types import (
|
|
12
12
|
AniRef,
|
|
13
13
|
AnyMediaRef,
|
|
14
|
+
ContentUnavailableError,
|
|
14
15
|
ImageRef,
|
|
15
16
|
MultimediaParseResult,
|
|
16
17
|
ParseError,
|
|
@@ -38,12 +39,16 @@ class TwitterParser(BaseParser):
|
|
|
38
39
|
x = Twitter(self.proxy, cookie=None)
|
|
39
40
|
try:
|
|
40
41
|
tweet = await x.fetch_tweet(url)
|
|
42
|
+
except ContentUnavailableError:
|
|
43
|
+
raise
|
|
41
44
|
except Exception as e:
|
|
42
45
|
if any(s in str(e) for s in ("error -2", "error -3")):
|
|
43
46
|
if cookie := self.cookie.get_value():
|
|
44
47
|
x2 = Twitter(self.proxy, cookie=cookie)
|
|
45
48
|
try:
|
|
46
49
|
tweet = await x2.fetch_tweet(url)
|
|
50
|
+
except ContentUnavailableError:
|
|
51
|
+
raise
|
|
47
52
|
except Exception as e2:
|
|
48
53
|
raise ParseError(f"Twitter 账号无权限或已被封禁\n\n使用的 Cookie: {self.cookie}") from e2
|
|
49
54
|
else:
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from ...provider_api.wxchannels import WXChannels, WXChannelsError
|
|
2
|
+
from ...types import ParseError, Platform, VideoParseResult, VideoRef
|
|
3
|
+
from ..base.base import BaseParser
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class WXChannelsParser(BaseParser):
|
|
7
|
+
__platform__ = Platform.WEIXIN_CHANNELS
|
|
8
|
+
__supported_type__ = ["视频"]
|
|
9
|
+
__match__ = (
|
|
10
|
+
r"^(http(s)?://)?(weixin\.qq\.com/sph/\w+|"
|
|
11
|
+
r"channels\.weixin\.qq\.com/finder-preview/pages/sph\?.*\bid=\w+)"
|
|
12
|
+
)
|
|
13
|
+
__reserved_parameters__ = ["id"]
|
|
14
|
+
|
|
15
|
+
async def _do_parse(self, raw_url: str) -> VideoParseResult:
|
|
16
|
+
cookie = self.cookie.get_value()
|
|
17
|
+
if not cookie:
|
|
18
|
+
raise ValueError("微信视频号需要配置腾讯元宝 (yuanbao.tencent.com) 登录 Cookie")
|
|
19
|
+
|
|
20
|
+
cookie_text = "; ".join(f"{key}={value}" for key, value in cookie.items())
|
|
21
|
+
try:
|
|
22
|
+
video = await WXChannels(cookie=cookie_text, proxy=self.proxy).parse(raw_url)
|
|
23
|
+
except WXChannelsError as e:
|
|
24
|
+
raise ParseError(f"微信视频号解析失败: {e}") from e
|
|
25
|
+
|
|
26
|
+
return VideoParseResult(title=video.title, video=VideoRef(url=video.video_url, thumb_url=video.thumb_url))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
__all__ = ["WXChannelsParser"]
|
|
@@ -5,6 +5,7 @@ import httpx
|
|
|
5
5
|
|
|
6
6
|
from ...provider_api.xhs import XHSAPI, XHSMedia, XHSMediaType, XHSPostType
|
|
7
7
|
from ...types import (
|
|
8
|
+
ContentUnavailableError,
|
|
8
9
|
ImageParseResult,
|
|
9
10
|
ImageRef,
|
|
10
11
|
LivePhotoRef,
|
|
@@ -35,7 +36,7 @@ class XHSParser(BaseParser):
|
|
|
35
36
|
if "/login" in url:
|
|
36
37
|
raise ParseError("该帖子需要登录后查看")
|
|
37
38
|
elif "/404" in url:
|
|
38
|
-
raise
|
|
39
|
+
raise ContentUnavailableError("帖子不存在")
|
|
39
40
|
return url
|
|
40
41
|
|
|
41
42
|
async def _do_parse(self, raw_url: str) -> Union["VideoParseResult", "ImageParseResult", "MultimediaParseResult"]:
|
|
@@ -12,6 +12,8 @@ from typing import Any, Self, cast
|
|
|
12
12
|
|
|
13
13
|
import httpx
|
|
14
14
|
|
|
15
|
+
from ..errors import ContentUnavailableError
|
|
16
|
+
|
|
15
17
|
USER_AGENT = (
|
|
16
18
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36"
|
|
17
19
|
)
|
|
@@ -68,7 +70,7 @@ class BiliAPI:
|
|
|
68
70
|
case -352:
|
|
69
71
|
raise Exception("获取动态信息失败: -352 风控限制")
|
|
70
72
|
case 4101152:
|
|
71
|
-
raise
|
|
73
|
+
raise ContentUnavailableError("动态不可见")
|
|
72
74
|
case _:
|
|
73
75
|
raise Exception(f"获取动态信息失败: {mj}")
|
|
74
76
|
return BiliDynamic.parse(cast(dict[str, Any], data))
|
|
@@ -27,6 +27,8 @@ DEFAULT_USER_AGENT = (
|
|
|
27
27
|
)
|
|
28
28
|
|
|
29
29
|
POST_DETAIL = "https://www.douyin.com/aweme/v1/web/aweme/detail/"
|
|
30
|
+
WEB_SIGN_SALT = "A96D855A08C0A9707F8BEF0D9A527E4E"
|
|
31
|
+
WEB_SIGN_UIFID_COOKIES = ("uifid", "uifid_temp", "uifidtemp", "UIFID", "UIFID_TEMP", "UIFIDTEMP")
|
|
30
32
|
MOBILE_BROWSER_USER_AGENT = (
|
|
31
33
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) "
|
|
32
34
|
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1"
|
|
@@ -754,6 +756,24 @@ class DouyinWebCrawler:
|
|
|
754
756
|
"Referer": "https://www.douyin.com/",
|
|
755
757
|
}
|
|
756
758
|
|
|
759
|
+
def _signed_query(self, params: dict[str, str]) -> tuple[str, dict[str, str]]:
|
|
760
|
+
uifid = next((v for name in WEB_SIGN_UIFID_COOKIES if (v := self.cookie.get(name))), None)
|
|
761
|
+
if not uifid:
|
|
762
|
+
raise ParseError("抖音 cookie 缺少 UIFID_TEMP, 无法生成网页签名")
|
|
763
|
+
pairs = [*params.items(), ("a_bogus", ABogus().get_value(params))]
|
|
764
|
+
if verify_fp := self.cookie.get("s_v_web_id"):
|
|
765
|
+
pairs += [("verifyFp", verify_fp), ("fp", verify_fp)]
|
|
766
|
+
timestamp = str(int(time.time()))
|
|
767
|
+
pairs += [("uifid", uifid), ("timestamp", timestamp)]
|
|
768
|
+
query = "&".join(f"{quote(k, safe='*-._')}={quote(str(v), safe='*-._')}" for k, v in pairs)
|
|
769
|
+
signature = hashlib.md5(f"{uifid}_{timestamp}_{WEB_SIGN_SALT}_{query}".encode()).hexdigest()
|
|
770
|
+
headers = {
|
|
771
|
+
"uifid": uifid,
|
|
772
|
+
"x-secsdk-web-signature": signature,
|
|
773
|
+
"x-secsdk-web-expire": timestamp,
|
|
774
|
+
}
|
|
775
|
+
return f"{query}&x-secsdk-web-signature={signature}", headers
|
|
776
|
+
|
|
757
777
|
@staticmethod
|
|
758
778
|
async def get_aweme_id(raw_url: str) -> str:
|
|
759
779
|
for pattern in [
|
|
@@ -799,17 +819,23 @@ class DouyinWebCrawler:
|
|
|
799
819
|
}
|
|
800
820
|
for attempt in range(3):
|
|
801
821
|
try:
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
response.raise_for_status()
|
|
806
|
-
return cast(dict[str, Any], response.json())
|
|
807
|
-
except Exception as e:
|
|
822
|
+
query, sign_headers = self._signed_query(params)
|
|
823
|
+
response = await client.get(f"{POST_DETAIL}?{query}", headers=sign_headers)
|
|
824
|
+
except httpx.HTTPError as e:
|
|
808
825
|
if attempt + 1 < 3:
|
|
809
826
|
await asyncio.sleep(1)
|
|
810
827
|
else:
|
|
811
|
-
raise ParseError("
|
|
812
|
-
|
|
828
|
+
raise ParseError(f"抖音网页端请求失败: {e}") from e
|
|
829
|
+
continue
|
|
830
|
+
if response.status_code != 200:
|
|
831
|
+
raise ParseError(f"抖音网页端请求被拒绝: HTTP {response.status_code} {response.text[:80]}")
|
|
832
|
+
if not response.content:
|
|
833
|
+
raise ParseError("抖音网页端返回空响应, cookie 可能已被限流")
|
|
834
|
+
try:
|
|
835
|
+
return cast(dict[str, Any], response.json())
|
|
836
|
+
except (ValueError, UnicodeDecodeError) as e:
|
|
837
|
+
raise ParseError("抖音网页端返回非 JSON 响应") from e
|
|
838
|
+
raise ParseError("抖音网页端请求失败: 未知错误")
|
|
813
839
|
|
|
814
840
|
async def parse(self, raw_url: str) -> dict:
|
|
815
841
|
aweme_id = await self.get_aweme_id(raw_url)
|
|
@@ -1101,11 +1127,23 @@ class DouyinMobileCrawler:
|
|
|
1101
1127
|
cls._device_pool_index = (cls._device_pool_index + 1) % len(cls._device_pool)
|
|
1102
1128
|
return device
|
|
1103
1129
|
|
|
1130
|
+
@classmethod
|
|
1131
|
+
def _evict_device(cls, device: DouyinMobileDevice) -> None:
|
|
1132
|
+
if device in cls._device_pool:
|
|
1133
|
+
cls._device_pool.remove(device)
|
|
1134
|
+
cls._device_pool_index = cls._device_pool_index % len(cls._device_pool) if cls._device_pool else 0
|
|
1135
|
+
|
|
1104
1136
|
async def _ensure_device_pool(self, client: httpx.AsyncClient) -> None:
|
|
1105
|
-
|
|
1137
|
+
cls = self.__class__
|
|
1138
|
+
if len(cls._device_pool) >= MOBILE_DEVICE_POOL_SIZE:
|
|
1106
1139
|
return
|
|
1107
|
-
|
|
1108
|
-
self.
|
|
1140
|
+
missing = MOBILE_DEVICE_POOL_SIZE - len(cls._device_pool)
|
|
1141
|
+
new_devices = await self.register_device_pool(client, count=missing)
|
|
1142
|
+
current_pool = cls._device_pool
|
|
1143
|
+
cls._device_pool = (current_pool + [device for device in new_devices if device not in current_pool])[
|
|
1144
|
+
:MOBILE_DEVICE_POOL_SIZE
|
|
1145
|
+
]
|
|
1146
|
+
cls._device_pool_index = cls._device_pool_index % len(cls._device_pool) if cls._device_pool else 0
|
|
1109
1147
|
|
|
1110
1148
|
async def _select_device(self, client: httpx.AsyncClient) -> DouyinMobileDevice:
|
|
1111
1149
|
if self._fixed_device:
|
|
@@ -1124,6 +1162,7 @@ class DouyinMobileCrawler:
|
|
|
1124
1162
|
await self._select_device(client)
|
|
1125
1163
|
params = self._mobile_query(aweme_id)
|
|
1126
1164
|
query = urlencode(params)
|
|
1165
|
+
replace_device = False
|
|
1127
1166
|
for profile in MOBILE_SIGN_PROFILES:
|
|
1128
1167
|
headers = self._signed_headers(params, profile)
|
|
1129
1168
|
for host in MOBILE_DETAIL_HOSTS:
|
|
@@ -1136,6 +1175,11 @@ class DouyinMobileCrawler:
|
|
|
1136
1175
|
continue
|
|
1137
1176
|
if not content:
|
|
1138
1177
|
last_error = f"{host} returned empty body"
|
|
1178
|
+
if not self._fixed_device:
|
|
1179
|
+
if self.device is not None:
|
|
1180
|
+
self.__class__._evict_device(self.device)
|
|
1181
|
+
replace_device = True
|
|
1182
|
+
break
|
|
1139
1183
|
continue
|
|
1140
1184
|
try:
|
|
1141
1185
|
payload = response.json()
|
|
@@ -1145,7 +1189,11 @@ class DouyinMobileCrawler:
|
|
|
1145
1189
|
if payload.get("aweme_detail"):
|
|
1146
1190
|
await self._attach_story_default_play(payload["aweme_detail"])
|
|
1147
1191
|
return cast(dict[str, Any], payload)
|
|
1192
|
+
if payload.get("filter_detail"):
|
|
1193
|
+
return cast(dict[str, Any], payload)
|
|
1148
1194
|
last_error = f"{host} missing aweme_detail: {payload.get('status_msg') or payload}"
|
|
1195
|
+
if replace_device:
|
|
1196
|
+
break
|
|
1149
1197
|
await asyncio.sleep(0.15)
|
|
1150
1198
|
raise ParseError(f"获取抖音作品失败: {last_error}")
|
|
1151
1199
|
|
|
@@ -7,6 +7,8 @@ from urllib.parse import urlparse
|
|
|
7
7
|
|
|
8
8
|
from curl_cffi import requests
|
|
9
9
|
|
|
10
|
+
from ..errors import ContentUnavailableError
|
|
11
|
+
|
|
10
12
|
_ID_RE = re.compile(r"(?:^|-)([0-9a-f]{8,12})$")
|
|
11
13
|
_GRAPHQL_URL = "https://medium.com/_/graphql"
|
|
12
14
|
_GRAPHQL_QUERY = """
|
|
@@ -99,7 +101,7 @@ class Medium:
|
|
|
99
101
|
|
|
100
102
|
post = result["data"]["post"]
|
|
101
103
|
if post is None:
|
|
102
|
-
raise
|
|
104
|
+
raise ContentUnavailableError("Medium 内容不存在或已删除")
|
|
103
105
|
|
|
104
106
|
paragraphs = ((post.get("content") or {}).get("bodyModel") or {}).get("paragraphs") or []
|
|
105
107
|
markdown_content, images = _render_paragraphs(paragraphs, post.get("title") or "")
|
|
@@ -10,6 +10,7 @@ import httpx
|
|
|
10
10
|
from bs4 import BeautifulSoup
|
|
11
11
|
from markdownify import MarkdownConverter
|
|
12
12
|
|
|
13
|
+
from ..errors import ContentUnavailableError
|
|
13
14
|
from ..utils.helpers import UA
|
|
14
15
|
|
|
15
16
|
_ID_RE = re.compile(r"toutiao\.com/(?:(?:article|video|w|group|item)/|a|i)(\d+)")
|
|
@@ -67,7 +68,7 @@ class Toutiao:
|
|
|
67
68
|
response.raise_for_status()
|
|
68
69
|
data = response.json().get("data")
|
|
69
70
|
if data is None:
|
|
70
|
-
raise
|
|
71
|
+
raise ContentUnavailableError("今日头条内容不存在或已删除")
|
|
71
72
|
|
|
72
73
|
if "thread" in data:
|
|
73
74
|
return self._parse_micro_post(data["thread"])
|
|
@@ -9,7 +9,7 @@ from typing import Literal, NamedTuple
|
|
|
9
9
|
import httpx
|
|
10
10
|
from loguru import logger
|
|
11
11
|
|
|
12
|
-
from ..types import
|
|
12
|
+
from ..types import ContentUnavailableError
|
|
13
13
|
from ..utils.helpers import UA
|
|
14
14
|
|
|
15
15
|
|
|
@@ -79,7 +79,7 @@ class Twitter:
|
|
|
79
79
|
|
|
80
80
|
result = result["data"]["tweetResult"].get("result")
|
|
81
81
|
if not result:
|
|
82
|
-
raise
|
|
82
|
+
raise ContentUnavailableError("error -4: 帖子或用户不存在")
|
|
83
83
|
|
|
84
84
|
if tweet := result.get("tweet"):
|
|
85
85
|
tweet_id = tweet.get("rest_id", {})
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import html
|
|
4
|
+
import re
|
|
5
|
+
import secrets
|
|
6
|
+
import time
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from urllib.parse import parse_qs, urlparse
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
|
|
12
|
+
from ..errors import ContentUnavailableError
|
|
13
|
+
from ..utils.helpers import UA
|
|
14
|
+
|
|
15
|
+
_SHARE_URL = "https://weixin.qq.com/sph/{}"
|
|
16
|
+
_YUANBAO_API = "https://yuanbao.tencent.com/api/weixin/get_parse_result"
|
|
17
|
+
_FEED_API = "https://channels.weixin.qq.com/finder-preview/api/feed/get_feed_info"
|
|
18
|
+
_FEED_PAGE = "https://channels.weixin.qq.com/finder-preview/pages/feed"
|
|
19
|
+
_ID_RE = re.compile(
|
|
20
|
+
r"(?:weixin\.qq\.com/sph/(\w+)|channels\.weixin\.qq\.com/finder-preview/pages/sph\?[^#]*?\bid=(\w+))"
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class WXChannelsVideo:
|
|
26
|
+
title: str
|
|
27
|
+
author: str
|
|
28
|
+
video_url: str
|
|
29
|
+
thumb_url: str | None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class WXChannelsError(Exception):
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class WXChannels:
|
|
37
|
+
def __init__(self, cookie: str, proxy: str | None = None):
|
|
38
|
+
self.cookie = cookie
|
|
39
|
+
self.proxy = proxy
|
|
40
|
+
|
|
41
|
+
async def parse(self, url: str) -> WXChannelsVideo:
|
|
42
|
+
match = _ID_RE.search(url)
|
|
43
|
+
if not match:
|
|
44
|
+
raise WXChannelsError("不支持的微信视频号链接")
|
|
45
|
+
share_id = match.group(1) or match.group(2)
|
|
46
|
+
|
|
47
|
+
async with httpx.AsyncClient(proxy=self.proxy, timeout=30) as client:
|
|
48
|
+
response = await client.post(
|
|
49
|
+
_YUANBAO_API,
|
|
50
|
+
json={"type": "video_channel_url", "url": _SHARE_URL.format(share_id), "scene": 1},
|
|
51
|
+
headers={
|
|
52
|
+
"content-type": "application/json",
|
|
53
|
+
"User-Agent": UA,
|
|
54
|
+
"origin": "https://yuanbao.tencent.com",
|
|
55
|
+
"referer": "https://yuanbao.tencent.com/",
|
|
56
|
+
"cookie": self.cookie,
|
|
57
|
+
},
|
|
58
|
+
)
|
|
59
|
+
if response.status_code == 401:
|
|
60
|
+
raise WXChannelsError("Cookie 无效或已过期")
|
|
61
|
+
response.raise_for_status()
|
|
62
|
+
result = response.json()
|
|
63
|
+
if error := result.get("error"):
|
|
64
|
+
if str(error.get("code")) == "20000":
|
|
65
|
+
raise WXChannelsError("Cookie 无效或已过期")
|
|
66
|
+
raise WXChannelsError(str(error.get("message") or error))
|
|
67
|
+
if result.get("code") != 0:
|
|
68
|
+
raise WXChannelsError(str(result.get("message") or "微信视频号解析失败"))
|
|
69
|
+
|
|
70
|
+
data = result.get("data") or {}
|
|
71
|
+
playable_url = data.get("playable_url") or ""
|
|
72
|
+
if not data.get("wx_export_id") or not playable_url:
|
|
73
|
+
raise ContentUnavailableError("微信视频号视频不存在或已删除")
|
|
74
|
+
|
|
75
|
+
playable_query = parse_qs(urlparse(playable_url).query)
|
|
76
|
+
token = (playable_query.get("token") or [""])[0]
|
|
77
|
+
export_id = (playable_query.get("eid") or [""])[0]
|
|
78
|
+
if not token or not export_id:
|
|
79
|
+
raise WXChannelsError("无法获取视频信息")
|
|
80
|
+
|
|
81
|
+
rid = f"{int(time.time()):x}-{secrets.token_hex(4)}"
|
|
82
|
+
response = await client.post(
|
|
83
|
+
_FEED_API,
|
|
84
|
+
params={"_rid": rid, "_pageUrl": _FEED_PAGE},
|
|
85
|
+
json={"baseReq": {"generalToken": token}, "exportId": export_id},
|
|
86
|
+
headers={
|
|
87
|
+
"content-type": "application/json",
|
|
88
|
+
"User-Agent": UA,
|
|
89
|
+
"origin": "https://channels.weixin.qq.com",
|
|
90
|
+
"referer": playable_url,
|
|
91
|
+
},
|
|
92
|
+
)
|
|
93
|
+
response.raise_for_status()
|
|
94
|
+
feed_result = response.json()
|
|
95
|
+
|
|
96
|
+
feed_data = feed_result.get("data") or {}
|
|
97
|
+
feed_error = feed_data.get("errMsg") or {}
|
|
98
|
+
error_text = " ".join(
|
|
99
|
+
_strip_html(str(feed_error.get(key) or "")) for key in ("title", "content")
|
|
100
|
+
).strip()
|
|
101
|
+
if feed_result.get("errCode") != 0:
|
|
102
|
+
message = feed_result.get("errMsg") or "微信视频号解析失败"
|
|
103
|
+
raise WXChannelsError(str(message))
|
|
104
|
+
if error_text:
|
|
105
|
+
raise WXChannelsError(error_text)
|
|
106
|
+
|
|
107
|
+
feed_info = feed_data.get("feedInfo") or {}
|
|
108
|
+
video_url = (
|
|
109
|
+
((feed_info.get("h264VideoInfo") or {}).get("videoUrl"))
|
|
110
|
+
or feed_info.get("videoUrl")
|
|
111
|
+
or ((feed_info.get("h265VideoInfo") or {}).get("videoUrl"))
|
|
112
|
+
)
|
|
113
|
+
if not video_url:
|
|
114
|
+
raise WXChannelsError("暂不支持该类型")
|
|
115
|
+
|
|
116
|
+
author_info = feed_data.get("authorInfo") or {}
|
|
117
|
+
return WXChannelsVideo(
|
|
118
|
+
title=feed_info.get("description") or data.get("desc") or "",
|
|
119
|
+
author=author_info.get("nickname") or data.get("author") or "",
|
|
120
|
+
video_url=video_url,
|
|
121
|
+
thumb_url=feed_info.get("coverUrl") or data.get("cover_url") or None,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _strip_html(value: str) -> str:
|
|
126
|
+
return html.unescape(re.sub(r"<[^>]*>", "", value)).strip()
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
__all__ = ["WXChannels", "WXChannelsError", "WXChannelsVideo"]
|
|
@@ -10,7 +10,7 @@ from typing import Any, cast
|
|
|
10
10
|
import httpx
|
|
11
11
|
from bs4 import BeautifulSoup
|
|
12
12
|
|
|
13
|
-
from ..errors import ParseError
|
|
13
|
+
from ..errors import ContentUnavailableError, ParseError
|
|
14
14
|
from ..utils.helpers import UA
|
|
15
15
|
|
|
16
16
|
|
|
@@ -33,7 +33,7 @@ class XHSAPI:
|
|
|
33
33
|
if "/login" in str(result.url):
|
|
34
34
|
raise ParseError("该帖子需要登录后查看")
|
|
35
35
|
elif "/404" in str(result.url):
|
|
36
|
-
raise
|
|
36
|
+
raise ContentUnavailableError("帖子不存在")
|
|
37
37
|
return result.text
|
|
38
38
|
|
|
39
39
|
@staticmethod
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from ..errors import DownloadError, ParseError
|
|
1
|
+
from ..errors import ContentUnavailableError, DownloadError, ParseError
|
|
2
2
|
from .callback import ProgressCallback, ProgressUnit
|
|
3
3
|
from .media_file import AniFile, AnyMediaFile, ImageFile, LivePhotoFile, MediaFile, VideoFile
|
|
4
4
|
from .media_ref import AniRef, AnyMediaRef, ImageRef, LivePhotoRef, MediaRef, VideoRef
|
|
@@ -17,6 +17,7 @@ from .result import (
|
|
|
17
17
|
__all__ = [
|
|
18
18
|
"DownloadError",
|
|
19
19
|
"ParseError",
|
|
20
|
+
"ContentUnavailableError",
|
|
20
21
|
"MediaRef",
|
|
21
22
|
"AniRef",
|
|
22
23
|
"VideoRef",
|
|
@@ -21,6 +21,7 @@ class Platform(Enum):
|
|
|
21
21
|
TWITTER = ("twitter", "Twitter")
|
|
22
22
|
WEIBO = ("weibo", "微博")
|
|
23
23
|
WEIXIN = ("weixin", "微信公众号")
|
|
24
|
+
WEIXIN_CHANNELS = ("weixin_channels", "微信视频号")
|
|
24
25
|
XHS = ("xhs", "小红书")
|
|
25
26
|
XIAOHEIHE = ("xiaoheihe", "小黑盒")
|
|
26
27
|
YOUTUBE = ("youtube", "Youtube")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: parsehub
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.2
|
|
4
4
|
Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
|
|
5
5
|
Author-email: 梓澪 <zilingmio@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -88,6 +88,7 @@ Dynamic: license-file
|
|
|
88
88
|
| **小红书** | ✅ | ✅ | |
|
|
89
89
|
| **贴吧** | ✅ | ✅ | |
|
|
90
90
|
| **微信公众号** | | ✅ | |
|
|
91
|
+
| **微信视频号** | ✅ | | |
|
|
91
92
|
| **快手** | ✅ | ✅ | |
|
|
92
93
|
| **酷安** | | ✅ | |
|
|
93
94
|
| **皮皮虾** | ✅ | ✅ | |
|
|
@@ -222,6 +223,7 @@ print(result)
|
|
|
222
223
|
- `小红书`
|
|
223
224
|
- `知乎`
|
|
224
225
|
- `豆瓣`
|
|
226
|
+
- `微信视频号`(需要填腾讯元宝 yuanbao.tencent.com 登录后的 Cookie)
|
|
225
227
|
|
|
226
228
|
```python
|
|
227
229
|
from parsehub import ParseHub
|
|
@@ -37,6 +37,7 @@ src/parsehub/parsers/parser/toutiao.py
|
|
|
37
37
|
src/parsehub/parsers/parser/twitter.py
|
|
38
38
|
src/parsehub/parsers/parser/weibo.py
|
|
39
39
|
src/parsehub/parsers/parser/weixin.py
|
|
40
|
+
src/parsehub/parsers/parser/wxchannels.py
|
|
40
41
|
src/parsehub/parsers/parser/xhs.py
|
|
41
42
|
src/parsehub/parsers/parser/xiaoheihe.py
|
|
42
43
|
src/parsehub/parsers/parser/youtube.py
|
|
@@ -59,6 +60,7 @@ src/parsehub/provider_api/toutiao.py
|
|
|
59
60
|
src/parsehub/provider_api/twitter.py
|
|
60
61
|
src/parsehub/provider_api/weibo.py
|
|
61
62
|
src/parsehub/provider_api/weixin.py
|
|
63
|
+
src/parsehub/provider_api/wxchannels.py
|
|
62
64
|
src/parsehub/provider_api/xhs.py
|
|
63
65
|
src/parsehub/provider_api/xiaoheihe.py
|
|
64
66
|
src/parsehub/provider_api/zhihu.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|