parsehub 2.3.16__tar.gz → 2.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {parsehub-2.3.16/src/parsehub.egg-info → parsehub-2.4.0}/PKG-INFO +2 -2
- {parsehub-2.3.16 → parsehub-2.4.0}/README.md +1 -1
- {parsehub-2.3.16 → parsehub-2.4.0}/pyproject.toml +1 -1
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/base/ytdlp.py +1 -1
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/bilibili.py +66 -3
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/instagram.py +3 -2
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/zuiyou.py +1 -1
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/bilibili.py +26 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/instagram.py +45 -2
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/zuiyou.py +10 -2
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/types/__init__.py +6 -2
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/types/media_file.py +16 -2
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/types/media_ref.py +18 -2
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/types/post.py +1 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/types/result.py +20 -3
- {parsehub-2.3.16 → parsehub-2.4.0/src/parsehub.egg-info}/PKG-INFO +2 -2
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub.egg-info/SOURCES.txt +4 -1
- parsehub-2.4.0/test/test_bilibili_music_offline.py +127 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/test/test_core_offline.py +5 -0
- parsehub-2.4.0/test/test_instagram_duration_offline.py +134 -0
- parsehub-2.4.0/test/test_zuiyou_duration_offline.py +69 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/LICENSE +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/setup.cfg +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/__init__.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/__init__.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/cli.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/cli_config.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/config/__init__.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/config/config.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/errors.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/__init__.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/base/__init__.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/base/base.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/__init__.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/bangumi.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/coolapk.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/douban.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/douyin.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/facebook.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/kuaishou.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/medium.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/pipix.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/snapchat.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/threads.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/tieba.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/tiktok.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/toutiao.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/twitter.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/weibo.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/weixin.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/wxchannels.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/xhs.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/xiaoheihe.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/youtube.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/parsers/parser/zhihu.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/__init__.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/bangumi.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/coolapk.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/douban.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/douyin.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/kuaishou.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/medium.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/pipix.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/threads.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/tieba.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/tiktok.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/toutiao.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/twitter.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/weibo.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/weixin.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/wxchannels.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/xhs.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/xiaoheihe.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/provider_api/zhihu.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/types/callback.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/types/platform.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/utils/downloader.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/utils/fallback.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/utils/helpers.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub/utils/media_info.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub.egg-info/dependency_links.txt +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub.egg-info/entry_points.txt +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub.egg-info/requires.txt +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/src/parsehub.egg-info/top_level.txt +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/test/test_bilibili_offline.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/test/test_cli.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/test/test_cli_config.py +0 -0
- {parsehub-2.3.16 → parsehub-2.4.0}/test/test_downloader.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: parsehub
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.4.0
|
|
4
4
|
Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
|
|
5
5
|
Author-email: 梓澪 <zilingmio@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -82,7 +82,7 @@ Dynamic: license-file
|
|
|
82
82
|
| **YouTube** | ✅ | | 🎵 音乐 |
|
|
83
83
|
| **Facebook** | ✅ | | |
|
|
84
84
|
| **Threads** | ✅ | ✅ | |
|
|
85
|
-
| **Bilibili** | ✅ | | 📝
|
|
85
|
+
| **Bilibili** | ✅ | | 📝 动态, 🎵 音乐 |
|
|
86
86
|
| **抖音** | ✅ | ✅ | ☀️日常 |
|
|
87
87
|
| **TikTok** | ✅ | ✅ | |
|
|
88
88
|
| **微博** | ✅ | ✅ | |
|
|
@@ -411,7 +411,7 @@ class YtParser(BaseParser, register=False):
|
|
|
411
411
|
title = dl["title"]
|
|
412
412
|
duration = dl.get("duration", 0)
|
|
413
413
|
thumbnail = dl.get("thumbnail")
|
|
414
|
-
description = dl
|
|
414
|
+
description = dl.get("description", "")
|
|
415
415
|
width = dl.get("width", 0)
|
|
416
416
|
height = dl.get("height", 0)
|
|
417
417
|
return YtVideoInfo(
|
|
@@ -12,6 +12,8 @@ from ...types import (
|
|
|
12
12
|
ImageParseResult,
|
|
13
13
|
ImageRef,
|
|
14
14
|
LivePhotoRef,
|
|
15
|
+
MusicParseResult,
|
|
16
|
+
MusicRef,
|
|
15
17
|
ParseError,
|
|
16
18
|
Platform,
|
|
17
19
|
ProgressCallback,
|
|
@@ -25,17 +27,21 @@ from ..base.ytdlp import YtParser, YtVideoParseResult
|
|
|
25
27
|
|
|
26
28
|
class BiliParse(BaseParser):
|
|
27
29
|
__platform__ = Platform.BILIBILI
|
|
28
|
-
__supported_type__ = ["视频", "动态"]
|
|
29
|
-
__match__ = r"^(http(s)?://)?((((w){3}.|(m).|(t).)?bilibili\.com)/(video|opus
|
|
30
|
+
__supported_type__ = ["视频", "动态", "音乐"]
|
|
31
|
+
__match__ = r"^(http(s)?://)?((((w){3}.|(m).|(t).)?bilibili\.com)/(video|opus|audio/au\d+|\b\d{18,19}\b)|b23.tv|bili2233.cn).*"
|
|
30
32
|
__reserved_parameters__ = ["p"]
|
|
31
33
|
__redirect_keywords__ = ["b23.tv", "bili2233.cn"]
|
|
32
34
|
|
|
33
|
-
async def _do_parse(
|
|
35
|
+
async def _do_parse(
|
|
36
|
+
self, raw_url: str
|
|
37
|
+
) -> YtVideoParseResult | BiliVideoParseResult | BiliMusicParseResult | ImageParseResult:
|
|
34
38
|
parsed = urlparse(raw_url)
|
|
35
39
|
if parsed.hostname == "mall.bilibili.com":
|
|
36
40
|
raise ParseError("不支持解析商品")
|
|
37
41
|
if parsed.path.startswith("/bangumi/"):
|
|
38
42
|
raise ParseError("不支持解析番剧和影视")
|
|
43
|
+
if BiliAPI.get_music_id(raw_url):
|
|
44
|
+
return await self.bili_music_parse(raw_url)
|
|
39
45
|
if await self.is_dynamic(raw_url):
|
|
40
46
|
dynamic = await self.get_dynamic_info(raw_url)
|
|
41
47
|
content = self.hashtag_handler(dynamic.content or "")
|
|
@@ -142,6 +148,36 @@ class BiliParse(BaseParser):
|
|
|
142
148
|
),
|
|
143
149
|
)
|
|
144
150
|
|
|
151
|
+
async def bili_music_parse(self, url: str) -> BiliMusicParseResult:
|
|
152
|
+
async with BiliAPI(proxy=self.proxy) as bili:
|
|
153
|
+
info_response = await bili.get_music_info(url)
|
|
154
|
+
if not (info := info_response.get("data")):
|
|
155
|
+
if info_response.get("code") == 4511001:
|
|
156
|
+
raise ContentUnavailableError("音乐未找到或已下架")
|
|
157
|
+
raise ParseError(f"获取音乐信息失败: {info_response.get('message') or '未知错误'}")
|
|
158
|
+
|
|
159
|
+
play_response = await bili.get_music_url(str(info["id"]))
|
|
160
|
+
cdns: list[str] = (play_response.get("data") or {}).get("cdns") or []
|
|
161
|
+
if not cdns:
|
|
162
|
+
raise ParseError("获取音乐链接失败")
|
|
163
|
+
|
|
164
|
+
return BiliMusicParseResult(
|
|
165
|
+
title=info.get("title") or "",
|
|
166
|
+
music=MusicRef(
|
|
167
|
+
url=cdns[0],
|
|
168
|
+
ext=self._music_ext(cdns[0]),
|
|
169
|
+
thumb_url=info.get("cover") or None,
|
|
170
|
+
backup_urls=cdns[1:],
|
|
171
|
+
duration=int(info.get("duration") or 0),
|
|
172
|
+
),
|
|
173
|
+
content=info.get("intro") or "",
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
@staticmethod
|
|
177
|
+
def _music_ext(url: str) -> str:
|
|
178
|
+
suffix = Path(urlparse(url).path).suffix
|
|
179
|
+
return suffix.removeprefix(".") or "m4a"
|
|
180
|
+
|
|
145
181
|
async def ytp_parse(self, url: str) -> YtVideoParseResult:
|
|
146
182
|
return await BiliYtParse(proxy=self.proxy, cookie=self.cookie)._do_parse(url)
|
|
147
183
|
|
|
@@ -205,7 +241,34 @@ class BiliVideoParseResult(VideoParseResult):
|
|
|
205
241
|
)
|
|
206
242
|
|
|
207
243
|
|
|
244
|
+
class BiliMusicParseResult(MusicParseResult):
|
|
245
|
+
async def _do_download(
|
|
246
|
+
self,
|
|
247
|
+
*,
|
|
248
|
+
output_dir: Path,
|
|
249
|
+
callback: ProgressCallback | None = None,
|
|
250
|
+
callback_args: tuple = (),
|
|
251
|
+
callback_kwargs: dict | None = None,
|
|
252
|
+
proxy: str | None = None,
|
|
253
|
+
headers: dict | None = None,
|
|
254
|
+
connections: int = 4,
|
|
255
|
+
concurrency: int = 4,
|
|
256
|
+
) -> DownloadResult:
|
|
257
|
+
headers = {"referer": "https://www.bilibili.com", "User-Agent": UA}
|
|
258
|
+
return await super()._do_download(
|
|
259
|
+
output_dir=output_dir,
|
|
260
|
+
callback=callback,
|
|
261
|
+
callback_args=callback_args,
|
|
262
|
+
callback_kwargs=callback_kwargs,
|
|
263
|
+
proxy=proxy,
|
|
264
|
+
headers=headers,
|
|
265
|
+
connections=connections,
|
|
266
|
+
concurrency=concurrency,
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
|
|
208
270
|
__all__ = [
|
|
209
271
|
"BiliParse",
|
|
210
272
|
"BiliVideoParseResult",
|
|
273
|
+
"BiliMusicParseResult",
|
|
211
274
|
]
|
|
@@ -71,11 +71,12 @@ class InstagramParser(BaseParser):
|
|
|
71
71
|
title=post.title,
|
|
72
72
|
content=post.caption,
|
|
73
73
|
video=VideoRef(
|
|
74
|
-
|
|
74
|
+
# yt-dlp 的 info.url 是帖子页,provider 的 video_url 才是媒体直链
|
|
75
|
+
url=item.video_url or info.url,
|
|
75
76
|
thumb_url=item.display_url or info.thumbnail,
|
|
76
77
|
width=info.width,
|
|
77
78
|
height=info.height,
|
|
78
|
-
duration=int(
|
|
79
|
+
duration=int(item.duration or info.duration),
|
|
79
80
|
),
|
|
80
81
|
)
|
|
81
82
|
if not item.video_url:
|
|
@@ -15,7 +15,7 @@ class ZuiYouParser(BaseParser):
|
|
|
15
15
|
return MultimediaParseResult(
|
|
16
16
|
content=zy.content,
|
|
17
17
|
media=[
|
|
18
|
-
VideoRef(url=i.url, thumb_url=i.thumb_url)
|
|
18
|
+
VideoRef(url=i.url, thumb_url=i.thumb_url, duration=i.duration)
|
|
19
19
|
if i.type == MediaType.VIDEO
|
|
20
20
|
else ImageRef(url=i.url, thumb_url=i.thumb_url)
|
|
21
21
|
for i in zy.media
|
|
@@ -125,6 +125,27 @@ class BiliAPI:
|
|
|
125
125
|
)
|
|
126
126
|
return cast(dict[str, Any], response.json())
|
|
127
127
|
|
|
128
|
+
async def get_music_info(self, url: str) -> dict:
|
|
129
|
+
"""获取音乐详细信息"""
|
|
130
|
+
sid = self.get_music_id(url)
|
|
131
|
+
if not sid:
|
|
132
|
+
raise ValueError(f"Invalid url: {url}")
|
|
133
|
+
response = await self._get_client().get(
|
|
134
|
+
"https://www.bilibili.com/audio/music-service-c/web/song/info",
|
|
135
|
+
params={"sid": sid},
|
|
136
|
+
)
|
|
137
|
+
response.raise_for_status()
|
|
138
|
+
return cast(dict[str, Any], response.json())
|
|
139
|
+
|
|
140
|
+
async def get_music_url(self, sid: str) -> dict:
|
|
141
|
+
"""获取音乐播放链接"""
|
|
142
|
+
response = await self._get_client().get(
|
|
143
|
+
"https://www.bilibili.com/audio/music-service-c/web/url",
|
|
144
|
+
params={"sid": sid},
|
|
145
|
+
)
|
|
146
|
+
response.raise_for_status()
|
|
147
|
+
return cast(dict[str, Any], response.json())
|
|
148
|
+
|
|
128
149
|
async def get_buvid(self):
|
|
129
150
|
"""获取 buvid"""
|
|
130
151
|
response = await self._get_client().get(
|
|
@@ -188,6 +209,11 @@ class BiliAPI:
|
|
|
188
209
|
tmp = tmp * BASE + idx
|
|
189
210
|
return f"av{(tmp & MASK_CODE) ^ XOR_CODE}"
|
|
190
211
|
|
|
212
|
+
@staticmethod
|
|
213
|
+
def get_music_id(url: str) -> str | None:
|
|
214
|
+
m = re.search(r"/audio/au(\d+)", url)
|
|
215
|
+
return m.group(1) if m else None
|
|
216
|
+
|
|
191
217
|
def get_bvid(self, url: str):
|
|
192
218
|
m_bv = re.search(r"BV[0-9A-Za-z]{10,}", url)
|
|
193
219
|
if m_bv:
|
|
@@ -116,11 +116,54 @@ class InstagramAPI:
|
|
|
116
116
|
cookies=self.cookie or None,
|
|
117
117
|
) as session:
|
|
118
118
|
if self.cookie:
|
|
119
|
-
|
|
120
|
-
|
|
119
|
+
post = await self._get_cookie_post(session, shortcode)
|
|
120
|
+
else:
|
|
121
|
+
post = await self._get_logged_out_post(session, shortcode)
|
|
122
|
+
return await self._fill_missing_video_fields(session, post, shortcode)
|
|
121
123
|
except requests.RequestsError as exc:
|
|
122
124
|
raise InstagramAPIError(f"请求 Instagram 失败: {exc}") from exc
|
|
123
125
|
|
|
126
|
+
async def _fill_missing_video_fields(
|
|
127
|
+
self,
|
|
128
|
+
session: requests.AsyncSession,
|
|
129
|
+
post: InstagramPost,
|
|
130
|
+
shortcode: str,
|
|
131
|
+
) -> InstagramPost:
|
|
132
|
+
"""用 embed 补齐 v1 / GraphQL 缺失的视频直链和时长。"""
|
|
133
|
+
if not any(
|
|
134
|
+
(post.type is InstagramMediaType.VIDEO or item.is_video)
|
|
135
|
+
and (not item.video_url or item.duration <= 0)
|
|
136
|
+
for item in post.media
|
|
137
|
+
):
|
|
138
|
+
return post
|
|
139
|
+
|
|
140
|
+
try:
|
|
141
|
+
node = await self._fetch_embed(session, shortcode)
|
|
142
|
+
except requests.RequestsError:
|
|
143
|
+
return post
|
|
144
|
+
if node is None:
|
|
145
|
+
return post
|
|
146
|
+
|
|
147
|
+
try:
|
|
148
|
+
embed = InstagramPost.from_graph(node)
|
|
149
|
+
except InstagramAPIError:
|
|
150
|
+
return post
|
|
151
|
+
|
|
152
|
+
for index, item in enumerate(post.media):
|
|
153
|
+
if post.type is not InstagramMediaType.VIDEO and not item.is_video:
|
|
154
|
+
continue
|
|
155
|
+
source = embed.media[index] if index < len(embed.media) else None
|
|
156
|
+
if source is None and len(post.media) == 1 and len(embed.media) == 1:
|
|
157
|
+
source = embed.media[0]
|
|
158
|
+
if source is None:
|
|
159
|
+
continue
|
|
160
|
+
if not item.video_url and source.video_url:
|
|
161
|
+
item.video_url = source.video_url
|
|
162
|
+
item.is_video = True
|
|
163
|
+
if item.duration <= 0 and source.duration > 0:
|
|
164
|
+
item.duration = source.duration
|
|
165
|
+
return post
|
|
166
|
+
|
|
124
167
|
async def _get_logged_out_post(self, session: requests.AsyncSession, shortcode: str) -> InstagramPost:
|
|
125
168
|
detail = "未返回帖子数据"
|
|
126
169
|
media_id = str(self.shortcode_to_media_id(shortcode))
|
|
@@ -15,6 +15,7 @@ class Media:
|
|
|
15
15
|
url: str
|
|
16
16
|
thumb_url: str
|
|
17
17
|
type: MediaType
|
|
18
|
+
duration: int = 0
|
|
18
19
|
|
|
19
20
|
|
|
20
21
|
@dataclass
|
|
@@ -34,8 +35,15 @@ class ZuiYouPost:
|
|
|
34
35
|
id_ = img["id"]
|
|
35
36
|
img_url = list(img["urls"].values())[-1]["urls"][-1]
|
|
36
37
|
if img.get("video"):
|
|
37
|
-
|
|
38
|
-
media.append(
|
|
38
|
+
video = videos[str(id_)]
|
|
39
|
+
media.append(
|
|
40
|
+
Media(
|
|
41
|
+
url=video["url"],
|
|
42
|
+
thumb_url=img_url,
|
|
43
|
+
type=MediaType.VIDEO,
|
|
44
|
+
duration=int(video.get("dur") or 0),
|
|
45
|
+
)
|
|
46
|
+
)
|
|
39
47
|
continue
|
|
40
48
|
media.append(Media(url=img_url, thumb_url=img_url, type=MediaType.PHOTO))
|
|
41
49
|
|
|
@@ -8,8 +8,8 @@ from ..errors import (
|
|
|
8
8
|
ParseError,
|
|
9
9
|
)
|
|
10
10
|
from .callback import ProgressCallback, ProgressUnit
|
|
11
|
-
from .media_file import AniFile, AnyMediaFile, ImageFile, LivePhotoFile, MediaFile, VideoFile
|
|
12
|
-
from .media_ref import AniRef, AnyMediaRef, ImageRef, LivePhotoRef, MediaRef, VideoRef
|
|
11
|
+
from .media_file import AniFile, AnyMediaFile, ImageFile, LivePhotoFile, MediaFile, MusicFile, VideoFile
|
|
12
|
+
from .media_ref import AniRef, AnyMediaRef, ImageRef, LivePhotoRef, MediaRef, MusicRef, VideoRef
|
|
13
13
|
from .platform import Platform
|
|
14
14
|
from .post import PostType
|
|
15
15
|
from .result import (
|
|
@@ -17,6 +17,7 @@ from .result import (
|
|
|
17
17
|
DownloadResult,
|
|
18
18
|
ImageParseResult,
|
|
19
19
|
MultimediaParseResult,
|
|
20
|
+
MusicParseResult,
|
|
20
21
|
ParseResult,
|
|
21
22
|
RichTextParseResult,
|
|
22
23
|
VideoParseResult,
|
|
@@ -33,12 +34,14 @@ __all__ = [
|
|
|
33
34
|
"MediaRef",
|
|
34
35
|
"AniRef",
|
|
35
36
|
"VideoRef",
|
|
37
|
+
"MusicRef",
|
|
36
38
|
"ImageRef",
|
|
37
39
|
"LivePhotoRef",
|
|
38
40
|
"DownloadResult",
|
|
39
41
|
"ParseResult",
|
|
40
42
|
"ImageParseResult",
|
|
41
43
|
"VideoParseResult",
|
|
44
|
+
"MusicParseResult",
|
|
42
45
|
"MultimediaParseResult",
|
|
43
46
|
"RichTextParseResult",
|
|
44
47
|
"AnyMediaRef",
|
|
@@ -49,6 +52,7 @@ __all__ = [
|
|
|
49
52
|
"ImageFile",
|
|
50
53
|
"AniFile",
|
|
51
54
|
"LivePhotoFile",
|
|
55
|
+
"MusicFile",
|
|
52
56
|
"AnyMediaFile",
|
|
53
57
|
"ProgressCallback",
|
|
54
58
|
"ProgressUnit",
|
|
@@ -78,6 +78,20 @@ class AniFile(MediaFile):
|
|
|
78
78
|
self.duration = info.duration
|
|
79
79
|
|
|
80
80
|
|
|
81
|
+
@dataclass(kw_only=True)
|
|
82
|
+
class MusicFile(MediaFile):
|
|
83
|
+
"""本地音乐
|
|
84
|
+
Attributes:
|
|
85
|
+
path: 路径
|
|
86
|
+
duration: 音乐时长,单位: 秒
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
duration: int = 0
|
|
90
|
+
|
|
91
|
+
def __post_init__(self) -> None:
|
|
92
|
+
"""音乐不做宽高探测;时长由解析结果传入。"""
|
|
93
|
+
|
|
94
|
+
|
|
81
95
|
@dataclass(kw_only=True)
|
|
82
96
|
class LivePhotoFile(MediaFile):
|
|
83
97
|
"""本地 Live Photo
|
|
@@ -100,6 +114,6 @@ class LivePhotoFile(MediaFile):
|
|
|
100
114
|
self.duration = info.duration
|
|
101
115
|
|
|
102
116
|
|
|
103
|
-
AnyMediaFile = MediaFile | VideoFile | ImageFile | AniFile | LivePhotoFile
|
|
117
|
+
AnyMediaFile = MediaFile | VideoFile | ImageFile | AniFile | LivePhotoFile | MusicFile
|
|
104
118
|
|
|
105
|
-
__all__ = ["MediaFile", "VideoFile", "ImageFile", "AniFile", "LivePhotoFile", "AnyMediaFile"]
|
|
119
|
+
__all__ = ["MediaFile", "VideoFile", "ImageFile", "AniFile", "LivePhotoFile", "MusicFile", "AnyMediaFile"]
|
|
@@ -37,6 +37,22 @@ class VideoRef(MediaRef):
|
|
|
37
37
|
duration: int = 0
|
|
38
38
|
|
|
39
39
|
|
|
40
|
+
@dataclass(kw_only=True)
|
|
41
|
+
class MusicRef(MediaRef):
|
|
42
|
+
"""音乐
|
|
43
|
+
Attributes:
|
|
44
|
+
url: URL
|
|
45
|
+
ext: 默认扩展名
|
|
46
|
+
thumb_url: 缩略图 URL
|
|
47
|
+
width: 宽度
|
|
48
|
+
height: 高度
|
|
49
|
+
duration: 音乐时长,单位: 秒
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
ext: str = "m4a"
|
|
53
|
+
duration: int = 0
|
|
54
|
+
|
|
55
|
+
|
|
40
56
|
@dataclass(kw_only=True)
|
|
41
57
|
class ImageRef(MediaRef):
|
|
42
58
|
"""图片
|
|
@@ -87,6 +103,6 @@ class LivePhotoRef(MediaRef):
|
|
|
87
103
|
duration: int = 3
|
|
88
104
|
|
|
89
105
|
|
|
90
|
-
AnyMediaRef = ImageRef | VideoRef | AniRef | LivePhotoRef
|
|
106
|
+
AnyMediaRef = ImageRef | VideoRef | AniRef | LivePhotoRef | MusicRef
|
|
91
107
|
|
|
92
|
-
__all__ = ["MediaRef", "VideoRef", "ImageRef", "AniRef", "LivePhotoRef", "AnyMediaRef"]
|
|
108
|
+
__all__ = ["MediaRef", "VideoRef", "MusicRef", "ImageRef", "AniRef", "LivePhotoRef", "AnyMediaRef"]
|
|
@@ -20,8 +20,8 @@ from ..utils.downloader import download
|
|
|
20
20
|
from ..utils.fallback import FallbackStage, run_fallback
|
|
21
21
|
from ..utils.helpers import run_sync
|
|
22
22
|
from .callback import ProgressCallback
|
|
23
|
-
from .media_file import AniFile, AnyMediaFile, ImageFile, LivePhotoFile, VideoFile
|
|
24
|
-
from .media_ref import AniRef, AnyMediaRef, ImageRef, LivePhotoRef, VideoRef
|
|
23
|
+
from .media_file import AniFile, AnyMediaFile, ImageFile, LivePhotoFile, MusicFile, VideoFile
|
|
24
|
+
from .media_ref import AniRef, AnyMediaRef, ImageRef, LivePhotoRef, MusicRef, VideoRef
|
|
25
25
|
from .platform import Platform
|
|
26
26
|
from .post import PostType
|
|
27
27
|
|
|
@@ -155,6 +155,8 @@ class ParseResult(ABC): # noqa: B024
|
|
|
155
155
|
mf = ImageFile(path=f, width=media.width, height=media.height)
|
|
156
156
|
case VideoRef():
|
|
157
157
|
mf = VideoFile(path=f, width=media.width, height=media.height, duration=media.duration)
|
|
158
|
+
case MusicRef():
|
|
159
|
+
mf = MusicFile(path=f, duration=media.duration)
|
|
158
160
|
case AniRef():
|
|
159
161
|
mf = AniFile(path=f, width=media.width, height=media.height, duration=media.duration)
|
|
160
162
|
case LivePhotoRef():
|
|
@@ -361,6 +363,21 @@ class VideoParseResult(ParseResult):
|
|
|
361
363
|
)
|
|
362
364
|
|
|
363
365
|
|
|
366
|
+
class MusicParseResult(ParseResult):
|
|
367
|
+
"""单个音乐"""
|
|
368
|
+
|
|
369
|
+
type = PostType.MUSIC
|
|
370
|
+
|
|
371
|
+
def __init__(
|
|
372
|
+
self,
|
|
373
|
+
title: str = "",
|
|
374
|
+
music: str | MusicRef | None = None,
|
|
375
|
+
content: str = "",
|
|
376
|
+
):
|
|
377
|
+
music = MusicRef(url=music) if isinstance(music, str) else music
|
|
378
|
+
super().__init__(title=title, media=music, content=content)
|
|
379
|
+
|
|
380
|
+
|
|
364
381
|
class ImageParseResult(ParseResult):
|
|
365
382
|
"""单图 / 多图 / 图集 / 实况照片"""
|
|
366
383
|
|
|
@@ -464,4 +481,4 @@ class DownloadResult:
|
|
|
464
481
|
return f"{self.__class__.__name__}(media={media_count}, output_dir={self.output_dir})"
|
|
465
482
|
|
|
466
483
|
|
|
467
|
-
AnyParseResult = VideoParseResult | ImageParseResult | MultimediaParseResult | RichTextParseResult
|
|
484
|
+
AnyParseResult = VideoParseResult | ImageParseResult | MultimediaParseResult | RichTextParseResult | MusicParseResult
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: parsehub
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.4.0
|
|
4
4
|
Summary: 轻量、异步、开箱即用的社交媒体聚合解析库
|
|
5
5
|
Author-email: 梓澪 <zilingmio@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -82,7 +82,7 @@ Dynamic: license-file
|
|
|
82
82
|
| **YouTube** | ✅ | | 🎵 音乐 |
|
|
83
83
|
| **Facebook** | ✅ | | |
|
|
84
84
|
| **Threads** | ✅ | ✅ | |
|
|
85
|
-
| **Bilibili** | ✅ | | 📝
|
|
85
|
+
| **Bilibili** | ✅ | | 📝 动态, 🎵 音乐 |
|
|
86
86
|
| **抖音** | ✅ | ✅ | ☀️日常 |
|
|
87
87
|
| **TikTok** | ✅ | ✅ | |
|
|
88
88
|
| **微博** | ✅ | ✅ | |
|
|
@@ -76,8 +76,11 @@ src/parsehub/utils/downloader.py
|
|
|
76
76
|
src/parsehub/utils/fallback.py
|
|
77
77
|
src/parsehub/utils/helpers.py
|
|
78
78
|
src/parsehub/utils/media_info.py
|
|
79
|
+
test/test_bilibili_music_offline.py
|
|
79
80
|
test/test_bilibili_offline.py
|
|
80
81
|
test/test_cli.py
|
|
81
82
|
test/test_cli_config.py
|
|
82
83
|
test/test_core_offline.py
|
|
83
|
-
test/test_downloader.py
|
|
84
|
+
test/test_downloader.py
|
|
85
|
+
test/test_instagram_duration_offline.py
|
|
86
|
+
test/test_zuiyou_duration_offline.py
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from unittest.mock import AsyncMock, patch
|
|
4
|
+
|
|
5
|
+
from parsehub.errors import ContentUnavailableError, ParseError
|
|
6
|
+
from parsehub.parsers.parser.bilibili import BiliMusicParseResult, BiliParse
|
|
7
|
+
from parsehub.provider_api.bilibili import BiliAPI
|
|
8
|
+
from parsehub.types import MusicFile, MusicParseResult, MusicRef, ParseResult
|
|
9
|
+
|
|
10
|
+
MUSIC_INFO = {
|
|
11
|
+
"code": 0,
|
|
12
|
+
"data": {
|
|
13
|
+
"id": 2854883,
|
|
14
|
+
"title": "残 酷 月 光",
|
|
15
|
+
"cover": "http://i0.hdslb.com/bfs/music/bc477834dc9b74abdf9869989fc7a0937b25e020.jpg",
|
|
16
|
+
"intro": "突然想唱 就录了一下",
|
|
17
|
+
"duration": 200,
|
|
18
|
+
},
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
MUSIC_PLAY = {
|
|
22
|
+
"code": 0,
|
|
23
|
+
"data": {"cdns": ["https://cdn.example/music-192k.m4a"], "size": 150},
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class FakeBiliMusicAPI:
|
|
28
|
+
"""响应取自线上实测: song/info 正常响应、4511001 与 -400 错误响应、单条 cdn"""
|
|
29
|
+
|
|
30
|
+
info_response: dict = MUSIC_INFO
|
|
31
|
+
play_response: dict = MUSIC_PLAY
|
|
32
|
+
last_sid: str | None = None
|
|
33
|
+
|
|
34
|
+
get_music_id = staticmethod(BiliAPI.get_music_id) # _do_parse 经 BiliAPI 路由, 替换后需保留
|
|
35
|
+
|
|
36
|
+
def __init__(self, proxy=None):
|
|
37
|
+
self.proxy = proxy
|
|
38
|
+
|
|
39
|
+
async def __aenter__(self):
|
|
40
|
+
return self
|
|
41
|
+
|
|
42
|
+
async def __aexit__(self, *args):
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
async def get_music_info(self, url):
|
|
46
|
+
return FakeBiliMusicAPI.info_response
|
|
47
|
+
|
|
48
|
+
async def get_music_url(self, sid):
|
|
49
|
+
FakeBiliMusicAPI.last_sid = sid
|
|
50
|
+
return FakeBiliMusicAPI.play_response
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class TestBiliMusicParse(unittest.IsolatedAsyncioTestCase):
|
|
54
|
+
async def test_mobile_music_url_parses_to_music_result(self):
|
|
55
|
+
with patch("parsehub.parsers.parser.bilibili.BiliAPI", FakeBiliMusicAPI):
|
|
56
|
+
result = await BiliParse().parse("https://m.bilibili.com/audio/au2854883?share_source=copy")
|
|
57
|
+
|
|
58
|
+
self.assertIsInstance(result, BiliMusicParseResult)
|
|
59
|
+
self.assertIsInstance(result, MusicParseResult)
|
|
60
|
+
self.assertEqual(result.to_dict()["type"], "music")
|
|
61
|
+
self.assertEqual(result.title, "残 酷 月 光")
|
|
62
|
+
self.assertEqual(result.content, "突然想唱 就录了一下")
|
|
63
|
+
self.assertEqual(
|
|
64
|
+
result.media,
|
|
65
|
+
MusicRef(
|
|
66
|
+
url="https://cdn.example/music-192k.m4a",
|
|
67
|
+
thumb_url="http://i0.hdslb.com/bfs/music/bc477834dc9b74abdf9869989fc7a0937b25e020.jpg",
|
|
68
|
+
duration=200,
|
|
69
|
+
),
|
|
70
|
+
)
|
|
71
|
+
self.assertEqual(result.raw_url, "https://m.bilibili.com/audio/au2854883")
|
|
72
|
+
self.assertEqual(FakeBiliMusicAPI.last_sid, "2854883")
|
|
73
|
+
|
|
74
|
+
async def test_missing_music_is_content_unavailable(self):
|
|
75
|
+
with (
|
|
76
|
+
patch("parsehub.parsers.parser.bilibili.BiliAPI", FakeBiliMusicAPI),
|
|
77
|
+
patch.object(
|
|
78
|
+
FakeBiliMusicAPI,
|
|
79
|
+
"info_response",
|
|
80
|
+
{"code": 4511001, "data": None, "message": "音乐未找到或已下架"},
|
|
81
|
+
),
|
|
82
|
+
):
|
|
83
|
+
with self.assertRaises(ContentUnavailableError):
|
|
84
|
+
await BiliParse()._do_parse("https://www.bilibili.com/audio/au2854883")
|
|
85
|
+
|
|
86
|
+
async def test_api_error_keeps_server_message(self):
|
|
87
|
+
with (
|
|
88
|
+
patch("parsehub.parsers.parser.bilibili.BiliAPI", FakeBiliMusicAPI),
|
|
89
|
+
patch.object(FakeBiliMusicAPI, "info_response", {"code": -400, "message": "请求错误"}),
|
|
90
|
+
):
|
|
91
|
+
with self.assertRaisesRegex(ParseError, "获取音乐信息失败: 请求错误"):
|
|
92
|
+
await BiliParse()._do_parse("https://www.bilibili.com/audio/au2854883")
|
|
93
|
+
|
|
94
|
+
async def test_missing_playurl_is_parse_error(self):
|
|
95
|
+
with (
|
|
96
|
+
patch("parsehub.parsers.parser.bilibili.BiliAPI", FakeBiliMusicAPI),
|
|
97
|
+
patch.object(
|
|
98
|
+
FakeBiliMusicAPI, "play_response", {"code": 4511001, "data": None, "message": "音乐未找到或已下架"}
|
|
99
|
+
),
|
|
100
|
+
):
|
|
101
|
+
with self.assertRaisesRegex(ParseError, "获取音乐链接失败"):
|
|
102
|
+
await BiliParse()._do_parse("https://www.bilibili.com/audio/au2854883")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class TestMusicResultDownload(unittest.IsolatedAsyncioTestCase):
|
|
106
|
+
async def test_music_result_downloads_with_m4a_and_keeps_duration(self):
|
|
107
|
+
result = MusicParseResult(title="clip", music=MusicRef(url="https://cdn.example/a", duration=12))
|
|
108
|
+
|
|
109
|
+
with patch("parsehub.types.result.download", new=AsyncMock(return_value="/tmp/clip.m4a")) as mocked:
|
|
110
|
+
out = await result._do_download(output_dir=Path("/tmp/does-not-matter"))
|
|
111
|
+
|
|
112
|
+
self.assertIsInstance(out.media, MusicFile)
|
|
113
|
+
self.assertEqual(out.media.duration, 12)
|
|
114
|
+
self.assertTrue(Path(mocked.await_args.args[1]).name.endswith("clip.m4a"))
|
|
115
|
+
|
|
116
|
+
async def test_download_injects_bilibili_referer(self):
|
|
117
|
+
result = BiliMusicParseResult(title="clip", music=MusicRef(url="https://cdn.example/a"))
|
|
118
|
+
|
|
119
|
+
with patch.object(ParseResult, "_do_download", new=AsyncMock()) as mocked:
|
|
120
|
+
await result._do_download(output_dir=Path("/tmp/x"))
|
|
121
|
+
|
|
122
|
+
# 音乐 CDN 无 Referer 返回 403
|
|
123
|
+
self.assertEqual(mocked.await_args.kwargs["headers"]["referer"], "https://www.bilibili.com")
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
if __name__ == "__main__":
|
|
127
|
+
unittest.main()
|
|
@@ -131,6 +131,7 @@ class TestParserRegistry(unittest.TestCase):
|
|
|
131
131
|
self.assertEqual(len(by_id), len(platforms))
|
|
132
132
|
self.assertEqual(by_id["tieba"]["name"], Platform.TIEBA.display_name)
|
|
133
133
|
self.assertIn("图文", by_id["tieba"]["supported_types"])
|
|
134
|
+
self.assertIn("音乐", by_id["bilibili"]["supported_types"])
|
|
134
135
|
self.assertEqual(parsehub.get_platform("https://tieba.baidu.com/p/9939510114"), Platform.TIEBA)
|
|
135
136
|
self.assertIsNone(parsehub.get_platform("https://example.invalid/not-supported"))
|
|
136
137
|
|
|
@@ -554,6 +555,8 @@ class TestPlatformUrlMatching(unittest.TestCase):
|
|
|
554
555
|
"https://www.bilibili.com/video/av123456",
|
|
555
556
|
"https://www.bilibili.com/opus/1234567890123456789",
|
|
556
557
|
"https://t.bilibili.com/1234567890123456789",
|
|
558
|
+
"https://www.bilibili.com/audio/au2854883",
|
|
559
|
+
"https://m.bilibili.com/audio/au2854883",
|
|
557
560
|
"https://b23.tv/abc123",
|
|
558
561
|
"https://bili2233.cn/abc123",
|
|
559
562
|
],
|
|
@@ -687,6 +690,8 @@ class TestPlatformUrlMatching(unittest.TestCase):
|
|
|
687
690
|
"https://www.youtube.com/live/1h_uc3K4Cpg",
|
|
688
691
|
"https://www.youtube.com/post/Ugkxexample",
|
|
689
692
|
"https://www.youtube.com/@example",
|
|
693
|
+
"https://www.bilibili.com/audio/home",
|
|
694
|
+
"https://www.bilibili.com/audio/am10624",
|
|
690
695
|
]
|
|
691
696
|
|
|
692
697
|
for url in urls:
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
from unittest.mock import AsyncMock, patch
|
|
3
|
+
|
|
4
|
+
from parsehub.parsers.base.ytdlp import YtParser, YtVideoInfo
|
|
5
|
+
from parsehub.parsers.parser.instagram import InstagramParser
|
|
6
|
+
from parsehub.provider_api.instagram import InstagramAPI, InstagramMedia, InstagramMediaType, InstagramPost
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class TestInstagramParserDuration(unittest.IsolatedAsyncioTestCase):
|
|
10
|
+
async def test_video_prefers_provider_url_and_duration(self):
|
|
11
|
+
post = InstagramPost(
|
|
12
|
+
shortcode="abc",
|
|
13
|
+
type=InstagramMediaType.VIDEO,
|
|
14
|
+
media=[
|
|
15
|
+
InstagramMedia(
|
|
16
|
+
is_video=True,
|
|
17
|
+
display_url="https://cdn.example/thumb.jpg",
|
|
18
|
+
video_url="https://cdn.example/v.mp4",
|
|
19
|
+
duration=18.018,
|
|
20
|
+
width=1920,
|
|
21
|
+
height=1080,
|
|
22
|
+
)
|
|
23
|
+
],
|
|
24
|
+
)
|
|
25
|
+
info = YtVideoInfo(
|
|
26
|
+
title="t",
|
|
27
|
+
description="",
|
|
28
|
+
thumbnail=None,
|
|
29
|
+
url="https://www.instagram.com/p/abc/",
|
|
30
|
+
info_json={},
|
|
31
|
+
duration=0,
|
|
32
|
+
width=1920,
|
|
33
|
+
height=1080,
|
|
34
|
+
)
|
|
35
|
+
parser = InstagramParser()
|
|
36
|
+
|
|
37
|
+
with (
|
|
38
|
+
patch.object(InstagramParser, "_parse", AsyncMock(return_value=post)),
|
|
39
|
+
patch(
|
|
40
|
+
"parsehub.parsers.parser.instagram.InstagramYtParse._parse",
|
|
41
|
+
AsyncMock(return_value=info),
|
|
42
|
+
),
|
|
43
|
+
):
|
|
44
|
+
result = await parser._do_parse("https://www.instagram.com/reel/abc/")
|
|
45
|
+
|
|
46
|
+
self.assertEqual(result.media.url, "https://cdn.example/v.mp4")
|
|
47
|
+
self.assertEqual(result.media.duration, 18)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class TestInstagramAPIDuration(unittest.IsolatedAsyncioTestCase):
|
|
51
|
+
@staticmethod
|
|
52
|
+
def _embed_node() -> dict:
|
|
53
|
+
return {
|
|
54
|
+
"__typename": "GraphVideo",
|
|
55
|
+
"shortcode": "abc",
|
|
56
|
+
"display_url": "https://cdn.example/thumb.jpg",
|
|
57
|
+
"video_url": "https://cdn.example/v.mp4",
|
|
58
|
+
"video_duration": 30.933,
|
|
59
|
+
"dimensions": {"width": 1080, "height": 1920},
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async def test_fills_duration_from_embed(self):
|
|
63
|
+
api = InstagramAPI()
|
|
64
|
+
post = InstagramPost(
|
|
65
|
+
shortcode="abc",
|
|
66
|
+
type=InstagramMediaType.VIDEO,
|
|
67
|
+
media=[
|
|
68
|
+
InstagramMedia(
|
|
69
|
+
is_video=True,
|
|
70
|
+
display_url="https://cdn.example/thumb.jpg",
|
|
71
|
+
video_url="https://cdn.example/v.mp4",
|
|
72
|
+
duration=0,
|
|
73
|
+
)
|
|
74
|
+
],
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
with patch.object(InstagramAPI, "_fetch_embed", AsyncMock(return_value=self._embed_node())):
|
|
78
|
+
result = await api._fill_missing_video_fields(object(), post, "abc")
|
|
79
|
+
|
|
80
|
+
self.assertEqual(result.media[0].duration, 30.933)
|
|
81
|
+
|
|
82
|
+
async def test_fills_missing_video_url_from_embed(self):
|
|
83
|
+
api = InstagramAPI()
|
|
84
|
+
post = InstagramPost(
|
|
85
|
+
shortcode="abc",
|
|
86
|
+
type=InstagramMediaType.VIDEO,
|
|
87
|
+
media=[
|
|
88
|
+
InstagramMedia(
|
|
89
|
+
is_video=False,
|
|
90
|
+
display_url="https://cdn.example/thumb.jpg",
|
|
91
|
+
duration=0,
|
|
92
|
+
)
|
|
93
|
+
],
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
with patch.object(InstagramAPI, "_fetch_embed", AsyncMock(return_value=self._embed_node())):
|
|
97
|
+
result = await api._fill_missing_video_fields(object(), post, "abc")
|
|
98
|
+
|
|
99
|
+
self.assertEqual(result.media[0].video_url, "https://cdn.example/v.mp4")
|
|
100
|
+
self.assertTrue(result.media[0].is_video)
|
|
101
|
+
self.assertEqual(result.media[0].duration, 30.933)
|
|
102
|
+
|
|
103
|
+
async def test_skips_embed_when_complete(self):
|
|
104
|
+
api = InstagramAPI()
|
|
105
|
+
post = InstagramPost(
|
|
106
|
+
shortcode="abc",
|
|
107
|
+
type=InstagramMediaType.VIDEO,
|
|
108
|
+
media=[
|
|
109
|
+
InstagramMedia(
|
|
110
|
+
is_video=True,
|
|
111
|
+
display_url="https://cdn.example/thumb.jpg",
|
|
112
|
+
video_url="https://cdn.example/v.mp4",
|
|
113
|
+
duration=18,
|
|
114
|
+
)
|
|
115
|
+
],
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
with patch.object(InstagramAPI, "_fetch_embed", AsyncMock()) as fetch:
|
|
119
|
+
await api._fill_missing_video_fields(object(), post, "abc")
|
|
120
|
+
|
|
121
|
+
fetch.assert_not_awaited()
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class _FakeYtParser(YtParser, register=False):
|
|
125
|
+
async def _extract_info(self, url: str) -> dict:
|
|
126
|
+
return {"title": "t", "duration": 10, "webpage_url": url}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class TestYtParserDescription(unittest.IsolatedAsyncioTestCase):
|
|
130
|
+
async def test_missing_description_does_not_raise(self):
|
|
131
|
+
info = await _FakeYtParser()._parse("https://example.com/v")
|
|
132
|
+
|
|
133
|
+
self.assertEqual(info.description, "")
|
|
134
|
+
self.assertEqual(info.duration, 10)
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
from unittest.mock import AsyncMock, patch
|
|
3
|
+
|
|
4
|
+
from parsehub.parsers.parser.zuiyou import ZuiYouParser
|
|
5
|
+
from parsehub.provider_api.zuiyou import Media, MediaType, ZuiYou, ZuiYouPost
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class TestZuiyouDuration(unittest.TestCase):
|
|
9
|
+
def test_parse_reads_video_dur(self):
|
|
10
|
+
payload = {
|
|
11
|
+
"data": {
|
|
12
|
+
"post": {
|
|
13
|
+
"content": "x",
|
|
14
|
+
"imgs": [
|
|
15
|
+
{
|
|
16
|
+
"id": 1,
|
|
17
|
+
"urls": {"a": {"urls": ["https://cdn.example/thumb.jpg"]}},
|
|
18
|
+
"video": True,
|
|
19
|
+
}
|
|
20
|
+
],
|
|
21
|
+
"videos": {"1": {"dur": 5, "url": "https://cdn.example/v.mp4"}},
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
post = ZuiYouPost.parse(payload)
|
|
27
|
+
|
|
28
|
+
self.assertEqual(post.media[0].type, MediaType.VIDEO)
|
|
29
|
+
self.assertEqual(post.media[0].duration, 5)
|
|
30
|
+
self.assertEqual(post.media[0].url, "https://cdn.example/v.mp4")
|
|
31
|
+
|
|
32
|
+
def test_photo_duration_defaults_to_zero(self):
|
|
33
|
+
payload = {
|
|
34
|
+
"data": {
|
|
35
|
+
"post": {
|
|
36
|
+
"imgs": [
|
|
37
|
+
{
|
|
38
|
+
"id": 1,
|
|
39
|
+
"urls": {"a": {"urls": ["https://cdn.example/p.jpg"]}},
|
|
40
|
+
}
|
|
41
|
+
]
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
post = ZuiYouPost.parse(payload)
|
|
47
|
+
|
|
48
|
+
self.assertEqual(post.media[0].duration, 0)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class TestZuiyouParserDuration(unittest.IsolatedAsyncioTestCase):
|
|
52
|
+
async def test_parser_sets_video_duration(self):
|
|
53
|
+
zy = ZuiYouPost(
|
|
54
|
+
content="x",
|
|
55
|
+
media=[
|
|
56
|
+
Media(
|
|
57
|
+
url="https://cdn.example/v.mp4",
|
|
58
|
+
thumb_url="https://cdn.example/thumb.jpg",
|
|
59
|
+
type=MediaType.VIDEO,
|
|
60
|
+
duration=19,
|
|
61
|
+
)
|
|
62
|
+
],
|
|
63
|
+
raw={},
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
with patch.object(ZuiYou, "parse", AsyncMock(return_value=zy)):
|
|
67
|
+
result = await ZuiYouParser()._do_parse("https://s.izuiyou.com/s/abc")
|
|
68
|
+
|
|
69
|
+
self.assertEqual(result.media[0].duration, 19)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|