wechat-article-parser 0.0.9__tar.gz → 0.0.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/PKG-INFO +19 -7
- {wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/README.md +18 -6
- {wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/pyproject.toml +1 -1
- {wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/src/wechat_article_parser/models.py +5 -2
- {wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/src/wechat_article_parser/parser.py +108 -33
- {wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/tests/test_models.py +10 -6
- wechat_article_parser-0.0.11/tests/test_paid_content.py +106 -0
- {wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/tests/test_parser.py +18 -2
- wechat_article_parser-0.0.11/tests/test_product_content.py +83 -0
- wechat_article_parser-0.0.11/tests/test_profile_content.py +127 -0
- wechat_article_parser-0.0.11/tests/test_repost_content.py +140 -0
- wechat_article_parser-0.0.11/tests/test_svg_content.py +98 -0
- {wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/tests/test_type_routing.py +8 -3
- wechat_article_parser-0.0.11/tests/test_video_content.py +85 -0
- {wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/.gitignore +0 -0
- {wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/LICENSE +0 -0
- {wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/src/wechat_article_parser/__init__.py +0 -0
- {wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/tests/conftest.py +0 -0
- {wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/tests/test_markdown.py +0 -0
- {wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/tests/test_swiper_content.py +0 -0
- {wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/tests/test_user_name.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: wechat-article-parser
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.11
|
|
4
4
|
Summary: WeChat MP article parser - extract metadata and content from WeChat public account articles
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -23,13 +23,22 @@ Description-Content-Type: text/markdown
|
|
|
23
23
|
- 转载式文章
|
|
24
24
|
- 纯文本文章
|
|
25
25
|
- 视频分享类文章
|
|
26
|
+
- 音频分享类文章
|
|
26
27
|
- 小红书风格图片轮播文章
|
|
27
28
|
- 全屏布局短文本文章
|
|
28
29
|
|
|
29
|
-
解析时优先读取页面全局的 `item_show_type`:`0` 对应图文、`5`
|
|
30
|
+
解析时优先读取页面全局的 `item_show_type`:`0` 对应图文、`5` 对应视频分享、`7` 对应音频分享、
|
|
30
31
|
`8` 对应图片轮播、`10` 对应文字消息,再通过 HTML 或脚本数据区分具体模板。
|
|
31
32
|
类型字段缺失、值未知或对应分支未提取到正文时,会按原有 HTML 特征匹配顺序兜底。
|
|
32
33
|
|
|
34
|
+
图文正文仅包含视频号视频卡片或微信视频 iframe 时,Markdown 返回“这是一篇纯视频内容。”。
|
|
35
|
+
正文只有付费预览占位标签时,Markdown 返回“这是一篇付费内容,正文需付费后查看。”,
|
|
36
|
+
并保留公开元数据;已有的可见预览正文仍正常提取。
|
|
37
|
+
图文中的公众号推荐卡片按原文顺序转换为 `**公众号名称**:简介`,每张卡片一段;
|
|
38
|
+
名称或简介缺失时,仅输出已有字段。
|
|
39
|
+
图文正文仅包含商品广告卡片(`mp-common-cpsad`)时,Markdown 返回“这是一篇商品推荐内容。”;
|
|
40
|
+
已有的文字和图片仍正常提取。关键元数据完整时占位结果也有效,不表示已获取商品详情。
|
|
41
|
+
|
|
33
42
|
## 安装
|
|
34
43
|
|
|
35
44
|
```bash
|
|
@@ -113,15 +122,17 @@ result = parse(
|
|
|
113
122
|
| `article_title` | `str` | 文章标题 |
|
|
114
123
|
| `article_cover_image` | `str` | 文章封面图链接 |
|
|
115
124
|
| `article_description` | `str` | 文章摘要 |
|
|
116
|
-
| `article_markdown` | `str` | 文章正文的 Markdown
|
|
125
|
+
| `article_markdown` | `str` | 文章正文的 Markdown 内容;视频或音频分享没有文字附言时可为空 |
|
|
117
126
|
| `article_publish_time` | `int` | 发布时间(Unix 时间戳) |
|
|
118
|
-
| `article_display_type` | `ArticleDisplayType` | 文章展示类型:`article`、`video`、`image`、`text` 或 `unknown` |
|
|
119
|
-
| `article_item_show_type` | `int \| None` | 微信页面中的原始展示类型:`0` 图文、`5` 视频、`8` 图片轮播、`10` 文字;其他已提取的数值原样保留,缺失或无法解析时为 `None` |
|
|
127
|
+
| `article_display_type` | `ArticleDisplayType` | 文章展示类型:`article`、`video`、`audio`、`image`、`text` 或 `unknown` |
|
|
128
|
+
| `article_item_show_type` | `int \| None` | 微信页面中的原始展示类型:`0` 图文、`5` 视频、`7` 音频、`8` 图片轮播、`10` 文字;其他已提取的数值原样保留,缺失或无法解析时为 `None` |
|
|
120
129
|
| `raw_html` | `str` | 原始 HTML 网页源代码(仅在 `include_raw_html=True` 时填充,否则为空字符串) |
|
|
121
130
|
| `images` | `list[str]` | 文章中提取的所有图片链接 |
|
|
122
131
|
| `is_valid` | `bool` | 关键字段是否全部解析成功(属性) |
|
|
123
132
|
|
|
124
|
-
`is_valid` 为 `True` 的条件:`mp_id`、`mp_name`、`article_id`、`article_msg_id`、`article_idx`、`article_sn`、`article_title`、`article_publish_time` 均不为空/零,且 `article_markdown` 非空或 `article_display_type
|
|
133
|
+
`is_valid` 为 `True` 的条件:`mp_id`、`mp_name`、`article_id`、`article_msg_id`、`article_idx`、`article_sn`、`article_title`、`article_publish_time` 均不为空/零,且 `article_markdown` 非空或 `article_display_type in ("video", "audio")`。视频和音频分享允许没有文字正文,仍须满足其他关键字段要求;有效性不表示已获取视频或音频文件。
|
|
134
|
+
|
|
135
|
+
付费受限页面返回提示且关键元数据完整时,`is_valid` 也为 `True`,这不表示已获取付费正文。
|
|
125
136
|
|
|
126
137
|
### 判断文章展示类型
|
|
127
138
|
|
|
@@ -131,6 +142,7 @@ result = parse(
|
|
|
131
142
|
|---|---|---|
|
|
132
143
|
| `ArticleDisplayType.ARTICLE` | `article` | 图文,包括转载式 |
|
|
133
144
|
| `ArticleDisplayType.VIDEO` | `video` | 视频分享 |
|
|
145
|
+
| `ArticleDisplayType.AUDIO` | `audio` | 音频分享 |
|
|
134
146
|
| `ArticleDisplayType.IMAGE` | `image` | 图片轮播 |
|
|
135
147
|
| `ArticleDisplayType.TEXT` | `text` | 文字消息 |
|
|
136
148
|
| `ArticleDisplayType.UNKNOWN` | `unknown` | 页面类型缺失或未支持 |
|
|
@@ -204,7 +216,7 @@ except httpx.HTTPStatusError as e:
|
|
|
204
216
|
|
|
205
217
|
### 解析不完整
|
|
206
218
|
|
|
207
|
-
|
|
219
|
+
缺少标题、发布时间等关键字段,或非视频/音频分享缺少正文时,解析结果的 `is_valid` 会返回 `False`。封面图、摘要等非必填字段为空不影响有效性。建议在业务逻辑中检查:
|
|
208
220
|
|
|
209
221
|
```python
|
|
210
222
|
result = parse("https://mp.weixin.qq.com/s/xxxxx")
|
|
@@ -8,13 +8,22 @@
|
|
|
8
8
|
- 转载式文章
|
|
9
9
|
- 纯文本文章
|
|
10
10
|
- 视频分享类文章
|
|
11
|
+
- 音频分享类文章
|
|
11
12
|
- 小红书风格图片轮播文章
|
|
12
13
|
- 全屏布局短文本文章
|
|
13
14
|
|
|
14
|
-
解析时优先读取页面全局的 `item_show_type`:`0` 对应图文、`5`
|
|
15
|
+
解析时优先读取页面全局的 `item_show_type`:`0` 对应图文、`5` 对应视频分享、`7` 对应音频分享、
|
|
15
16
|
`8` 对应图片轮播、`10` 对应文字消息,再通过 HTML 或脚本数据区分具体模板。
|
|
16
17
|
类型字段缺失、值未知或对应分支未提取到正文时,会按原有 HTML 特征匹配顺序兜底。
|
|
17
18
|
|
|
19
|
+
图文正文仅包含视频号视频卡片或微信视频 iframe 时,Markdown 返回“这是一篇纯视频内容。”。
|
|
20
|
+
正文只有付费预览占位标签时,Markdown 返回“这是一篇付费内容,正文需付费后查看。”,
|
|
21
|
+
并保留公开元数据;已有的可见预览正文仍正常提取。
|
|
22
|
+
图文中的公众号推荐卡片按原文顺序转换为 `**公众号名称**:简介`,每张卡片一段;
|
|
23
|
+
名称或简介缺失时,仅输出已有字段。
|
|
24
|
+
图文正文仅包含商品广告卡片(`mp-common-cpsad`)时,Markdown 返回“这是一篇商品推荐内容。”;
|
|
25
|
+
已有的文字和图片仍正常提取。关键元数据完整时占位结果也有效,不表示已获取商品详情。
|
|
26
|
+
|
|
18
27
|
## 安装
|
|
19
28
|
|
|
20
29
|
```bash
|
|
@@ -98,15 +107,17 @@ result = parse(
|
|
|
98
107
|
| `article_title` | `str` | 文章标题 |
|
|
99
108
|
| `article_cover_image` | `str` | 文章封面图链接 |
|
|
100
109
|
| `article_description` | `str` | 文章摘要 |
|
|
101
|
-
| `article_markdown` | `str` | 文章正文的 Markdown
|
|
110
|
+
| `article_markdown` | `str` | 文章正文的 Markdown 内容;视频或音频分享没有文字附言时可为空 |
|
|
102
111
|
| `article_publish_time` | `int` | 发布时间(Unix 时间戳) |
|
|
103
|
-
| `article_display_type` | `ArticleDisplayType` | 文章展示类型:`article`、`video`、`image`、`text` 或 `unknown` |
|
|
104
|
-
| `article_item_show_type` | `int \| None` | 微信页面中的原始展示类型:`0` 图文、`5` 视频、`8` 图片轮播、`10` 文字;其他已提取的数值原样保留,缺失或无法解析时为 `None` |
|
|
112
|
+
| `article_display_type` | `ArticleDisplayType` | 文章展示类型:`article`、`video`、`audio`、`image`、`text` 或 `unknown` |
|
|
113
|
+
| `article_item_show_type` | `int \| None` | 微信页面中的原始展示类型:`0` 图文、`5` 视频、`7` 音频、`8` 图片轮播、`10` 文字;其他已提取的数值原样保留,缺失或无法解析时为 `None` |
|
|
105
114
|
| `raw_html` | `str` | 原始 HTML 网页源代码(仅在 `include_raw_html=True` 时填充,否则为空字符串) |
|
|
106
115
|
| `images` | `list[str]` | 文章中提取的所有图片链接 |
|
|
107
116
|
| `is_valid` | `bool` | 关键字段是否全部解析成功(属性) |
|
|
108
117
|
|
|
109
|
-
`is_valid` 为 `True` 的条件:`mp_id`、`mp_name`、`article_id`、`article_msg_id`、`article_idx`、`article_sn`、`article_title`、`article_publish_time` 均不为空/零,且 `article_markdown` 非空或 `article_display_type
|
|
118
|
+
`is_valid` 为 `True` 的条件:`mp_id`、`mp_name`、`article_id`、`article_msg_id`、`article_idx`、`article_sn`、`article_title`、`article_publish_time` 均不为空/零,且 `article_markdown` 非空或 `article_display_type in ("video", "audio")`。视频和音频分享允许没有文字正文,仍须满足其他关键字段要求;有效性不表示已获取视频或音频文件。
|
|
119
|
+
|
|
120
|
+
付费受限页面返回提示且关键元数据完整时,`is_valid` 也为 `True`,这不表示已获取付费正文。
|
|
110
121
|
|
|
111
122
|
### 判断文章展示类型
|
|
112
123
|
|
|
@@ -116,6 +127,7 @@ result = parse(
|
|
|
116
127
|
|---|---|---|
|
|
117
128
|
| `ArticleDisplayType.ARTICLE` | `article` | 图文,包括转载式 |
|
|
118
129
|
| `ArticleDisplayType.VIDEO` | `video` | 视频分享 |
|
|
130
|
+
| `ArticleDisplayType.AUDIO` | `audio` | 音频分享 |
|
|
119
131
|
| `ArticleDisplayType.IMAGE` | `image` | 图片轮播 |
|
|
120
132
|
| `ArticleDisplayType.TEXT` | `text` | 文字消息 |
|
|
121
133
|
| `ArticleDisplayType.UNKNOWN` | `unknown` | 页面类型缺失或未支持 |
|
|
@@ -189,7 +201,7 @@ except httpx.HTTPStatusError as e:
|
|
|
189
201
|
|
|
190
202
|
### 解析不完整
|
|
191
203
|
|
|
192
|
-
|
|
204
|
+
缺少标题、发布时间等关键字段,或非视频/音频分享缺少正文时,解析结果的 `is_valid` 会返回 `False`。封面图、摘要等非必填字段为空不影响有效性。建议在业务逻辑中检查:
|
|
193
205
|
|
|
194
206
|
```python
|
|
195
207
|
result = parse("https://mp.weixin.qq.com/s/xxxxx")
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "wechat-article-parser"
|
|
7
|
-
version = "0.0.
|
|
7
|
+
version = "0.0.11"
|
|
8
8
|
description = "WeChat MP article parser - extract metadata and content from WeChat public account articles"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
{wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/src/wechat_article_parser/models.py
RENAMED
|
@@ -28,6 +28,7 @@ class ArticleDisplayType(str, Enum):
|
|
|
28
28
|
UNKNOWN = "unknown"
|
|
29
29
|
ARTICLE = "article"
|
|
30
30
|
VIDEO = "video"
|
|
31
|
+
AUDIO = "audio"
|
|
31
32
|
IMAGE = "image"
|
|
32
33
|
TEXT = "text"
|
|
33
34
|
|
|
@@ -77,7 +78,7 @@ class ArticleResult:
|
|
|
77
78
|
|
|
78
79
|
@property
|
|
79
80
|
def is_valid(self) -> bool:
|
|
80
|
-
"""
|
|
81
|
+
"""检查关键字段是否解析成功;视频和音频分享允许正文为空。"""
|
|
81
82
|
return bool(
|
|
82
83
|
self.mp_id
|
|
83
84
|
and self.mp_name
|
|
@@ -88,7 +89,9 @@ class ArticleResult:
|
|
|
88
89
|
and self.article_title
|
|
89
90
|
and (
|
|
90
91
|
self.article_markdown
|
|
91
|
-
or self.article_display_type
|
|
92
|
+
or self.article_display_type in (
|
|
93
|
+
ArticleDisplayType.VIDEO, ArticleDisplayType.AUDIO,
|
|
94
|
+
)
|
|
92
95
|
)
|
|
93
96
|
and self.article_publish_time
|
|
94
97
|
)
|
{wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/src/wechat_article_parser/parser.py
RENAMED
|
@@ -78,6 +78,21 @@ def _decode_hex_escapes(text: str) -> str:
|
|
|
78
78
|
return re.sub(r"\\x([0-9a-fA-F]{2})", lambda m: chr(int(m.group(1), 16)), text)
|
|
79
79
|
|
|
80
80
|
|
|
81
|
+
def _decode_js_string(text: str) -> str:
|
|
82
|
+
"""解码 JS 字符串转义,保留 HTML 实体、百分号和正文中的空白。"""
|
|
83
|
+
escapes = {
|
|
84
|
+
"n": "\n", "r": "\r", "t": "\t", "b": "\b", "f": "\f", "v": "\v",
|
|
85
|
+
"0": "\0", "\n": "", "\r": "", "\r\n": "",
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
def decode(match: re.Match[str]) -> str:
|
|
89
|
+
if match[1] or match[2]:
|
|
90
|
+
return chr(int(match[1] or match[2], 16))
|
|
91
|
+
return escapes.get(match[3], match[3])
|
|
92
|
+
|
|
93
|
+
return re.sub(r"\\(?:x([\da-fA-F]{2})|u([\da-fA-F]{4})|(\r\n|[\s\S]))", decode, text)
|
|
94
|
+
|
|
95
|
+
|
|
81
96
|
def _decode_text(text: str, *, preserve_newlines: bool = False) -> str:
|
|
82
97
|
if not text:
|
|
83
98
|
return ""
|
|
@@ -265,8 +280,13 @@ def _extract_user_name(scripts: list[Tag]) -> str:
|
|
|
265
280
|
|
|
266
281
|
def _extract_cgi_user_name(scripts: list[Tag]) -> str:
|
|
267
282
|
"""从当前文章的 cgiDataNew 对象中读取 user_name,排除嵌套账号。"""
|
|
283
|
+
return _decode_text(_extract_cgi_string(scripts, "user_name"))
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _extract_cgi_string(scripts: list[Tag], field: str) -> str:
|
|
287
|
+
"""读取当前 cgiDataNew 的直接字符串属性,不执行表达式或读取嵌套对象。"""
|
|
268
288
|
for script in scripts:
|
|
269
|
-
if "cgiDataNew" not in script.text or
|
|
289
|
+
if "cgiDataNew" not in script.text or field not in script.text:
|
|
270
290
|
continue
|
|
271
291
|
history: deque[str] = deque(maxlen=5)
|
|
272
292
|
stack: list[str] = []
|
|
@@ -289,12 +309,12 @@ def _extract_cgi_user_name(scripts: list[Tag]) -> str:
|
|
|
289
309
|
and token in (",", "}")
|
|
290
310
|
and len(previous) >= 4
|
|
291
311
|
and previous[-4] in ("{", ",")
|
|
292
|
-
and previous[-3].strip("\"'") ==
|
|
312
|
+
and previous[-3].strip("\"'") == field
|
|
293
313
|
and previous[-2] == ":"
|
|
294
314
|
and previous[-1].startswith(("'", '"'))
|
|
295
315
|
):
|
|
296
|
-
value =
|
|
297
|
-
if value:
|
|
316
|
+
value = previous[-1][1:-1]
|
|
317
|
+
if value.strip():
|
|
298
318
|
return value
|
|
299
319
|
if token in ("{", "["):
|
|
300
320
|
stack.append(token)
|
|
@@ -421,7 +441,26 @@ def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None
|
|
|
421
441
|
"""将 rich_media_content 区块转换为 Markdown。"""
|
|
422
442
|
# prettify 会在行内标签之间插入空白,改变代码和引用链接的内容。
|
|
423
443
|
soup = BeautifulSoup(str(content_tag), "html.parser")
|
|
424
|
-
|
|
444
|
+
has_video = soup.select_one(
|
|
445
|
+
'mp-common-videosnap[data-type="video"], iframe.video_iframe, iframe.wx_video_iframe'
|
|
446
|
+
) is not None
|
|
447
|
+
|
|
448
|
+
# 原生公众号卡片的正文保存在属性中,转换为“名称 + 简介”的独立段落。
|
|
449
|
+
for profile in soup.find_all("mp-common-profile"):
|
|
450
|
+
nickname = profile.get("data-nickname", "").strip()
|
|
451
|
+
signature = profile.get("data-signature", "").strip()
|
|
452
|
+
if not nickname and not signature:
|
|
453
|
+
continue
|
|
454
|
+
paragraph = soup.new_tag("p")
|
|
455
|
+
if nickname:
|
|
456
|
+
name = soup.new_tag("strong")
|
|
457
|
+
name.string = nickname
|
|
458
|
+
paragraph.append(name)
|
|
459
|
+
if signature:
|
|
460
|
+
if nickname:
|
|
461
|
+
paragraph.append(":")
|
|
462
|
+
paragraph.append(signature)
|
|
463
|
+
profile.replace_with(paragraph)
|
|
425
464
|
|
|
426
465
|
# 处理 <img> 标签
|
|
427
466
|
to_remove = []
|
|
@@ -430,17 +469,13 @@ def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None
|
|
|
430
469
|
if src and src.startswith("http"):
|
|
431
470
|
normalized = _normalize_image_url(src)
|
|
432
471
|
img["src"] = normalized
|
|
433
|
-
if normalized not in seen:
|
|
434
|
-
seen.add(normalized)
|
|
435
|
-
result.images.append(normalized)
|
|
436
472
|
else:
|
|
437
473
|
to_remove.append(img)
|
|
438
474
|
for tag in to_remove:
|
|
439
475
|
tag.decompose()
|
|
440
476
|
|
|
441
|
-
#
|
|
442
|
-
#
|
|
443
|
-
# 需从外向内收集所有图片 URL,一次性替换为多个 <img> 标签
|
|
477
|
+
# SVG 可通过 background-image 或 foreignObject 内的 <img> 承载正文图片。
|
|
478
|
+
# 按文档顺序收集两种图片,再统一替换外层 SVG,避免删除内部正文图片。
|
|
444
479
|
to_remove = []
|
|
445
480
|
for svg in soup.find_all("svg", recursive=True):
|
|
446
481
|
# 跳过已被外层 SVG 处理过的嵌套 SVG(已脱离文档树)
|
|
@@ -450,11 +485,14 @@ def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None
|
|
|
450
485
|
if svg.find_parent("svg"):
|
|
451
486
|
continue
|
|
452
487
|
|
|
453
|
-
#
|
|
454
|
-
|
|
488
|
+
# 包含所有嵌套 SVG 和普通图片,保留正文中的重复图片及 alt 文本。
|
|
489
|
+
image_elements = [svg] + svg.find_all(["svg", "img"])
|
|
455
490
|
img_tags = []
|
|
456
|
-
for
|
|
457
|
-
|
|
491
|
+
for element in image_elements:
|
|
492
|
+
if element.name == "img":
|
|
493
|
+
img_tags.append(soup.new_tag("img", attrs=element.attrs.copy()))
|
|
494
|
+
continue
|
|
495
|
+
style = element.get("style", "")
|
|
458
496
|
if "background-image" not in style:
|
|
459
497
|
continue
|
|
460
498
|
m = re.search(r'url\("([^"]+)"\)', style)
|
|
@@ -462,9 +500,6 @@ def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None
|
|
|
462
500
|
continue
|
|
463
501
|
normalized = _normalize_image_url(m.group(1))
|
|
464
502
|
img_tags.append(soup.new_tag("img", src=normalized))
|
|
465
|
-
if normalized not in seen:
|
|
466
|
-
seen.add(normalized)
|
|
467
|
-
result.images.append(normalized)
|
|
468
503
|
|
|
469
504
|
if img_tags:
|
|
470
505
|
svg.replace_with(*img_tags)
|
|
@@ -473,30 +508,68 @@ def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None
|
|
|
473
508
|
for tag in to_remove:
|
|
474
509
|
tag.decompose()
|
|
475
510
|
|
|
476
|
-
|
|
477
|
-
|
|
511
|
+
# 从最终正文收集图片,使列表顺序与 Markdown 一致,并只对列表去重。
|
|
512
|
+
seen: set[str] = set()
|
|
513
|
+
for img in soup.find_all("img"):
|
|
514
|
+
src = img["src"]
|
|
515
|
+
if src not in seen:
|
|
516
|
+
seen.add(src)
|
|
517
|
+
result.images.append(src)
|
|
478
518
|
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
519
|
+
result.article_markdown = _to_markdown(str(soup))
|
|
520
|
+
if not result.article_markdown.strip():
|
|
521
|
+
if soup.find("mp-pay-preview-filter") is not None:
|
|
522
|
+
result.article_markdown = "这是一篇付费内容,正文需付费后查看。"
|
|
523
|
+
elif has_video:
|
|
524
|
+
result.article_markdown = "这是一篇纯视频内容。"
|
|
525
|
+
elif soup.find("mp-common-cpsad") is not None:
|
|
526
|
+
result.article_markdown = "这是一篇商品推荐内容。"
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def _extract_repost_content(
|
|
530
|
+
content_tag: Tag, scripts: list[Tag], result: ArticleResult,
|
|
531
|
+
) -> None:
|
|
532
|
+
"""读取转载附言、cgiDataNew 中的富文本正文和原文链接。"""
|
|
533
|
+
soup = BeautifulSoup(str(content_tag), "html.parser")
|
|
534
|
+
parts: list[str] = []
|
|
482
535
|
notice = soup.find("p", id="js_share_notice")
|
|
483
|
-
if
|
|
484
|
-
|
|
536
|
+
if notice:
|
|
537
|
+
m = re.search(
|
|
538
|
+
r'''innerHTML\s*=\s*("(?:\\[\s\S]|[^"\\])*"|'(?:\\[\s\S]|[^'\\])*')''',
|
|
539
|
+
str(notice),
|
|
540
|
+
)
|
|
541
|
+
if m:
|
|
542
|
+
text = _decode_text(_decode_js_string(m[1][1:-1]), preserve_newlines=True)
|
|
543
|
+
notice_markdown = _to_markdown_plain(f"<p>{text}</p>")
|
|
544
|
+
else:
|
|
545
|
+
for script in notice.find_all("script"):
|
|
546
|
+
script.decompose()
|
|
547
|
+
notice_markdown = _to_markdown_plain(str(notice))
|
|
548
|
+
if notice_markdown.strip():
|
|
549
|
+
parts.append(notice_markdown.strip())
|
|
550
|
+
|
|
551
|
+
for field in ("content_noencode", "content"):
|
|
552
|
+
body = _extract_cgi_string(scripts, field)
|
|
553
|
+
if body:
|
|
554
|
+
# 与普通富文本共用图片和 Markdown 处理,避免 data-src 变成空图片链接。
|
|
555
|
+
body_soup = BeautifulSoup(_decode_js_string(body), "html.parser")
|
|
556
|
+
_extract_rich_media_content(body_soup, result)
|
|
557
|
+
if result.article_markdown.strip():
|
|
558
|
+
parts.append(result.article_markdown.strip())
|
|
559
|
+
break
|
|
485
560
|
|
|
486
|
-
|
|
487
|
-
if not m:
|
|
561
|
+
if not parts:
|
|
488
562
|
return
|
|
489
563
|
|
|
490
|
-
text = _decode_text(m.group(1), preserve_newlines=True)
|
|
491
|
-
html_content = f"<p>{text}</p>"
|
|
492
|
-
|
|
493
564
|
share_link = soup.find("span", id="js_share_source")
|
|
494
565
|
if share_link:
|
|
495
566
|
href = share_link.get("data-url")
|
|
496
567
|
if href:
|
|
497
|
-
|
|
568
|
+
link = soup.new_tag("a", href=href)
|
|
569
|
+
link.string = "查看原文"
|
|
570
|
+
parts.append(_to_markdown_plain(str(link)))
|
|
498
571
|
|
|
499
|
-
result.article_markdown =
|
|
572
|
+
result.article_markdown = "\n\n".join(parts)
|
|
500
573
|
|
|
501
574
|
|
|
502
575
|
def _extract_plain_text_content(soup: BeautifulSoup, result: ArticleResult) -> None:
|
|
@@ -589,12 +662,14 @@ _CONTENT_SELECTORS = {
|
|
|
589
662
|
_ITEM_SHOW_TYPE_BRANCHES = {
|
|
590
663
|
0: ("rich_text", "repost"),
|
|
591
664
|
5: ("video",),
|
|
665
|
+
7: ("video",), # 音频分享复用视频分享的文字附言提取。
|
|
592
666
|
8: ("swiper",),
|
|
593
667
|
10: ("plain_text", "fullscreen"),
|
|
594
668
|
}
|
|
595
669
|
_ARTICLE_DISPLAY_TYPES = {
|
|
596
670
|
0: ArticleDisplayType.ARTICLE,
|
|
597
671
|
5: ArticleDisplayType.VIDEO,
|
|
672
|
+
7: ArticleDisplayType.AUDIO,
|
|
598
673
|
8: ArticleDisplayType.IMAGE,
|
|
599
674
|
10: ArticleDisplayType.TEXT,
|
|
600
675
|
}
|
|
@@ -622,7 +697,7 @@ def _try_content_branch(
|
|
|
622
697
|
if branch == "rich_text":
|
|
623
698
|
_extract_rich_media_content(content, candidate)
|
|
624
699
|
elif branch == "repost":
|
|
625
|
-
_extract_repost_content(content, candidate)
|
|
700
|
+
_extract_repost_content(content, scripts, candidate)
|
|
626
701
|
elif branch in ("plain_text", "video"):
|
|
627
702
|
_extract_plain_text_content(soup, candidate)
|
|
628
703
|
elif branch == "swiper":
|
|
@@ -22,14 +22,16 @@ _COMPLETE = ArticleResult(
|
|
|
22
22
|
|
|
23
23
|
@pytest.mark.parametrize("display_type", list(ArticleDisplayType))
|
|
24
24
|
@pytest.mark.parametrize("markdown", ["", "正文"])
|
|
25
|
-
def
|
|
25
|
+
def test_only_video_and_audio_can_be_valid_without_markdown(
|
|
26
26
|
display_type: ArticleDisplayType, markdown: str,
|
|
27
27
|
) -> None:
|
|
28
28
|
result = replace(
|
|
29
29
|
_COMPLETE, article_display_type=display_type, article_markdown=markdown,
|
|
30
30
|
)
|
|
31
31
|
|
|
32
|
-
assert result.is_valid is (
|
|
32
|
+
assert result.is_valid is (
|
|
33
|
+
bool(markdown) or display_type in (ArticleDisplayType.VIDEO, ArticleDisplayType.AUDIO)
|
|
34
|
+
)
|
|
33
35
|
|
|
34
36
|
|
|
35
37
|
@pytest.mark.parametrize(
|
|
@@ -45,13 +47,14 @@ def test_only_video_can_be_valid_without_markdown(
|
|
|
45
47
|
("article_publish_time", 0),
|
|
46
48
|
],
|
|
47
49
|
)
|
|
48
|
-
@pytest.mark.parametrize("markdown", ["", "
|
|
49
|
-
|
|
50
|
-
|
|
50
|
+
@pytest.mark.parametrize("markdown", ["", "媒体附言"])
|
|
51
|
+
@pytest.mark.parametrize("display_type", [ArticleDisplayType.VIDEO, ArticleDisplayType.AUDIO])
|
|
52
|
+
def test_video_and_audio_still_require_all_key_metadata(
|
|
53
|
+
field_name: str, empty_value: str | int, markdown: str, display_type: ArticleDisplayType,
|
|
51
54
|
) -> None:
|
|
52
55
|
result = replace(
|
|
53
56
|
_COMPLETE,
|
|
54
|
-
article_display_type=
|
|
57
|
+
article_display_type=display_type,
|
|
55
58
|
article_markdown=markdown,
|
|
56
59
|
**{field_name: empty_value},
|
|
57
60
|
)
|
|
@@ -64,6 +67,7 @@ def test_video_still_requires_all_key_metadata(
|
|
|
64
67
|
[
|
|
65
68
|
"article",
|
|
66
69
|
"video",
|
|
70
|
+
"audio",
|
|
67
71
|
"image",
|
|
68
72
|
"text",
|
|
69
73
|
"unknown",
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""付费受限正文的提示、元数据和预览内容回归测试。"""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from wechat_article_parser import ArticleDisplayType
|
|
6
|
+
from wechat_article_parser.parser import _parse_html
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
_URL = "https://mp.weixin.qq.com/s/n2q09-I95zwggfbXKAmbuQ"
|
|
10
|
+
_NOTICE = "这是一篇付费内容,正文需付费后查看。"
|
|
11
|
+
_FILTER = (
|
|
12
|
+
'<p class="js_pay_preview_filter" style="height: 0; overflow: hidden;">'
|
|
13
|
+
'<mp-pay-preview-filter></mp-pay-preview-filter></p>'
|
|
14
|
+
)
|
|
15
|
+
_METADATA = """
|
|
16
|
+
var biz = "MzkwMTYxMzYwNg==";
|
|
17
|
+
var mid = "2247533326";
|
|
18
|
+
var idx = "5";
|
|
19
|
+
var sn = "5ae1c1d004a9ccbf774610a8be1de3d7";
|
|
20
|
+
window.__allowLoadResFromMp = true;
|
|
21
|
+
var hd_head_img = "https://example.com/avatar";
|
|
22
|
+
var nickname = htmlDecode("真锂研究");
|
|
23
|
+
var oriCreateTime = '1791450000';
|
|
24
|
+
var user_name = "gh_61cfd6274c18";
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _html(body: str, metadata: str = _METADATA, data: str = "") -> str:
|
|
29
|
+
return (
|
|
30
|
+
'<meta property="og:title" content="付费文章标题">'
|
|
31
|
+
f'<div class="rich_media_content" id="js_content">{body}</div>'
|
|
32
|
+
'<div id="js_fullscreen_layout_padding"></div>'
|
|
33
|
+
'<script type="text/javascript">var item_show_type = "0";</script>'
|
|
34
|
+
f'<script type="text/javascript">{metadata}</script>'
|
|
35
|
+
f'<script type="text/javascript">{data}</script>'
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@pytest.mark.parametrize("metadata", [_METADATA, ""])
|
|
40
|
+
def test_paid_placeholder_keeps_metadata_and_does_not_use_unrelated_images(
|
|
41
|
+
metadata: str,
|
|
42
|
+
) -> None:
|
|
43
|
+
# 对应真实页面:正文只有付费占位符,脚本另有图片数组。
|
|
44
|
+
data = """
|
|
45
|
+
window.cgiDataNew = {
|
|
46
|
+
is_pay_subscribe: '1' * 1,
|
|
47
|
+
pay_subscribe_info: {preview_percent: '0' * 1},
|
|
48
|
+
picture_page_info_list: [{cdn_url: 'https://example.com/unrelated.jpg'}]
|
|
49
|
+
};
|
|
50
|
+
"""
|
|
51
|
+
result = _parse_html(_URL, _html(_FILTER, metadata, data))
|
|
52
|
+
|
|
53
|
+
assert result.article_markdown == _NOTICE
|
|
54
|
+
assert result.images == []
|
|
55
|
+
assert result.article_display_type is ArticleDisplayType.ARTICLE
|
|
56
|
+
assert result.article_item_show_type == 0
|
|
57
|
+
assert result.is_valid is bool(metadata)
|
|
58
|
+
if metadata:
|
|
59
|
+
assert result.mp_id == 3901613606
|
|
60
|
+
assert result.mp_name == "真锂研究"
|
|
61
|
+
assert result.mp_user_name == "gh_61cfd6274c18"
|
|
62
|
+
assert result.article_msg_id == 2247533326
|
|
63
|
+
assert result.article_idx == 5
|
|
64
|
+
assert result.article_sn == "5ae1c1d004a9ccbf774610a8be1de3d7"
|
|
65
|
+
assert result.article_publish_time == 1791450000
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@pytest.mark.parametrize("body", [
|
|
69
|
+
'<p>可见的免费预览正文。</p>',
|
|
70
|
+
'<img data-src="https://example.com/preview.jpg">',
|
|
71
|
+
])
|
|
72
|
+
def test_paid_filter_does_not_replace_visible_preview(body: str) -> None:
|
|
73
|
+
result = _parse_html(_URL, _html(body + _FILTER))
|
|
74
|
+
|
|
75
|
+
assert result.is_valid
|
|
76
|
+
assert _NOTICE not in result.article_markdown
|
|
77
|
+
if "免费预览" in body:
|
|
78
|
+
assert result.article_markdown == "可见的免费预览正文。"
|
|
79
|
+
else:
|
|
80
|
+
assert result.article_markdown == ""
|
|
81
|
+
assert result.images == ["https://example.com/preview.jpg"]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_paid_filter_takes_priority_over_empty_video_embed() -> None:
|
|
85
|
+
body = '<iframe class="wx_video_iframe"></iframe>' + _FILTER
|
|
86
|
+
result = _parse_html(_URL, _html(body))
|
|
87
|
+
|
|
88
|
+
assert result.is_valid
|
|
89
|
+
assert result.article_markdown == _NOTICE
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def test_paid_marker_outside_article_does_not_fill_empty_body() -> None:
|
|
93
|
+
result = _parse_html(_URL, _html("") + _FILTER)
|
|
94
|
+
|
|
95
|
+
assert result.article_markdown == ""
|
|
96
|
+
assert not result.is_valid
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_paid_flag_without_filter_keeps_accessible_body() -> None:
|
|
100
|
+
result = _parse_html(
|
|
101
|
+
_URL,
|
|
102
|
+
_html('<p>已提供的完整正文。</p>', data="window.cgiDataNew = {is_pay_subscribe: '1' * 1};"),
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
assert result.is_valid
|
|
106
|
+
assert result.article_markdown == "已提供的完整正文。"
|
|
@@ -16,6 +16,22 @@ TEST_URLS = [
|
|
|
16
16
|
"https://mp.weixin.qq.com/s/ySQdtsRlRmAl_skdc5HQ-A",
|
|
17
17
|
"https://mp.weixin.qq.com/s/MnkArbYQNp3tF29gujMUnQ",
|
|
18
18
|
"https://mp.weixin.qq.com/s/ncPoZnxKdvDRLM_s4DBgmA",
|
|
19
|
+
# 转载页面的静态附言及脚本正文。
|
|
20
|
+
"https://mp.weixin.qq.com/s/Pq-JkEjhoIlLvTMzWKVtOw",
|
|
21
|
+
"https://mp.weixin.qq.com/s/X51M6PfionI6UpX4zsG78Q",
|
|
22
|
+
# 视频号卡片和微信视频 iframe 的纯视频正文。
|
|
23
|
+
"https://mp.weixin.qq.com/s/wAyM7yYKDfTH21-XpS9agQ",
|
|
24
|
+
"https://mp.weixin.qq.com/s/a7P56VOipWNB0UyNaVOZYQ",
|
|
25
|
+
# 付费受限正文。
|
|
26
|
+
"https://mp.weixin.qq.com/s/n2q09-I95zwggfbXKAmbuQ",
|
|
27
|
+
# SVG foreignObject 承载的正文图片。
|
|
28
|
+
"https://mp.weixin.qq.com/s/3ixeC9whuEgpup2G1yvDZg",
|
|
29
|
+
# 纯公众号推荐卡片正文。
|
|
30
|
+
"https://mp.weixin.qq.com/s/PXZuh3fFEi3VXUUI1uBnlw",
|
|
31
|
+
# 无文字正文的音频分享。
|
|
32
|
+
"https://mp.weixin.qq.com/s/y4QxlYs7u0X9TdvZatCfIA",
|
|
33
|
+
# 纯商品广告卡片正文。
|
|
34
|
+
"https://mp.weixin.qq.com/s/CBmoIFPJwV4BkidYGp-l1Q",
|
|
19
35
|
]
|
|
20
36
|
|
|
21
37
|
|
|
@@ -90,8 +106,8 @@ def _assert_result(result: ArticleResult, url: str) -> None:
|
|
|
90
106
|
assert result.article_idx > 0, "article_idx should be positive"
|
|
91
107
|
assert result.article_sn, "article_sn should not be empty"
|
|
92
108
|
assert result.article_title, "article_title should not be empty"
|
|
93
|
-
if result.article_display_type
|
|
94
|
-
assert result.article_markdown, "article_markdown should not be empty for non-video articles"
|
|
109
|
+
if result.article_display_type not in ("video", "audio"):
|
|
110
|
+
assert result.article_markdown, "article_markdown should not be empty for non-video/audio articles"
|
|
95
111
|
assert result.article_publish_time > 0, "article_publish_time should be positive"
|
|
96
112
|
assert result.is_valid
|
|
97
113
|
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""商品卡片占位提示及混合正文的回归测试。"""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from wechat_article_parser import ArticleDisplayType
|
|
6
|
+
from wechat_article_parser.parser import _parse_html
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
_URL = "https://mp.weixin.qq.com/s/CBmoIFPJwV4BkidYGp-l1Q"
|
|
10
|
+
_NOTICE = "这是一篇商品推荐内容。"
|
|
11
|
+
_CARD = '<section><mp-common-cpsad data-pid="104_630468802"></mp-common-cpsad></section>'
|
|
12
|
+
_METADATA = """
|
|
13
|
+
var biz = "MzkwNDQyNzY2NQ==";
|
|
14
|
+
var mid = "2247501127";
|
|
15
|
+
var idx = "2";
|
|
16
|
+
var sn = "bc04bcc079a380252791ace32e8a91ba";
|
|
17
|
+
window.__allowLoadResFromMp = true;
|
|
18
|
+
var hd_head_img = "https://example.com/avatar";
|
|
19
|
+
var nickname = htmlDecode("老鱼讲古");
|
|
20
|
+
var oriCreateTime = '1791711648';
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _html(body: str, metadata: str = _METADATA) -> str:
|
|
25
|
+
return (
|
|
26
|
+
'<meta property="og:title" content="推荐几本好书">'
|
|
27
|
+
f'<div class="rich_media_content" id="js_content">{body}</div>'
|
|
28
|
+
'<script type="text/javascript">var item_show_type = "0";</script>'
|
|
29
|
+
f'<script type="text/javascript">{metadata}</script>'
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@pytest.mark.parametrize("metadata", [_METADATA, ""])
|
|
34
|
+
def test_product_only_body_returns_one_notice_and_requires_metadata(metadata: str) -> None:
|
|
35
|
+
result = _parse_html(_URL, _html(
|
|
36
|
+
_CARD * 4 + '<section><span><br/></span></section>'
|
|
37
|
+
'<p style="display: none;"><mp-style-type data-value="3"></mp-style-type></p>',
|
|
38
|
+
metadata,
|
|
39
|
+
))
|
|
40
|
+
|
|
41
|
+
assert result.article_markdown == _NOTICE
|
|
42
|
+
assert result.article_display_type is ArticleDisplayType.ARTICLE
|
|
43
|
+
assert result.article_item_show_type == 0
|
|
44
|
+
assert result.article_title == "推荐几本好书"
|
|
45
|
+
assert result.images == []
|
|
46
|
+
assert result.is_valid is bool(metadata)
|
|
47
|
+
if metadata:
|
|
48
|
+
assert result.mp_name == "老鱼讲古"
|
|
49
|
+
assert result.article_msg_id == 2247501127
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@pytest.mark.parametrize(("body", "expected", "images"), [
|
|
53
|
+
('<p>推荐书单。</p>' + _CARD + '<p>阅读愉快。</p>',
|
|
54
|
+
'推荐书单。\n\n阅读愉快。', []),
|
|
55
|
+
(_CARD + '<img data-src="https://example.com/book.jpg">',
|
|
56
|
+
'', ['https://example.com/book.jpg']),
|
|
57
|
+
])
|
|
58
|
+
def test_product_cards_do_not_replace_visible_content(
|
|
59
|
+
body: str, expected: str, images: list[str],
|
|
60
|
+
) -> None:
|
|
61
|
+
result = _parse_html(_URL, _html(body))
|
|
62
|
+
|
|
63
|
+
assert result.article_markdown == expected
|
|
64
|
+
assert result.images == images
|
|
65
|
+
assert result.is_valid
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_product_card_outside_body_does_not_fill_empty_body() -> None:
|
|
69
|
+
result = _parse_html(_URL, _html("") + _CARD)
|
|
70
|
+
|
|
71
|
+
assert result.article_markdown == ""
|
|
72
|
+
assert not result.is_valid
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@pytest.mark.parametrize(("embed", "expected"), [
|
|
76
|
+
('<mp-pay-preview-filter></mp-pay-preview-filter>', '这是一篇付费内容,正文需付费后查看。'),
|
|
77
|
+
('<iframe class="wx_video_iframe"></iframe>', '这是一篇纯视频内容。'),
|
|
78
|
+
])
|
|
79
|
+
def test_existing_notices_keep_priority(embed: str, expected: str) -> None:
|
|
80
|
+
result = _parse_html(_URL, _html(_CARD + embed))
|
|
81
|
+
|
|
82
|
+
assert result.article_markdown == expected
|
|
83
|
+
assert result.is_valid
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""公众号推荐卡片转换及正文顺序的离线回归测试。"""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from wechat_article_parser import ArticleDisplayType
|
|
8
|
+
from wechat_article_parser.parser import _parse_html
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
_URL = "https://mp.weixin.qq.com/s/PXZuh3fFEi3VXUUI1uBnlw"
|
|
12
|
+
_METADATA = """
|
|
13
|
+
var item_show_type = "0";
|
|
14
|
+
var biz = "MzUyNTgyNjUzMw==";
|
|
15
|
+
var mid = "2247923648";
|
|
16
|
+
var idx = "2";
|
|
17
|
+
var sn = "d744c25ecdf2ac7f361d880dffb0a2ca";
|
|
18
|
+
window.__allowLoadResFromMp = true;
|
|
19
|
+
var hd_head_img = "https://example.com/publisher-avatar";
|
|
20
|
+
var nickname = htmlDecode("药融圈");
|
|
21
|
+
var oriCreateTime = '1791441362';
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _html(body: str) -> str:
|
|
26
|
+
return (
|
|
27
|
+
'<meta property="og:title" content="公众号推荐文章">'
|
|
28
|
+
f'<div class="rich_media_content">{body}</div>'
|
|
29
|
+
'<div id="js_fullscreen_layout_padding"></div>'
|
|
30
|
+
f'<script type="text/javascript">{_METADATA}</script>'
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_profile_only_article_preserves_names_signatures_and_metadata() -> None:
|
|
35
|
+
body = (
|
|
36
|
+
'<section><span><mp-common-profile data-nickname="药融圈" '
|
|
37
|
+
'data-signature="中国医药信息和人脉的链接平台" '
|
|
38
|
+
'data-headimg="https://example.com/card-avatar"></mp-common-profile></span></section>'
|
|
39
|
+
'<section><mp-common-profile data-nickname="药通社" '
|
|
40
|
+
'data-signature="政策、资讯、产业、技术"></mp-common-profile></section>'
|
|
41
|
+
)
|
|
42
|
+
result = _parse_html(
|
|
43
|
+
_URL,
|
|
44
|
+
_html(body)
|
|
45
|
+
+ '<mp-common-profile data-nickname="页外推荐" data-signature="不属于正文"></mp-common-profile>',
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
assert result.article_markdown == (
|
|
49
|
+
'**药融圈**:中国医药信息和人脉的链接平台\n\n'
|
|
50
|
+
'**药通社**:政策、资讯、产业、技术'
|
|
51
|
+
)
|
|
52
|
+
assert result.images == []
|
|
53
|
+
assert result.is_valid
|
|
54
|
+
assert result.mp_id == 3525826533
|
|
55
|
+
assert result.mp_name == "药融圈"
|
|
56
|
+
assert result.article_msg_id == 2247923648
|
|
57
|
+
assert result.article_idx == 2
|
|
58
|
+
assert result.article_sn == "d744c25ecdf2ac7f361d880dffb0a2ca"
|
|
59
|
+
assert result.article_publish_time == 1791441362
|
|
60
|
+
assert result.article_display_type is ArticleDisplayType.ARTICLE
|
|
61
|
+
assert result.article_item_show_type == 0
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def test_profile_mixed_with_text_and_images_preserves_order_without_duplicate_card() -> None:
|
|
65
|
+
result = _parse_html(_URL, _html(
|
|
66
|
+
'<p>推荐以下公众号。</p><mp-common-profile data-nickname="药融圈" '
|
|
67
|
+
'data-signature="医药信息平台">'
|
|
68
|
+
'<img src="https://example.com/card-avatar"><p>药融圈</p><p>医药信息平台</p>'
|
|
69
|
+
'</mp-common-profile>'
|
|
70
|
+
'<p><img data-src="https://mmbiz.qpic.cn/mmbiz_jpg/body/0?wx_fmt=jpeg"></p>'
|
|
71
|
+
'<p>正文结尾。</p>'
|
|
72
|
+
))
|
|
73
|
+
|
|
74
|
+
assert result.article_markdown == (
|
|
75
|
+
'推荐以下公众号。\n\n**药融圈**:医药信息平台\n\n'
|
|
76
|
+
'\n\n正文结尾。'
|
|
77
|
+
)
|
|
78
|
+
assert result.images == ["https://mmbiz.qpic.cn/mmbiz_jpg/body/640"]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@pytest.mark.parametrize(("attributes", "expected"), [
|
|
82
|
+
('data-nickname=" 药融圈 "', '**药融圈**'),
|
|
83
|
+
('data-signature=" 医药信息平台 "', '医药信息平台'),
|
|
84
|
+
('data-nickname=" " data-signature=" "', ''),
|
|
85
|
+
('', ''),
|
|
86
|
+
])
|
|
87
|
+
def test_profile_missing_fields_keeps_available_text_without_empty_placeholders(
|
|
88
|
+
attributes: str, expected: str,
|
|
89
|
+
) -> None:
|
|
90
|
+
result = _parse_html(_URL, _html(f'<mp-common-profile {attributes}></mp-common-profile>'))
|
|
91
|
+
|
|
92
|
+
assert result.article_markdown == expected
|
|
93
|
+
assert result.is_valid is bool(expected)
|
|
94
|
+
assert result.images == []
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def test_profile_attributes_keep_entities_and_escape_markdown_characters() -> None:
|
|
98
|
+
result = _parse_html(_URL, _html(
|
|
99
|
+
'<mp-common-profile data-nickname="药品&研发*圈*" '
|
|
100
|
+
'data-signature="A & B,进度 100%,保留 %20 和 <tag>。">'
|
|
101
|
+
'</mp-common-profile>'
|
|
102
|
+
))
|
|
103
|
+
|
|
104
|
+
assert result.article_markdown == r'**药品&研发\*圈\***:A & B,进度 100%,保留 %20 和 <tag>。'
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_repost_script_body_uses_same_profile_conversion() -> None:
|
|
108
|
+
body = (
|
|
109
|
+
'<mp-common-profile data-nickname="药通社" '
|
|
110
|
+
'data-signature="政策、资讯、产业、技术"></mp-common-profile>'
|
|
111
|
+
)
|
|
112
|
+
html = (
|
|
113
|
+
'<meta property="og:title" content="转载推荐文章">'
|
|
114
|
+
'<div class="original_page"><span id="js_share_source" '
|
|
115
|
+
'data-url="https://example.com/original"></span></div>'
|
|
116
|
+
f'<script type="text/javascript">{_METADATA}</script>'
|
|
117
|
+
'<script type="text/javascript">window.cgiDataNew = '
|
|
118
|
+
f'{{content_noencode: {json.dumps(body, ensure_ascii=False)}}};</script>'
|
|
119
|
+
)
|
|
120
|
+
result = _parse_html(_URL, html)
|
|
121
|
+
|
|
122
|
+
assert result.is_valid
|
|
123
|
+
assert result.article_markdown == (
|
|
124
|
+
'**药通社**:政策、资讯、产业、技术\n\n'
|
|
125
|
+
'[查看原文](https://example.com/original)'
|
|
126
|
+
)
|
|
127
|
+
assert result.images == []
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""转载页面的静态附言、脚本正文及元数据回归测试。"""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from wechat_article_parser import ArticleDisplayType
|
|
6
|
+
from wechat_article_parser.parser import _parse_html
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
_URL = "https://mp.weixin.qq.com/s/Pq-JkEjhoIlLvTMzWKVtOw"
|
|
10
|
+
_SOURCE = "https://mp.weixin.qq.com/s?__biz=original&mid=456&idx=1&sn=source"
|
|
11
|
+
_BODY = """<section>
|
|
12
|
+
<p>转载正文第一段,保留 100% 和 %20。</p>
|
|
13
|
+
<img data-src="https://mmbiz.qpic.cn/mmbiz_png/first/0?wx_fmt=png&from=appmsg">
|
|
14
|
+
<p>正文里的 <tag>、O'Reilly 和反斜杠 \\。</p>
|
|
15
|
+
<pre><code>if ready:
|
|
16
|
+
print("<tag>")
|
|
17
|
+
</code></pre>
|
|
18
|
+
<img data-src="https://mmbiz.qpic.cn/mmbiz_png/second/0?wx_fmt=png">
|
|
19
|
+
<p>转载正文最后一段。</p>
|
|
20
|
+
</section>"""
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _html(data: str = "", notice: str = "分享一篇文章。") -> str:
|
|
24
|
+
# 对应真实页面:正文容器尚未填充,正文和元数据分散在多个脚本。
|
|
25
|
+
return f"""
|
|
26
|
+
<meta property="og:title" content="转载文章标题">
|
|
27
|
+
<div class="original_page">
|
|
28
|
+
<p id="js_share_notice">{notice}</p>
|
|
29
|
+
<div id="js_share_headimg"><img src="https://example.com/avatar"></div>
|
|
30
|
+
<div id="js_content"></div>
|
|
31
|
+
<span id="js_share_source" data-url="{_SOURCE}">阅读全文</span>
|
|
32
|
+
</div>
|
|
33
|
+
<div id="js_fullscreen_layout_padding"></div>
|
|
34
|
+
<script type="text/javascript">
|
|
35
|
+
var biz = "MzAxNjI5NzI4MA==" || "";
|
|
36
|
+
var mid = "2247577355" || "";
|
|
37
|
+
var idx = "4" || "";
|
|
38
|
+
var sn = "signature" || "";
|
|
39
|
+
window.__allowLoadResFromMp = true;
|
|
40
|
+
</script>
|
|
41
|
+
<script type="text/javascript">
|
|
42
|
+
var hd_head_img = "https://example.com/publisher-avatar";
|
|
43
|
+
var nickname = htmlDecode("新浪医药");
|
|
44
|
+
var item_show_type = "0";
|
|
45
|
+
var user_name = "gh_3675b3bc7097";
|
|
46
|
+
</script>
|
|
47
|
+
<script type="text/javascript">
|
|
48
|
+
function setTime() {{ var oriCreateTime = '1791459000'; }}
|
|
49
|
+
</script>
|
|
50
|
+
<script type="text/javascript">{data}</script>
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _js_body(body: str, quote: str = "'") -> str:
|
|
55
|
+
escaped = body.translate(str.maketrans({
|
|
56
|
+
"\\": "\\\\", "'": "\\'", '"': r"\x22", "<": r"\x3c",
|
|
57
|
+
">": r"\x3e", "&": r"\x26", "\n": r"\n",
|
|
58
|
+
}))
|
|
59
|
+
return quote + escaped + quote
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@pytest.mark.parametrize("field", ["content_noencode", "content"])
|
|
63
|
+
@pytest.mark.parametrize("quote", ["'", '"'])
|
|
64
|
+
def test_static_repost_reads_script_body_and_preserves_metadata(
|
|
65
|
+
field: str, quote: str,
|
|
66
|
+
) -> None:
|
|
67
|
+
data = f"""
|
|
68
|
+
window.cgiDataNew = {{
|
|
69
|
+
reference: {{content_noencode: '嵌套的错误正文', content: '嵌套的错误正文'}},
|
|
70
|
+
{quote}{field}{quote}: {_js_body(_BODY, quote)},
|
|
71
|
+
picture_page_info_list: [{{cdn_url: 'https://example.com/wrong-image'}}]
|
|
72
|
+
}};
|
|
73
|
+
"""
|
|
74
|
+
result = _parse_html(_URL, _html(data))
|
|
75
|
+
|
|
76
|
+
assert result.is_valid
|
|
77
|
+
assert result.article_display_type is ArticleDisplayType.ARTICLE
|
|
78
|
+
assert result.article_item_show_type == 0
|
|
79
|
+
assert result.mp_id == 3016297280
|
|
80
|
+
assert result.mp_name == "新浪医药"
|
|
81
|
+
assert result.mp_user_name == "gh_3675b3bc7097"
|
|
82
|
+
assert result.article_msg_id == 2247577355
|
|
83
|
+
assert result.article_idx == 4
|
|
84
|
+
assert result.article_sn == "signature"
|
|
85
|
+
assert result.article_publish_time == 1791459000
|
|
86
|
+
assert result.images == [
|
|
87
|
+
"https://mmbiz.qpic.cn/mmbiz_png/first/640",
|
|
88
|
+
"https://mmbiz.qpic.cn/mmbiz_png/second/640",
|
|
89
|
+
]
|
|
90
|
+
markdown = result.article_markdown
|
|
91
|
+
assert markdown.startswith("分享一篇文章。")
|
|
92
|
+
assert "转载正文第一段,保留 100% 和 %20。" in markdown
|
|
93
|
+
assert "正文里的 <tag>、O'Reilly 和反斜杠" in markdown
|
|
94
|
+
assert '```\nif ready:\n print("<tag>")\n```' in markdown
|
|
95
|
+
assert "转载正文最后一段。" in markdown
|
|
96
|
+
assert markdown.endswith(f"[查看原文]({_SOURCE})")
|
|
97
|
+
assert markdown.count("![") == 2
|
|
98
|
+
assert "![]()" not in markdown
|
|
99
|
+
assert "嵌套的错误正文" not in markdown
|
|
100
|
+
assert "avatar" not in markdown
|
|
101
|
+
assert "wrong-image" not in markdown
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def test_static_notice_without_script_body_still_parses() -> None:
|
|
105
|
+
result = _parse_html(_URL, _html())
|
|
106
|
+
|
|
107
|
+
assert result.is_valid
|
|
108
|
+
assert result.article_markdown == f"分享一篇文章。\n\n[查看原文]({_SOURCE})"
|
|
109
|
+
assert result.images == []
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def test_legacy_script_notice_keeps_body_and_newlines() -> None:
|
|
113
|
+
notice = r'<script>notice.innerHTML = "旧版附言\x0a第二行";</script>'
|
|
114
|
+
result = _parse_html(_URL, _html(notice=notice))
|
|
115
|
+
|
|
116
|
+
assert result.is_valid
|
|
117
|
+
assert result.article_markdown == f"旧版附言 \n第二行\n\n[查看原文]({_SOURCE})"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_empty_unencoded_body_falls_back_to_content() -> None:
|
|
121
|
+
data = "window.cgiDataNew = {content_noencode: '', content: '<p>备用正文</p>'};"
|
|
122
|
+
result = _parse_html(_URL, _html(data, notice=""))
|
|
123
|
+
|
|
124
|
+
assert result.is_valid
|
|
125
|
+
assert result.article_markdown == f"备用正文\n\n[查看原文]({_SOURCE})"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def test_repost_ignores_body_in_comments_strings_and_other_objects() -> None:
|
|
129
|
+
data = """
|
|
130
|
+
// window.cgiDataNew = {content_noencode: '注释中的错误正文'};
|
|
131
|
+
var example = "window.cgiDataNew = {content: '字符串中的错误正文'};";
|
|
132
|
+
other.window.cgiDataNew = {content_noencode: '其他对象中的错误正文'};
|
|
133
|
+
window.cgiDataNew = {
|
|
134
|
+
reference: {content_noencode: '嵌套的错误正文'},
|
|
135
|
+
content_noencode: loadUntrustedContent()
|
|
136
|
+
};
|
|
137
|
+
"""
|
|
138
|
+
result = _parse_html(_URL, _html(data))
|
|
139
|
+
|
|
140
|
+
assert result.article_markdown == f"分享一篇文章。\n\n[查看原文]({_SOURCE})"
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""SVG 容器承载正文图片的离线回归测试。"""
|
|
2
|
+
|
|
3
|
+
from wechat_article_parser import ArticleDisplayType, ArticleResult
|
|
4
|
+
from wechat_article_parser.parser import _parse_html
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
_URL = "https://mp.weixin.qq.com/s/3ixeC9whuEgpup2G1yvDZg"
|
|
8
|
+
_METADATA = """
|
|
9
|
+
var item_show_type = "0";
|
|
10
|
+
var biz = "MjM5MjY2MTc5NQ==";
|
|
11
|
+
var mid = "2652403355";
|
|
12
|
+
var idx = "1";
|
|
13
|
+
var sn = "9c582b2554db23b592ed758e95680f0a";
|
|
14
|
+
window.__allowLoadResFromMp = true;
|
|
15
|
+
var hd_head_img = "https://example.com/avatar";
|
|
16
|
+
var nickname = htmlDecode("CFMOTO春风动力");
|
|
17
|
+
var oriCreateTime = '1791454818';
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _parse(body: str) -> ArticleResult:
|
|
22
|
+
return _parse_html(
|
|
23
|
+
_URL,
|
|
24
|
+
'<meta property="og:title" content="SVG 图片正文">'
|
|
25
|
+
f'<div class="rich_media_content">{body}</div>'
|
|
26
|
+
'<div id="js_fullscreen_layout_padding"></div>'
|
|
27
|
+
f'<script type="text/javascript">{_METADATA}</script>',
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_foreignobject_images_keep_body_order_and_rich_text_metadata() -> None:
|
|
32
|
+
# 对应真实页面:整个正文由三个 SVG 中的长图组成,没有背景图。
|
|
33
|
+
body = "".join(
|
|
34
|
+
'<svg viewBox="0 0 1080 2362"><foreignObject>'
|
|
35
|
+
f'<span><img data-src="https://mmbiz.qpic.cn/mmbiz_jpg/body-{i}/0?wx_fmt=jpeg"></span>'
|
|
36
|
+
'</foreignObject></svg>'
|
|
37
|
+
for i in range(3)
|
|
38
|
+
)
|
|
39
|
+
result = _parse(body)
|
|
40
|
+
urls = [f"https://mmbiz.qpic.cn/mmbiz_jpg/body-{i}/640" for i in range(3)]
|
|
41
|
+
|
|
42
|
+
assert result.article_markdown == "".join(f"" for url in urls)
|
|
43
|
+
assert result.images == urls
|
|
44
|
+
assert result.is_valid
|
|
45
|
+
assert result.mp_id == 2392661795
|
|
46
|
+
assert result.mp_name == "CFMOTO春风动力"
|
|
47
|
+
assert result.article_msg_id == 2652403355
|
|
48
|
+
assert result.article_idx == 1
|
|
49
|
+
assert result.article_sn == "9c582b2554db23b592ed758e95680f0a"
|
|
50
|
+
assert result.article_publish_time == 1791454818
|
|
51
|
+
assert result.article_display_type is ArticleDisplayType.ARTICLE
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_nested_svg_images_and_backgrounds_keep_source_order() -> None:
|
|
55
|
+
result = _parse(
|
|
56
|
+
'<svg style=\'background-image: url("https://example.com/first.jpg")\'>'
|
|
57
|
+
'<foreignObject><img data-src="https://example.com/second.jpg" alt="第二张">'
|
|
58
|
+
'<svg style=\'background-image: url("https://example.com/third.jpg")\'>'
|
|
59
|
+
'<foreignObject><img src="https://example.com/fourth.jpg"></foreignObject>'
|
|
60
|
+
'</svg></foreignObject></svg>'
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
assert result.article_markdown == "".join([
|
|
64
|
+
"",
|
|
65
|
+
"",
|
|
66
|
+
"",
|
|
67
|
+
"",
|
|
68
|
+
])
|
|
69
|
+
assert result.images == [
|
|
70
|
+
"https://example.com/first.jpg",
|
|
71
|
+
"https://example.com/second.jpg",
|
|
72
|
+
"https://example.com/third.jpg",
|
|
73
|
+
"https://example.com/fourth.jpg",
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def test_repeated_svg_images_remain_in_body_but_image_list_is_unique() -> None:
|
|
78
|
+
result = _parse(
|
|
79
|
+
'<img src="https://mmbiz.qpic.cn/mmbiz_jpg/body/0">'
|
|
80
|
+
'<svg><foreignObject><svg><foreignObject>'
|
|
81
|
+
'<img data-src="https://mmbiz.qpic.cn/mmbiz_jpg/body/640?wx_fmt=jpeg">'
|
|
82
|
+
'</foreignObject></svg></foreignObject></svg>'
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
url = "https://mmbiz.qpic.cn/mmbiz_jpg/body/640"
|
|
86
|
+
assert result.images == [url]
|
|
87
|
+
assert result.article_markdown.count(f"") == 2
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_decorative_svg_and_invalid_images_do_not_pollute_body() -> None:
|
|
91
|
+
result = _parse(
|
|
92
|
+
'<p>正文</p><svg><title>装饰分隔线</title><path d="M0 0 L100 0"></path></svg>'
|
|
93
|
+
'<svg><foreignObject><img><img src="data:image/png;base64,AA==">'
|
|
94
|
+
'</foreignObject></svg>'
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
assert result.article_markdown == "正文"
|
|
98
|
+
assert result.images == []
|
|
@@ -49,6 +49,7 @@ def _html(body: str = "", script: str = "", title: str = "原标题") -> str:
|
|
|
49
49
|
("window.item_show_type = '8' || '';", 8),
|
|
50
50
|
("window.item_show_type = 10;", 10),
|
|
51
51
|
("var /* 注释 */ item_show_type = 5;", 5),
|
|
52
|
+
("window.item_show_type = '7';", 7),
|
|
52
53
|
("(function() { window.item_show_type = '10'; })();", 10),
|
|
53
54
|
("window.item_show_type = '999';", 999),
|
|
54
55
|
("window.item_show_type = '';", None),
|
|
@@ -120,6 +121,7 @@ def test_type_zero_uses_html_to_distinguish_rich_text_and_repost(
|
|
|
120
121
|
("item_type", "data", "expected", "display_type"),
|
|
121
122
|
[
|
|
122
123
|
(5, _PLAIN_DATA, "文字正文", ArticleDisplayType.VIDEO),
|
|
124
|
+
(7, _PLAIN_DATA, "文字正文", ArticleDisplayType.AUDIO),
|
|
123
125
|
(8, _SWIPER_DATA, "轮播正文", ArticleDisplayType.IMAGE),
|
|
124
126
|
(10, _PLAIN_DATA, "文字正文", ArticleDisplayType.TEXT),
|
|
125
127
|
(10, _FULLSCREEN_DATA, "全屏正文", ArticleDisplayType.TEXT),
|
|
@@ -153,7 +155,7 @@ def test_missing_unknown_or_invalid_type_falls_back_to_html(declaration: str) ->
|
|
|
153
155
|
assert result.article_display_type is ArticleDisplayType.UNKNOWN
|
|
154
156
|
|
|
155
157
|
|
|
156
|
-
@pytest.mark.parametrize("item_type", [5, 8, 10])
|
|
158
|
+
@pytest.mark.parametrize("item_type", [5, 7, 8, 10])
|
|
157
159
|
def test_known_type_without_matching_data_falls_back_to_html(item_type: int) -> None:
|
|
158
160
|
result = parser._parse_html(
|
|
159
161
|
_URL,
|
|
@@ -215,9 +217,10 @@ def test_unrecognized_content_still_returns_metadata() -> None:
|
|
|
215
217
|
[
|
|
216
218
|
("", ArticleDisplayType.UNKNOWN),
|
|
217
219
|
("window.item_show_type = '5';", ArticleDisplayType.VIDEO),
|
|
220
|
+
("window.item_show_type = '7';", ArticleDisplayType.AUDIO),
|
|
218
221
|
],
|
|
219
222
|
)
|
|
220
|
-
def
|
|
223
|
+
def test_video_and_audio_without_description_keep_type_and_metadata(
|
|
221
224
|
declaration: str, expected_type: ArticleDisplayType,
|
|
222
225
|
) -> None:
|
|
223
226
|
metadata = """
|
|
@@ -247,4 +250,6 @@ def test_video_without_description_keeps_type_and_metadata(
|
|
|
247
250
|
assert result.mp_id == 1
|
|
248
251
|
assert result.mp_name == "测试公众号"
|
|
249
252
|
assert result.article_markdown == ""
|
|
250
|
-
assert result.is_valid is (
|
|
253
|
+
assert result.is_valid is (
|
|
254
|
+
expected_type in (ArticleDisplayType.VIDEO, ArticleDisplayType.AUDIO)
|
|
255
|
+
)
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""纯视频号卡片正文的离线回归测试。"""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from wechat_article_parser import ArticleDisplayType
|
|
6
|
+
from wechat_article_parser.parser import _parse_html
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
_URL = "https://mp.weixin.qq.com/s/wAyM7yYKDfTH21-XpS9agQ"
|
|
10
|
+
_CARD = (
|
|
11
|
+
'<mp-common-videosnap data-type="video" data-id="video-id" '
|
|
12
|
+
'data-desc="视频描述" data-url="https://example.com/cover.jpg">'
|
|
13
|
+
'</mp-common-videosnap>'
|
|
14
|
+
)
|
|
15
|
+
_IFRAME = (
|
|
16
|
+
'<iframe class="video_iframe rich_pages wx_video_iframe" data-mpvid="wxv_123" '
|
|
17
|
+
'data-src="https://mp.weixin.qq.com/mp/readtemplate?t=pages/video_player_tmpl'
|
|
18
|
+
'&action=mpvideo&vid=wxv_123"></iframe>'
|
|
19
|
+
)
|
|
20
|
+
_METADATA = """
|
|
21
|
+
var biz = "MjM5NTAwMzE0MA==";
|
|
22
|
+
var mid = "2908209265";
|
|
23
|
+
var idx = "4";
|
|
24
|
+
var sn = "signature";
|
|
25
|
+
window.__allowLoadResFromMp = true;
|
|
26
|
+
var hd_head_img = "https://example.com/avatar";
|
|
27
|
+
var nickname = htmlDecode("卡车之家");
|
|
28
|
+
var oriCreateTime = '1791462000';
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _html(body: str, metadata: str = _METADATA) -> str:
|
|
33
|
+
return (
|
|
34
|
+
'<meta property="og:title" content="后双胎大轻客,江淮星锐EV6">'
|
|
35
|
+
f'<div class="rich_media_content">{body}</div>'
|
|
36
|
+
'<script type="text/javascript">var item_show_type = "0";</script>'
|
|
37
|
+
f'<script type="text/javascript">{metadata}</script>'
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@pytest.mark.parametrize("metadata", [_METADATA, ""])
|
|
42
|
+
@pytest.mark.parametrize("video", [_CARD, _IFRAME])
|
|
43
|
+
def test_video_only_body_gets_one_sentence_and_still_requires_metadata(
|
|
44
|
+
metadata: str, video: str,
|
|
45
|
+
) -> None:
|
|
46
|
+
result = _parse_html(
|
|
47
|
+
_URL,
|
|
48
|
+
_html(f'<section>{video}</section><p><br></p><mp-style-type></mp-style-type>', metadata),
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
assert result.article_markdown == "这是一篇纯视频内容。"
|
|
52
|
+
assert result.article_display_type is ArticleDisplayType.ARTICLE
|
|
53
|
+
assert result.article_item_show_type == 0
|
|
54
|
+
assert result.images == []
|
|
55
|
+
assert result.is_valid is bool(metadata)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@pytest.mark.parametrize("body", [
|
|
59
|
+
'<p>实际文字正文</p>',
|
|
60
|
+
'<img data-src="https://example.com/body.jpg">',
|
|
61
|
+
])
|
|
62
|
+
@pytest.mark.parametrize("video", [_CARD, _IFRAME])
|
|
63
|
+
def test_video_with_other_content_keeps_existing_markdown(body: str, video: str) -> None:
|
|
64
|
+
result = _parse_html(_URL, _html(body + video))
|
|
65
|
+
|
|
66
|
+
assert result.is_valid
|
|
67
|
+
assert "这是一篇纯视频内容。" not in result.article_markdown
|
|
68
|
+
assert "cover.jpg" not in result.article_markdown
|
|
69
|
+
if "实际文字正文" in body:
|
|
70
|
+
assert result.article_markdown == "实际文字正文"
|
|
71
|
+
else:
|
|
72
|
+
assert result.images == ["https://example.com/body.jpg"]
|
|
73
|
+
assert result.article_markdown == ""
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@pytest.mark.parametrize("body", [
|
|
77
|
+
"",
|
|
78
|
+
_CARD.replace('data-type="video"', 'data-type="image"'),
|
|
79
|
+
'<iframe src="https://example.com/unrelated"></iframe>',
|
|
80
|
+
])
|
|
81
|
+
def test_empty_or_non_video_body_does_not_get_video_notice(body: str) -> None:
|
|
82
|
+
result = _parse_html(_URL, _html(body))
|
|
83
|
+
|
|
84
|
+
assert result.article_markdown == ""
|
|
85
|
+
assert not result.is_valid
|
|
File without changes
|
|
File without changes
|
{wechat_article_parser-0.0.9 → wechat_article_parser-0.0.11}/src/wechat_article_parser/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|