wechat-article-parser 0.0.10__tar.gz → 0.0.12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/PKG-INFO +15 -7
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/README.md +14 -6
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/pyproject.toml +1 -1
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/src/wechat_article_parser/models.py +5 -2
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/src/wechat_article_parser/parser.py +24 -0
- wechat_article_parser-0.0.12/tests/test_miniprogram_content.py +108 -0
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/tests/test_models.py +10 -6
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/tests/test_parser.py +8 -2
- wechat_article_parser-0.0.12/tests/test_product_content.py +83 -0
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/tests/test_type_routing.py +8 -3
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/.gitignore +0 -0
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/LICENSE +0 -0
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/src/wechat_article_parser/__init__.py +0 -0
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/tests/conftest.py +0 -0
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/tests/test_markdown.py +0 -0
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/tests/test_paid_content.py +0 -0
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/tests/test_profile_content.py +0 -0
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/tests/test_repost_content.py +0 -0
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/tests/test_svg_content.py +0 -0
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/tests/test_swiper_content.py +0 -0
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/tests/test_user_name.py +0 -0
- {wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/tests/test_video_content.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: wechat-article-parser
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.12
|
|
4
4
|
Summary: WeChat MP article parser - extract metadata and content from WeChat public account articles
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -23,10 +23,11 @@ Description-Content-Type: text/markdown
|
|
|
23
23
|
- 转载式文章
|
|
24
24
|
- 纯文本文章
|
|
25
25
|
- 视频分享类文章
|
|
26
|
+
- 音频分享类文章
|
|
26
27
|
- 小红书风格图片轮播文章
|
|
27
28
|
- 全屏布局短文本文章
|
|
28
29
|
|
|
29
|
-
解析时优先读取页面全局的 `item_show_type`:`0` 对应图文、`5`
|
|
30
|
+
解析时优先读取页面全局的 `item_show_type`:`0` 对应图文、`5` 对应视频分享、`7` 对应音频分享、
|
|
30
31
|
`8` 对应图片轮播、`10` 对应文字消息,再通过 HTML 或脚本数据区分具体模板。
|
|
31
32
|
类型字段缺失、值未知或对应分支未提取到正文时,会按原有 HTML 特征匹配顺序兜底。
|
|
32
33
|
|
|
@@ -35,6 +36,12 @@ Description-Content-Type: text/markdown
|
|
|
35
36
|
并保留公开元数据;已有的可见预览正文仍正常提取。
|
|
36
37
|
图文中的公众号推荐卡片按原文顺序转换为 `**公众号名称**:简介`,每张卡片一段;
|
|
37
38
|
名称或简介缺失时,仅输出已有字段。
|
|
39
|
+
图文正文仅包含商品广告卡片(`mp-common-cpsad`)时,Markdown 返回“这是一篇商品推荐内容。”;
|
|
40
|
+
已有的文字和图片仍正常提取。关键元数据完整时占位结果也有效,不表示已获取商品详情。
|
|
41
|
+
小程序卡片(`mp-common-miniprogram`)按原文顺序转换为加粗标题、`小程序:名称`及
|
|
42
|
+
“请在微信中打开小程序查看内容。”,标题或名称缺失时省略对应段落。
|
|
43
|
+
小程序卡片的封面和头像不输出到 Markdown,也不加入 `images`;不生成小程序内部路径链接。
|
|
44
|
+
关键元数据完整时卡片结果也有效,不表示已获取小程序内的文档或其他内容。
|
|
38
45
|
|
|
39
46
|
## 安装
|
|
40
47
|
|
|
@@ -119,15 +126,15 @@ result = parse(
|
|
|
119
126
|
| `article_title` | `str` | 文章标题 |
|
|
120
127
|
| `article_cover_image` | `str` | 文章封面图链接 |
|
|
121
128
|
| `article_description` | `str` | 文章摘要 |
|
|
122
|
-
| `article_markdown` | `str` | 文章正文的 Markdown
|
|
129
|
+
| `article_markdown` | `str` | 文章正文的 Markdown 内容;视频或音频分享没有文字附言时可为空 |
|
|
123
130
|
| `article_publish_time` | `int` | 发布时间(Unix 时间戳) |
|
|
124
|
-
| `article_display_type` | `ArticleDisplayType` | 文章展示类型:`article`、`video`、`image`、`text` 或 `unknown` |
|
|
125
|
-
| `article_item_show_type` | `int \| None` | 微信页面中的原始展示类型:`0` 图文、`5` 视频、`8` 图片轮播、`10` 文字;其他已提取的数值原样保留,缺失或无法解析时为 `None` |
|
|
131
|
+
| `article_display_type` | `ArticleDisplayType` | 文章展示类型:`article`、`video`、`audio`、`image`、`text` 或 `unknown` |
|
|
132
|
+
| `article_item_show_type` | `int \| None` | 微信页面中的原始展示类型:`0` 图文、`5` 视频、`7` 音频、`8` 图片轮播、`10` 文字;其他已提取的数值原样保留,缺失或无法解析时为 `None` |
|
|
126
133
|
| `raw_html` | `str` | 原始 HTML 网页源代码(仅在 `include_raw_html=True` 时填充,否则为空字符串) |
|
|
127
134
|
| `images` | `list[str]` | 文章中提取的所有图片链接 |
|
|
128
135
|
| `is_valid` | `bool` | 关键字段是否全部解析成功(属性) |
|
|
129
136
|
|
|
130
|
-
`is_valid` 为 `True` 的条件:`mp_id`、`mp_name`、`article_id`、`article_msg_id`、`article_idx`、`article_sn`、`article_title`、`article_publish_time` 均不为空/零,且 `article_markdown` 非空或 `article_display_type
|
|
137
|
+
`is_valid` 为 `True` 的条件:`mp_id`、`mp_name`、`article_id`、`article_msg_id`、`article_idx`、`article_sn`、`article_title`、`article_publish_time` 均不为空/零,且 `article_markdown` 非空或 `article_display_type in ("video", "audio")`。视频和音频分享允许没有文字正文,仍须满足其他关键字段要求;有效性不表示已获取视频或音频文件。
|
|
131
138
|
|
|
132
139
|
付费受限页面返回提示且关键元数据完整时,`is_valid` 也为 `True`,这不表示已获取付费正文。
|
|
133
140
|
|
|
@@ -139,6 +146,7 @@ result = parse(
|
|
|
139
146
|
|---|---|---|
|
|
140
147
|
| `ArticleDisplayType.ARTICLE` | `article` | 图文,包括转载式 |
|
|
141
148
|
| `ArticleDisplayType.VIDEO` | `video` | 视频分享 |
|
|
149
|
+
| `ArticleDisplayType.AUDIO` | `audio` | 音频分享 |
|
|
142
150
|
| `ArticleDisplayType.IMAGE` | `image` | 图片轮播 |
|
|
143
151
|
| `ArticleDisplayType.TEXT` | `text` | 文字消息 |
|
|
144
152
|
| `ArticleDisplayType.UNKNOWN` | `unknown` | 页面类型缺失或未支持 |
|
|
@@ -212,7 +220,7 @@ except httpx.HTTPStatusError as e:
|
|
|
212
220
|
|
|
213
221
|
### 解析不完整
|
|
214
222
|
|
|
215
|
-
|
|
223
|
+
缺少标题、发布时间等关键字段,或非视频/音频分享缺少正文时,解析结果的 `is_valid` 会返回 `False`。封面图、摘要等非必填字段为空不影响有效性。建议在业务逻辑中检查:
|
|
216
224
|
|
|
217
225
|
```python
|
|
218
226
|
result = parse("https://mp.weixin.qq.com/s/xxxxx")
|
|
@@ -8,10 +8,11 @@
|
|
|
8
8
|
- 转载式文章
|
|
9
9
|
- 纯文本文章
|
|
10
10
|
- 视频分享类文章
|
|
11
|
+
- 音频分享类文章
|
|
11
12
|
- 小红书风格图片轮播文章
|
|
12
13
|
- 全屏布局短文本文章
|
|
13
14
|
|
|
14
|
-
解析时优先读取页面全局的 `item_show_type`:`0` 对应图文、`5`
|
|
15
|
+
解析时优先读取页面全局的 `item_show_type`:`0` 对应图文、`5` 对应视频分享、`7` 对应音频分享、
|
|
15
16
|
`8` 对应图片轮播、`10` 对应文字消息,再通过 HTML 或脚本数据区分具体模板。
|
|
16
17
|
类型字段缺失、值未知或对应分支未提取到正文时,会按原有 HTML 特征匹配顺序兜底。
|
|
17
18
|
|
|
@@ -20,6 +21,12 @@
|
|
|
20
21
|
并保留公开元数据;已有的可见预览正文仍正常提取。
|
|
21
22
|
图文中的公众号推荐卡片按原文顺序转换为 `**公众号名称**:简介`,每张卡片一段;
|
|
22
23
|
名称或简介缺失时,仅输出已有字段。
|
|
24
|
+
图文正文仅包含商品广告卡片(`mp-common-cpsad`)时,Markdown 返回“这是一篇商品推荐内容。”;
|
|
25
|
+
已有的文字和图片仍正常提取。关键元数据完整时占位结果也有效,不表示已获取商品详情。
|
|
26
|
+
小程序卡片(`mp-common-miniprogram`)按原文顺序转换为加粗标题、`小程序:名称`及
|
|
27
|
+
“请在微信中打开小程序查看内容。”,标题或名称缺失时省略对应段落。
|
|
28
|
+
小程序卡片的封面和头像不输出到 Markdown,也不加入 `images`;不生成小程序内部路径链接。
|
|
29
|
+
关键元数据完整时卡片结果也有效,不表示已获取小程序内的文档或其他内容。
|
|
23
30
|
|
|
24
31
|
## 安装
|
|
25
32
|
|
|
@@ -104,15 +111,15 @@ result = parse(
|
|
|
104
111
|
| `article_title` | `str` | 文章标题 |
|
|
105
112
|
| `article_cover_image` | `str` | 文章封面图链接 |
|
|
106
113
|
| `article_description` | `str` | 文章摘要 |
|
|
107
|
-
| `article_markdown` | `str` | 文章正文的 Markdown
|
|
114
|
+
| `article_markdown` | `str` | 文章正文的 Markdown 内容;视频或音频分享没有文字附言时可为空 |
|
|
108
115
|
| `article_publish_time` | `int` | 发布时间(Unix 时间戳) |
|
|
109
|
-
| `article_display_type` | `ArticleDisplayType` | 文章展示类型:`article`、`video`、`image`、`text` 或 `unknown` |
|
|
110
|
-
| `article_item_show_type` | `int \| None` | 微信页面中的原始展示类型:`0` 图文、`5` 视频、`8` 图片轮播、`10` 文字;其他已提取的数值原样保留,缺失或无法解析时为 `None` |
|
|
116
|
+
| `article_display_type` | `ArticleDisplayType` | 文章展示类型:`article`、`video`、`audio`、`image`、`text` 或 `unknown` |
|
|
117
|
+
| `article_item_show_type` | `int \| None` | 微信页面中的原始展示类型:`0` 图文、`5` 视频、`7` 音频、`8` 图片轮播、`10` 文字;其他已提取的数值原样保留,缺失或无法解析时为 `None` |
|
|
111
118
|
| `raw_html` | `str` | 原始 HTML 网页源代码(仅在 `include_raw_html=True` 时填充,否则为空字符串) |
|
|
112
119
|
| `images` | `list[str]` | 文章中提取的所有图片链接 |
|
|
113
120
|
| `is_valid` | `bool` | 关键字段是否全部解析成功(属性) |
|
|
114
121
|
|
|
115
|
-
`is_valid` 为 `True` 的条件:`mp_id`、`mp_name`、`article_id`、`article_msg_id`、`article_idx`、`article_sn`、`article_title`、`article_publish_time` 均不为空/零,且 `article_markdown` 非空或 `article_display_type
|
|
122
|
+
`is_valid` 为 `True` 的条件:`mp_id`、`mp_name`、`article_id`、`article_msg_id`、`article_idx`、`article_sn`、`article_title`、`article_publish_time` 均不为空/零,且 `article_markdown` 非空或 `article_display_type in ("video", "audio")`。视频和音频分享允许没有文字正文,仍须满足其他关键字段要求;有效性不表示已获取视频或音频文件。
|
|
116
123
|
|
|
117
124
|
付费受限页面返回提示且关键元数据完整时,`is_valid` 也为 `True`,这不表示已获取付费正文。
|
|
118
125
|
|
|
@@ -124,6 +131,7 @@ result = parse(
|
|
|
124
131
|
|---|---|---|
|
|
125
132
|
| `ArticleDisplayType.ARTICLE` | `article` | 图文,包括转载式 |
|
|
126
133
|
| `ArticleDisplayType.VIDEO` | `video` | 视频分享 |
|
|
134
|
+
| `ArticleDisplayType.AUDIO` | `audio` | 音频分享 |
|
|
127
135
|
| `ArticleDisplayType.IMAGE` | `image` | 图片轮播 |
|
|
128
136
|
| `ArticleDisplayType.TEXT` | `text` | 文字消息 |
|
|
129
137
|
| `ArticleDisplayType.UNKNOWN` | `unknown` | 页面类型缺失或未支持 |
|
|
@@ -197,7 +205,7 @@ except httpx.HTTPStatusError as e:
|
|
|
197
205
|
|
|
198
206
|
### 解析不完整
|
|
199
207
|
|
|
200
|
-
|
|
208
|
+
缺少标题、发布时间等关键字段,或非视频/音频分享缺少正文时,解析结果的 `is_valid` 会返回 `False`。封面图、摘要等非必填字段为空不影响有效性。建议在业务逻辑中检查:
|
|
201
209
|
|
|
202
210
|
```python
|
|
203
211
|
result = parse("https://mp.weixin.qq.com/s/xxxxx")
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "wechat-article-parser"
|
|
7
|
-
version = "0.0.
|
|
7
|
+
version = "0.0.12"
|
|
8
8
|
description = "WeChat MP article parser - extract metadata and content from WeChat public account articles"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
{wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/src/wechat_article_parser/models.py
RENAMED
|
@@ -28,6 +28,7 @@ class ArticleDisplayType(str, Enum):
|
|
|
28
28
|
UNKNOWN = "unknown"
|
|
29
29
|
ARTICLE = "article"
|
|
30
30
|
VIDEO = "video"
|
|
31
|
+
AUDIO = "audio"
|
|
31
32
|
IMAGE = "image"
|
|
32
33
|
TEXT = "text"
|
|
33
34
|
|
|
@@ -77,7 +78,7 @@ class ArticleResult:
|
|
|
77
78
|
|
|
78
79
|
@property
|
|
79
80
|
def is_valid(self) -> bool:
|
|
80
|
-
"""
|
|
81
|
+
"""检查关键字段是否解析成功;视频和音频分享允许正文为空。"""
|
|
81
82
|
return bool(
|
|
82
83
|
self.mp_id
|
|
83
84
|
and self.mp_name
|
|
@@ -88,7 +89,9 @@ class ArticleResult:
|
|
|
88
89
|
and self.article_title
|
|
89
90
|
and (
|
|
90
91
|
self.article_markdown
|
|
91
|
-
or self.article_display_type
|
|
92
|
+
or self.article_display_type in (
|
|
93
|
+
ArticleDisplayType.VIDEO, ArticleDisplayType.AUDIO,
|
|
94
|
+
)
|
|
92
95
|
)
|
|
93
96
|
and self.article_publish_time
|
|
94
97
|
)
|
{wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/src/wechat_article_parser/parser.py
RENAMED
|
@@ -462,6 +462,26 @@ def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None
|
|
|
462
462
|
paragraph.append(signature)
|
|
463
463
|
profile.replace_with(paragraph)
|
|
464
464
|
|
|
465
|
+
# 小程序卡片只保留公开文字和查看提示,不收集封面、头像或内部路径。
|
|
466
|
+
for card in soup.find_all("mp-common-miniprogram"):
|
|
467
|
+
title = card.get("data-miniprogram-title", "").strip()
|
|
468
|
+
nickname = card.get("data-miniprogram-nickname", "").strip()
|
|
469
|
+
paragraphs = []
|
|
470
|
+
if title:
|
|
471
|
+
paragraph = soup.new_tag("p")
|
|
472
|
+
heading = soup.new_tag("strong")
|
|
473
|
+
heading.string = title
|
|
474
|
+
paragraph.append(heading)
|
|
475
|
+
paragraphs.append(paragraph)
|
|
476
|
+
if nickname:
|
|
477
|
+
paragraph = soup.new_tag("p")
|
|
478
|
+
paragraph.string = f"小程序:{nickname}"
|
|
479
|
+
paragraphs.append(paragraph)
|
|
480
|
+
notice = soup.new_tag("p")
|
|
481
|
+
notice.string = "请在微信中打开小程序查看内容。"
|
|
482
|
+
paragraphs.append(notice)
|
|
483
|
+
card.replace_with(*paragraphs)
|
|
484
|
+
|
|
465
485
|
# 处理 <img> 标签
|
|
466
486
|
to_remove = []
|
|
467
487
|
for img in soup.find_all("img"):
|
|
@@ -522,6 +542,8 @@ def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None
|
|
|
522
542
|
result.article_markdown = "这是一篇付费内容,正文需付费后查看。"
|
|
523
543
|
elif has_video:
|
|
524
544
|
result.article_markdown = "这是一篇纯视频内容。"
|
|
545
|
+
elif soup.find("mp-common-cpsad") is not None:
|
|
546
|
+
result.article_markdown = "这是一篇商品推荐内容。"
|
|
525
547
|
|
|
526
548
|
|
|
527
549
|
def _extract_repost_content(
|
|
@@ -660,12 +682,14 @@ _CONTENT_SELECTORS = {
|
|
|
660
682
|
_ITEM_SHOW_TYPE_BRANCHES = {
|
|
661
683
|
0: ("rich_text", "repost"),
|
|
662
684
|
5: ("video",),
|
|
685
|
+
7: ("video",), # 音频分享复用视频分享的文字附言提取。
|
|
663
686
|
8: ("swiper",),
|
|
664
687
|
10: ("plain_text", "fullscreen"),
|
|
665
688
|
}
|
|
666
689
|
_ARTICLE_DISPLAY_TYPES = {
|
|
667
690
|
0: ArticleDisplayType.ARTICLE,
|
|
668
691
|
5: ArticleDisplayType.VIDEO,
|
|
692
|
+
7: ArticleDisplayType.AUDIO,
|
|
669
693
|
8: ArticleDisplayType.IMAGE,
|
|
670
694
|
10: ArticleDisplayType.TEXT,
|
|
671
695
|
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""小程序卡片文字提取、图片排除及混合正文顺序的回归测试。"""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from wechat_article_parser import ArticleDisplayType
|
|
6
|
+
from wechat_article_parser.parser import _parse_html
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
_URL = "https://mp.weixin.qq.com/s/qs2YxgdwWDplsmb6H8OS-w"
|
|
10
|
+
_NOTICE = "请在微信中打开小程序查看内容。"
|
|
11
|
+
_METADATA = """
|
|
12
|
+
var item_show_type = "0";
|
|
13
|
+
var biz = "MzIxMjg3NDA0Nw==";
|
|
14
|
+
var mid = "2247487342";
|
|
15
|
+
var idx = "1";
|
|
16
|
+
var sn = "6cda9d4c31fd025a8a9a319f96173d4a";
|
|
17
|
+
window.__allowLoadResFromMp = true;
|
|
18
|
+
var hd_head_img = "https://example.com/publisher-avatar";
|
|
19
|
+
var nickname = htmlDecode("苏瑞膜");
|
|
20
|
+
var oriCreateTime = '1775961027';
|
|
21
|
+
"""
|
|
22
|
+
_CARD = (
|
|
23
|
+
'<mp-common-miniprogram data-miniprogram-title="苏瑞膜反渗透与纳滤产品手册(2026版)" '
|
|
24
|
+
'data-miniprogram-nickname="快文档" '
|
|
25
|
+
'data-miniprogram-imageurl="https://example.com/cover.jpg" '
|
|
26
|
+
'data-miniprogram-avatar="https://example.com/avatar.jpg" '
|
|
27
|
+
'data-miniprogram-path="/pages/download/download?fid=13695" '
|
|
28
|
+
'data-miniprogram-appid="wxc24ac90045c531a4">'
|
|
29
|
+
'</mp-common-miniprogram>'
|
|
30
|
+
)
|
|
31
|
+
_EXPECTED = f"**苏瑞膜反渗透与纳滤产品手册(2026版)**\n\n小程序:快文档\n\n{_NOTICE}"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _html(body: str, metadata: str = _METADATA) -> str:
|
|
35
|
+
return (
|
|
36
|
+
'<meta property="og:title" content="苏瑞膜SURO®反渗透与纳滤膜产品手册(2026)">'
|
|
37
|
+
f'<div class="rich_media_content">{body}</div>'
|
|
38
|
+
f'<script type="text/javascript">{metadata}</script>'
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@pytest.mark.parametrize("metadata", [_METADATA, ""])
|
|
43
|
+
def test_miniprogram_only_article_extracts_text_and_requires_metadata(metadata: str) -> None:
|
|
44
|
+
result = _parse_html(_URL, _html(f'<section>{_CARD}</section><p><br></p>', metadata))
|
|
45
|
+
|
|
46
|
+
assert result.article_markdown == _EXPECTED
|
|
47
|
+
assert result.images == []
|
|
48
|
+
assert result.is_valid is bool(metadata)
|
|
49
|
+
if metadata:
|
|
50
|
+
assert result.article_display_type is ArticleDisplayType.ARTICLE
|
|
51
|
+
assert result.article_item_show_type == 0
|
|
52
|
+
assert result.mp_name == "苏瑞膜"
|
|
53
|
+
assert result.article_msg_id == 2247487342
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_mixed_cards_preserve_order_and_exclude_nested_card_images() -> None:
|
|
57
|
+
rendered_card = _CARD.replace(
|
|
58
|
+
'</mp-common-miniprogram>',
|
|
59
|
+
'<img src="https://example.com/cover.jpg"><p>重复卡片文字</p>'
|
|
60
|
+
'</mp-common-miniprogram>',
|
|
61
|
+
)
|
|
62
|
+
result = _parse_html(_URL, _html(
|
|
63
|
+
'<p>正文开头。</p>' + rendered_card
|
|
64
|
+
+ '<p><img data-src="https://example.com/body.jpg"></p>'
|
|
65
|
+
+ '<mp-common-miniprogram data-miniprogram-title="第二份文档" '
|
|
66
|
+
'data-miniprogram-nickname="文档助手"></mp-common-miniprogram>'
|
|
67
|
+
'<p>正文结尾。</p>'
|
|
68
|
+
))
|
|
69
|
+
|
|
70
|
+
assert result.article_markdown == (
|
|
71
|
+
f'正文开头。\n\n{_EXPECTED}\n\n\n\n'
|
|
72
|
+
f'**第二份文档**\n\n小程序:文档助手\n\n{_NOTICE}\n\n正文结尾。'
|
|
73
|
+
)
|
|
74
|
+
assert result.images == ["https://example.com/body.jpg"]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@pytest.mark.parametrize(("attributes", "expected"), [
|
|
78
|
+
('data-miniprogram-title=" 手册 "', f'**手册**\n\n{_NOTICE}'),
|
|
79
|
+
('data-miniprogram-nickname=" 快文档 "', f'小程序:快文档\n\n{_NOTICE}'),
|
|
80
|
+
('data-miniprogram-title=" " data-miniprogram-nickname=" "', _NOTICE),
|
|
81
|
+
('', _NOTICE),
|
|
82
|
+
])
|
|
83
|
+
def test_missing_card_fields_omit_empty_paragraphs(attributes: str, expected: str) -> None:
|
|
84
|
+
result = _parse_html(_URL, _html(
|
|
85
|
+
f'<mp-common-miniprogram {attributes}></mp-common-miniprogram>'
|
|
86
|
+
))
|
|
87
|
+
|
|
88
|
+
assert result.article_markdown == expected
|
|
89
|
+
assert result.images == []
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def test_card_attributes_preserve_entities_and_escape_markdown() -> None:
|
|
93
|
+
result = _parse_html(_URL, _html(
|
|
94
|
+
'<mp-common-miniprogram data-miniprogram-title="A & B *手册* <tag>" '
|
|
95
|
+
'data-miniprogram-nickname="文档&助手"></mp-common-miniprogram>'
|
|
96
|
+
))
|
|
97
|
+
|
|
98
|
+
assert result.article_markdown == (
|
|
99
|
+
r'**A & B \*手册\* <tag>**' f'\n\n小程序:文档&助手\n\n{_NOTICE}'
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_card_outside_article_body_is_not_extracted() -> None:
|
|
104
|
+
result = _parse_html(_URL, _html("") + _CARD)
|
|
105
|
+
|
|
106
|
+
assert result.article_markdown == ""
|
|
107
|
+
assert result.images == []
|
|
108
|
+
assert not result.is_valid
|
|
@@ -22,14 +22,16 @@ _COMPLETE = ArticleResult(
|
|
|
22
22
|
|
|
23
23
|
@pytest.mark.parametrize("display_type", list(ArticleDisplayType))
|
|
24
24
|
@pytest.mark.parametrize("markdown", ["", "正文"])
|
|
25
|
-
def
|
|
25
|
+
def test_only_video_and_audio_can_be_valid_without_markdown(
|
|
26
26
|
display_type: ArticleDisplayType, markdown: str,
|
|
27
27
|
) -> None:
|
|
28
28
|
result = replace(
|
|
29
29
|
_COMPLETE, article_display_type=display_type, article_markdown=markdown,
|
|
30
30
|
)
|
|
31
31
|
|
|
32
|
-
assert result.is_valid is (
|
|
32
|
+
assert result.is_valid is (
|
|
33
|
+
bool(markdown) or display_type in (ArticleDisplayType.VIDEO, ArticleDisplayType.AUDIO)
|
|
34
|
+
)
|
|
33
35
|
|
|
34
36
|
|
|
35
37
|
@pytest.mark.parametrize(
|
|
@@ -45,13 +47,14 @@ def test_only_video_can_be_valid_without_markdown(
|
|
|
45
47
|
("article_publish_time", 0),
|
|
46
48
|
],
|
|
47
49
|
)
|
|
48
|
-
@pytest.mark.parametrize("markdown", ["", "
|
|
49
|
-
|
|
50
|
-
|
|
50
|
+
@pytest.mark.parametrize("markdown", ["", "媒体附言"])
|
|
51
|
+
@pytest.mark.parametrize("display_type", [ArticleDisplayType.VIDEO, ArticleDisplayType.AUDIO])
|
|
52
|
+
def test_video_and_audio_still_require_all_key_metadata(
|
|
53
|
+
field_name: str, empty_value: str | int, markdown: str, display_type: ArticleDisplayType,
|
|
51
54
|
) -> None:
|
|
52
55
|
result = replace(
|
|
53
56
|
_COMPLETE,
|
|
54
|
-
article_display_type=
|
|
57
|
+
article_display_type=display_type,
|
|
55
58
|
article_markdown=markdown,
|
|
56
59
|
**{field_name: empty_value},
|
|
57
60
|
)
|
|
@@ -64,6 +67,7 @@ def test_video_still_requires_all_key_metadata(
|
|
|
64
67
|
[
|
|
65
68
|
"article",
|
|
66
69
|
"video",
|
|
70
|
+
"audio",
|
|
67
71
|
"image",
|
|
68
72
|
"text",
|
|
69
73
|
"unknown",
|
|
@@ -28,6 +28,12 @@ TEST_URLS = [
|
|
|
28
28
|
"https://mp.weixin.qq.com/s/3ixeC9whuEgpup2G1yvDZg",
|
|
29
29
|
# 纯公众号推荐卡片正文。
|
|
30
30
|
"https://mp.weixin.qq.com/s/PXZuh3fFEi3VXUUI1uBnlw",
|
|
31
|
+
# 无文字正文的音频分享。
|
|
32
|
+
"https://mp.weixin.qq.com/s/y4QxlYs7u0X9TdvZatCfIA",
|
|
33
|
+
# 纯商品广告卡片正文。
|
|
34
|
+
"https://mp.weixin.qq.com/s/CBmoIFPJwV4BkidYGp-l1Q",
|
|
35
|
+
# 纯小程序卡片正文。
|
|
36
|
+
"https://mp.weixin.qq.com/s/qs2YxgdwWDplsmb6H8OS-w",
|
|
31
37
|
]
|
|
32
38
|
|
|
33
39
|
|
|
@@ -102,8 +108,8 @@ def _assert_result(result: ArticleResult, url: str) -> None:
|
|
|
102
108
|
assert result.article_idx > 0, "article_idx should be positive"
|
|
103
109
|
assert result.article_sn, "article_sn should not be empty"
|
|
104
110
|
assert result.article_title, "article_title should not be empty"
|
|
105
|
-
if result.article_display_type
|
|
106
|
-
assert result.article_markdown, "article_markdown should not be empty for non-video articles"
|
|
111
|
+
if result.article_display_type not in ("video", "audio"):
|
|
112
|
+
assert result.article_markdown, "article_markdown should not be empty for non-video/audio articles"
|
|
107
113
|
assert result.article_publish_time > 0, "article_publish_time should be positive"
|
|
108
114
|
assert result.is_valid
|
|
109
115
|
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""商品卡片占位提示及混合正文的回归测试。"""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from wechat_article_parser import ArticleDisplayType
|
|
6
|
+
from wechat_article_parser.parser import _parse_html
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
_URL = "https://mp.weixin.qq.com/s/CBmoIFPJwV4BkidYGp-l1Q"
|
|
10
|
+
_NOTICE = "这是一篇商品推荐内容。"
|
|
11
|
+
_CARD = '<section><mp-common-cpsad data-pid="104_630468802"></mp-common-cpsad></section>'
|
|
12
|
+
_METADATA = """
|
|
13
|
+
var biz = "MzkwNDQyNzY2NQ==";
|
|
14
|
+
var mid = "2247501127";
|
|
15
|
+
var idx = "2";
|
|
16
|
+
var sn = "bc04bcc079a380252791ace32e8a91ba";
|
|
17
|
+
window.__allowLoadResFromMp = true;
|
|
18
|
+
var hd_head_img = "https://example.com/avatar";
|
|
19
|
+
var nickname = htmlDecode("老鱼讲古");
|
|
20
|
+
var oriCreateTime = '1791711648';
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _html(body: str, metadata: str = _METADATA) -> str:
|
|
25
|
+
return (
|
|
26
|
+
'<meta property="og:title" content="推荐几本好书">'
|
|
27
|
+
f'<div class="rich_media_content" id="js_content">{body}</div>'
|
|
28
|
+
'<script type="text/javascript">var item_show_type = "0";</script>'
|
|
29
|
+
f'<script type="text/javascript">{metadata}</script>'
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@pytest.mark.parametrize("metadata", [_METADATA, ""])
|
|
34
|
+
def test_product_only_body_returns_one_notice_and_requires_metadata(metadata: str) -> None:
|
|
35
|
+
result = _parse_html(_URL, _html(
|
|
36
|
+
_CARD * 4 + '<section><span><br/></span></section>'
|
|
37
|
+
'<p style="display: none;"><mp-style-type data-value="3"></mp-style-type></p>',
|
|
38
|
+
metadata,
|
|
39
|
+
))
|
|
40
|
+
|
|
41
|
+
assert result.article_markdown == _NOTICE
|
|
42
|
+
assert result.article_display_type is ArticleDisplayType.ARTICLE
|
|
43
|
+
assert result.article_item_show_type == 0
|
|
44
|
+
assert result.article_title == "推荐几本好书"
|
|
45
|
+
assert result.images == []
|
|
46
|
+
assert result.is_valid is bool(metadata)
|
|
47
|
+
if metadata:
|
|
48
|
+
assert result.mp_name == "老鱼讲古"
|
|
49
|
+
assert result.article_msg_id == 2247501127
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@pytest.mark.parametrize(("body", "expected", "images"), [
|
|
53
|
+
('<p>推荐书单。</p>' + _CARD + '<p>阅读愉快。</p>',
|
|
54
|
+
'推荐书单。\n\n阅读愉快。', []),
|
|
55
|
+
(_CARD + '<img data-src="https://example.com/book.jpg">',
|
|
56
|
+
'', ['https://example.com/book.jpg']),
|
|
57
|
+
])
|
|
58
|
+
def test_product_cards_do_not_replace_visible_content(
|
|
59
|
+
body: str, expected: str, images: list[str],
|
|
60
|
+
) -> None:
|
|
61
|
+
result = _parse_html(_URL, _html(body))
|
|
62
|
+
|
|
63
|
+
assert result.article_markdown == expected
|
|
64
|
+
assert result.images == images
|
|
65
|
+
assert result.is_valid
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_product_card_outside_body_does_not_fill_empty_body() -> None:
|
|
69
|
+
result = _parse_html(_URL, _html("") + _CARD)
|
|
70
|
+
|
|
71
|
+
assert result.article_markdown == ""
|
|
72
|
+
assert not result.is_valid
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@pytest.mark.parametrize(("embed", "expected"), [
|
|
76
|
+
('<mp-pay-preview-filter></mp-pay-preview-filter>', '这是一篇付费内容,正文需付费后查看。'),
|
|
77
|
+
('<iframe class="wx_video_iframe"></iframe>', '这是一篇纯视频内容。'),
|
|
78
|
+
])
|
|
79
|
+
def test_existing_notices_keep_priority(embed: str, expected: str) -> None:
|
|
80
|
+
result = _parse_html(_URL, _html(_CARD + embed))
|
|
81
|
+
|
|
82
|
+
assert result.article_markdown == expected
|
|
83
|
+
assert result.is_valid
|
|
@@ -49,6 +49,7 @@ def _html(body: str = "", script: str = "", title: str = "原标题") -> str:
|
|
|
49
49
|
("window.item_show_type = '8' || '';", 8),
|
|
50
50
|
("window.item_show_type = 10;", 10),
|
|
51
51
|
("var /* 注释 */ item_show_type = 5;", 5),
|
|
52
|
+
("window.item_show_type = '7';", 7),
|
|
52
53
|
("(function() { window.item_show_type = '10'; })();", 10),
|
|
53
54
|
("window.item_show_type = '999';", 999),
|
|
54
55
|
("window.item_show_type = '';", None),
|
|
@@ -120,6 +121,7 @@ def test_type_zero_uses_html_to_distinguish_rich_text_and_repost(
|
|
|
120
121
|
("item_type", "data", "expected", "display_type"),
|
|
121
122
|
[
|
|
122
123
|
(5, _PLAIN_DATA, "文字正文", ArticleDisplayType.VIDEO),
|
|
124
|
+
(7, _PLAIN_DATA, "文字正文", ArticleDisplayType.AUDIO),
|
|
123
125
|
(8, _SWIPER_DATA, "轮播正文", ArticleDisplayType.IMAGE),
|
|
124
126
|
(10, _PLAIN_DATA, "文字正文", ArticleDisplayType.TEXT),
|
|
125
127
|
(10, _FULLSCREEN_DATA, "全屏正文", ArticleDisplayType.TEXT),
|
|
@@ -153,7 +155,7 @@ def test_missing_unknown_or_invalid_type_falls_back_to_html(declaration: str) ->
|
|
|
153
155
|
assert result.article_display_type is ArticleDisplayType.UNKNOWN
|
|
154
156
|
|
|
155
157
|
|
|
156
|
-
@pytest.mark.parametrize("item_type", [5, 8, 10])
|
|
158
|
+
@pytest.mark.parametrize("item_type", [5, 7, 8, 10])
|
|
157
159
|
def test_known_type_without_matching_data_falls_back_to_html(item_type: int) -> None:
|
|
158
160
|
result = parser._parse_html(
|
|
159
161
|
_URL,
|
|
@@ -215,9 +217,10 @@ def test_unrecognized_content_still_returns_metadata() -> None:
|
|
|
215
217
|
[
|
|
216
218
|
("", ArticleDisplayType.UNKNOWN),
|
|
217
219
|
("window.item_show_type = '5';", ArticleDisplayType.VIDEO),
|
|
220
|
+
("window.item_show_type = '7';", ArticleDisplayType.AUDIO),
|
|
218
221
|
],
|
|
219
222
|
)
|
|
220
|
-
def
|
|
223
|
+
def test_video_and_audio_without_description_keep_type_and_metadata(
|
|
221
224
|
declaration: str, expected_type: ArticleDisplayType,
|
|
222
225
|
) -> None:
|
|
223
226
|
metadata = """
|
|
@@ -247,4 +250,6 @@ def test_video_without_description_keeps_type_and_metadata(
|
|
|
247
250
|
assert result.mp_id == 1
|
|
248
251
|
assert result.mp_name == "测试公众号"
|
|
249
252
|
assert result.article_markdown == ""
|
|
250
|
-
assert result.is_valid is (
|
|
253
|
+
assert result.is_valid is (
|
|
254
|
+
expected_type in (ArticleDisplayType.VIDEO, ArticleDisplayType.AUDIO)
|
|
255
|
+
)
|
|
File without changes
|
|
File without changes
|
{wechat_article_parser-0.0.10 → wechat_article_parser-0.0.12}/src/wechat_article_parser/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|