wechat-article-parser 0.0.8__tar.gz → 0.0.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/PKG-INFO +13 -2
- {wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/README.md +12 -1
- {wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/pyproject.toml +1 -1
- {wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/src/wechat_article_parser/models.py +5 -1
- {wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/src/wechat_article_parser/parser.py +178 -27
- {wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/tests/test_models.py +2 -2
- wechat_article_parser-0.0.10/tests/test_paid_content.py +106 -0
- {wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/tests/test_parser.py +17 -0
- wechat_article_parser-0.0.10/tests/test_profile_content.py +127 -0
- wechat_article_parser-0.0.10/tests/test_repost_content.py +140 -0
- wechat_article_parser-0.0.10/tests/test_svg_content.py +98 -0
- {wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/tests/test_type_routing.py +10 -1
- wechat_article_parser-0.0.10/tests/test_user_name.py +137 -0
- wechat_article_parser-0.0.10/tests/test_video_content.py +85 -0
- {wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/.gitignore +0 -0
- {wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/LICENSE +0 -0
- {wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/src/wechat_article_parser/__init__.py +0 -0
- {wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/tests/conftest.py +0 -0
- {wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/tests/test_markdown.py +0 -0
- {wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/tests/test_swiper_content.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: wechat-article-parser
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.10
|
|
4
4
|
Summary: WeChat MP article parser - extract metadata and content from WeChat public account articles
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -30,6 +30,12 @@ Description-Content-Type: text/markdown
|
|
|
30
30
|
`8` 对应图片轮播、`10` 对应文字消息,再通过 HTML 或脚本数据区分具体模板。
|
|
31
31
|
类型字段缺失、值未知或对应分支未提取到正文时,会按原有 HTML 特征匹配顺序兜底。
|
|
32
32
|
|
|
33
|
+
图文正文仅包含视频号视频卡片或微信视频 iframe 时,Markdown 返回“这是一篇纯视频内容。”。
|
|
34
|
+
正文只有付费预览占位标签时,Markdown 返回“这是一篇付费内容,正文需付费后查看。”,
|
|
35
|
+
并保留公开元数据;已有的可见预览正文仍正常提取。
|
|
36
|
+
图文中的公众号推荐卡片按原文顺序转换为 `**公众号名称**:简介`,每张卡片一段;
|
|
37
|
+
名称或简介缺失时,仅输出已有字段。
|
|
38
|
+
|
|
33
39
|
## 安装
|
|
34
40
|
|
|
35
41
|
```bash
|
|
@@ -102,6 +108,7 @@ result = parse(
|
|
|
102
108
|
| `mp_id` | `int` | 公众号 ID(解码后的整数) |
|
|
103
109
|
| `mp_name` | `str` | 公众号名称 |
|
|
104
110
|
| `mp_alias` | `str` | 公众号别名 |
|
|
111
|
+
| `mp_user_name` | `str` | 公众号用户名(如 `aifaner383402` 或 `gh_9a1919baf888`),未获取到时为空字符串 |
|
|
105
112
|
| `mp_image` | `str` | 公众号头像链接 |
|
|
106
113
|
| `mp_description` | `str` | 公众号简介 |
|
|
107
114
|
| `mp_account_type` | `AccountType` | 账号类型:`AccountType.SUBSCRIPTION`(订阅号)/ `AccountType.SERVICE`(服务号)/ `AccountType.UNKNOWN`(未识别) |
|
|
@@ -115,12 +122,15 @@ result = parse(
|
|
|
115
122
|
| `article_markdown` | `str` | 文章正文的 Markdown 内容;视频分享没有文字附言时可为空 |
|
|
116
123
|
| `article_publish_time` | `int` | 发布时间(Unix 时间戳) |
|
|
117
124
|
| `article_display_type` | `ArticleDisplayType` | 文章展示类型:`article`、`video`、`image`、`text` 或 `unknown` |
|
|
125
|
+
| `article_item_show_type` | `int \| None` | 微信页面中的原始展示类型:`0` 图文、`5` 视频、`8` 图片轮播、`10` 文字;其他已提取的数值原样保留,缺失或无法解析时为 `None` |
|
|
118
126
|
| `raw_html` | `str` | 原始 HTML 网页源代码(仅在 `include_raw_html=True` 时填充,否则为空字符串) |
|
|
119
127
|
| `images` | `list[str]` | 文章中提取的所有图片链接 |
|
|
120
128
|
| `is_valid` | `bool` | 关键字段是否全部解析成功(属性) |
|
|
121
129
|
|
|
122
130
|
`is_valid` 为 `True` 的条件:`mp_id`、`mp_name`、`article_id`、`article_msg_id`、`article_idx`、`article_sn`、`article_title`、`article_publish_time` 均不为空/零,且 `article_markdown` 非空或 `article_display_type == "video"`。视频分享允许没有文字正文,仍须满足其他关键字段要求;有效性不表示已获取视频文件。
|
|
123
131
|
|
|
132
|
+
付费受限页面返回提示且关键元数据完整时,`is_valid` 也为 `True`,这不表示已获取付费正文。
|
|
133
|
+
|
|
124
134
|
### 判断文章展示类型
|
|
125
135
|
|
|
126
136
|
`ArticleDisplayType` 是字符串枚举,既支持枚举比较,也支持语义字符串比较:
|
|
@@ -144,10 +154,11 @@ if result.article_display_type == "video":
|
|
|
144
154
|
print("视频分享可以没有文字正文")
|
|
145
155
|
|
|
146
156
|
print(result.article_display_type) # 例如 video
|
|
157
|
+
print(result.article_item_show_type) # 例如 5
|
|
147
158
|
```
|
|
148
159
|
|
|
149
160
|
`article_display_type` 是返回结果中的数据字段,包含在 `dataclasses.asdict(result)` 中,JSON 序列化时输出语义字符串。
|
|
150
|
-
|
|
161
|
+
`article_item_show_type` 返回微信页面中 `item_show_type` 的原始数值,也包含在 `dataclasses.asdict(result)` 中;JSON 序列化时输出数字或 `null`。
|
|
151
162
|
|
|
152
163
|
### 判断账号类型
|
|
153
164
|
|
|
@@ -15,6 +15,12 @@
|
|
|
15
15
|
`8` 对应图片轮播、`10` 对应文字消息,再通过 HTML 或脚本数据区分具体模板。
|
|
16
16
|
类型字段缺失、值未知或对应分支未提取到正文时,会按原有 HTML 特征匹配顺序兜底。
|
|
17
17
|
|
|
18
|
+
图文正文仅包含视频号视频卡片或微信视频 iframe 时,Markdown 返回“这是一篇纯视频内容。”。
|
|
19
|
+
正文只有付费预览占位标签时,Markdown 返回“这是一篇付费内容,正文需付费后查看。”,
|
|
20
|
+
并保留公开元数据;已有的可见预览正文仍正常提取。
|
|
21
|
+
图文中的公众号推荐卡片按原文顺序转换为 `**公众号名称**:简介`,每张卡片一段;
|
|
22
|
+
名称或简介缺失时,仅输出已有字段。
|
|
23
|
+
|
|
18
24
|
## 安装
|
|
19
25
|
|
|
20
26
|
```bash
|
|
@@ -87,6 +93,7 @@ result = parse(
|
|
|
87
93
|
| `mp_id` | `int` | 公众号 ID(解码后的整数) |
|
|
88
94
|
| `mp_name` | `str` | 公众号名称 |
|
|
89
95
|
| `mp_alias` | `str` | 公众号别名 |
|
|
96
|
+
| `mp_user_name` | `str` | 公众号用户名(如 `aifaner383402` 或 `gh_9a1919baf888`),未获取到时为空字符串 |
|
|
90
97
|
| `mp_image` | `str` | 公众号头像链接 |
|
|
91
98
|
| `mp_description` | `str` | 公众号简介 |
|
|
92
99
|
| `mp_account_type` | `AccountType` | 账号类型:`AccountType.SUBSCRIPTION`(订阅号)/ `AccountType.SERVICE`(服务号)/ `AccountType.UNKNOWN`(未识别) |
|
|
@@ -100,12 +107,15 @@ result = parse(
|
|
|
100
107
|
| `article_markdown` | `str` | 文章正文的 Markdown 内容;视频分享没有文字附言时可为空 |
|
|
101
108
|
| `article_publish_time` | `int` | 发布时间(Unix 时间戳) |
|
|
102
109
|
| `article_display_type` | `ArticleDisplayType` | 文章展示类型:`article`、`video`、`image`、`text` 或 `unknown` |
|
|
110
|
+
| `article_item_show_type` | `int \| None` | 微信页面中的原始展示类型:`0` 图文、`5` 视频、`8` 图片轮播、`10` 文字;其他已提取的数值原样保留,缺失或无法解析时为 `None` |
|
|
103
111
|
| `raw_html` | `str` | 原始 HTML 网页源代码(仅在 `include_raw_html=True` 时填充,否则为空字符串) |
|
|
104
112
|
| `images` | `list[str]` | 文章中提取的所有图片链接 |
|
|
105
113
|
| `is_valid` | `bool` | 关键字段是否全部解析成功(属性) |
|
|
106
114
|
|
|
107
115
|
`is_valid` 为 `True` 的条件:`mp_id`、`mp_name`、`article_id`、`article_msg_id`、`article_idx`、`article_sn`、`article_title`、`article_publish_time` 均不为空/零,且 `article_markdown` 非空或 `article_display_type == "video"`。视频分享允许没有文字正文,仍须满足其他关键字段要求;有效性不表示已获取视频文件。
|
|
108
116
|
|
|
117
|
+
付费受限页面返回提示且关键元数据完整时,`is_valid` 也为 `True`,这不表示已获取付费正文。
|
|
118
|
+
|
|
109
119
|
### 判断文章展示类型
|
|
110
120
|
|
|
111
121
|
`ArticleDisplayType` 是字符串枚举,既支持枚举比较,也支持语义字符串比较:
|
|
@@ -129,10 +139,11 @@ if result.article_display_type == "video":
|
|
|
129
139
|
print("视频分享可以没有文字正文")
|
|
130
140
|
|
|
131
141
|
print(result.article_display_type) # 例如 video
|
|
142
|
+
print(result.article_item_show_type) # 例如 5
|
|
132
143
|
```
|
|
133
144
|
|
|
134
145
|
`article_display_type` 是返回结果中的数据字段,包含在 `dataclasses.asdict(result)` 中,JSON 序列化时输出语义字符串。
|
|
135
|
-
|
|
146
|
+
`article_item_show_type` 返回微信页面中 `item_show_type` 的原始数值,也包含在 `dataclasses.asdict(result)` 中;JSON 序列化时输出数字或 `null`。
|
|
136
147
|
|
|
137
148
|
### 判断账号类型
|
|
138
149
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "wechat-article-parser"
|
|
7
|
-
version = "0.0.
|
|
7
|
+
version = "0.0.10"
|
|
8
8
|
description = "WeChat MP article parser - extract metadata and content from WeChat public account articles"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
{wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/src/wechat_article_parser/models.py
RENAMED
|
@@ -68,8 +68,12 @@ class ArticleResult:
|
|
|
68
68
|
# 文章中提取的图片列表
|
|
69
69
|
images: list[str] = field(default_factory=list)
|
|
70
70
|
|
|
71
|
-
#
|
|
71
|
+
# 语义展示类型和微信页面中的原始数值。
|
|
72
72
|
article_display_type: ArticleDisplayType = ArticleDisplayType.UNKNOWN
|
|
73
|
+
article_item_show_type: int | None = None
|
|
74
|
+
|
|
75
|
+
# 当前公众号的页面全局 user_name,未提取到时为空字符串。
|
|
76
|
+
mp_user_name: str = ""
|
|
73
77
|
|
|
74
78
|
@property
|
|
75
79
|
def is_valid(self) -> bool:
|
{wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/src/wechat_article_parser/parser.py
RENAMED
|
@@ -78,6 +78,21 @@ def _decode_hex_escapes(text: str) -> str:
|
|
|
78
78
|
return re.sub(r"\\x([0-9a-fA-F]{2})", lambda m: chr(int(m.group(1), 16)), text)
|
|
79
79
|
|
|
80
80
|
|
|
81
|
+
def _decode_js_string(text: str) -> str:
|
|
82
|
+
"""解码 JS 字符串转义,保留 HTML 实体、百分号和正文中的空白。"""
|
|
83
|
+
escapes = {
|
|
84
|
+
"n": "\n", "r": "\r", "t": "\t", "b": "\b", "f": "\f", "v": "\v",
|
|
85
|
+
"0": "\0", "\n": "", "\r": "", "\r\n": "",
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
def decode(match: re.Match[str]) -> str:
|
|
89
|
+
if match[1] or match[2]:
|
|
90
|
+
return chr(int(match[1] or match[2], 16))
|
|
91
|
+
return escapes.get(match[3], match[3])
|
|
92
|
+
|
|
93
|
+
return re.sub(r"\\(?:x([\da-fA-F]{2})|u([\da-fA-F]{4})|(\r\n|[\s\S]))", decode, text)
|
|
94
|
+
|
|
95
|
+
|
|
81
96
|
def _decode_text(text: str, *, preserve_newlines: bool = False) -> str:
|
|
82
97
|
if not text:
|
|
83
98
|
return ""
|
|
@@ -229,6 +244,89 @@ def _extract_item_show_type(scripts: list[Tag]) -> int | None:
|
|
|
229
244
|
return None
|
|
230
245
|
|
|
231
246
|
|
|
247
|
+
def _extract_user_name(scripts: list[Tag]) -> str:
|
|
248
|
+
"""读取全局 user_name,缺失时读取 window.cgiDataNew 的直接属性。
|
|
249
|
+
|
|
250
|
+
文章元数据脚本中该变量位于 nickname、ct 附近;不依赖行号或账号值。
|
|
251
|
+
使用词法标记跳过注释和字符串,不执行页面脚本。
|
|
252
|
+
"""
|
|
253
|
+
for script in scripts:
|
|
254
|
+
if "user_name" not in script.text:
|
|
255
|
+
continue
|
|
256
|
+
history: deque[str] = deque(maxlen=4)
|
|
257
|
+
brace_depth = 0
|
|
258
|
+
for match in _JS_DATA_TOKEN_RE.finditer(script.text):
|
|
259
|
+
if match.lastgroup == "comment":
|
|
260
|
+
continue
|
|
261
|
+
token = match.group()
|
|
262
|
+
previous = tuple(history)
|
|
263
|
+
if (
|
|
264
|
+
brace_depth == 0
|
|
265
|
+
and token == ";"
|
|
266
|
+
and previous[:3] == ("var", "user_name", "=")
|
|
267
|
+
and len(previous) == 4
|
|
268
|
+
and previous[3].startswith(("'", '"'))
|
|
269
|
+
):
|
|
270
|
+
value = _decode_text(previous[3][1:-1])
|
|
271
|
+
if value:
|
|
272
|
+
return value
|
|
273
|
+
if token == "{":
|
|
274
|
+
brace_depth += 1
|
|
275
|
+
elif token == "}":
|
|
276
|
+
brace_depth -= 1
|
|
277
|
+
history.append(token)
|
|
278
|
+
return _extract_cgi_user_name(scripts)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _extract_cgi_user_name(scripts: list[Tag]) -> str:
|
|
282
|
+
"""从当前文章的 cgiDataNew 对象中读取 user_name,排除嵌套账号。"""
|
|
283
|
+
return _decode_text(_extract_cgi_string(scripts, "user_name"))
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _extract_cgi_string(scripts: list[Tag], field: str) -> str:
|
|
287
|
+
"""读取当前 cgiDataNew 的直接字符串属性,不执行表达式或读取嵌套对象。"""
|
|
288
|
+
for script in scripts:
|
|
289
|
+
if "cgiDataNew" not in script.text or field not in script.text:
|
|
290
|
+
continue
|
|
291
|
+
history: deque[str] = deque(maxlen=5)
|
|
292
|
+
stack: list[str] = []
|
|
293
|
+
for match in _JS_DATA_TOKEN_RE.finditer(script.text):
|
|
294
|
+
if match.lastgroup == "comment":
|
|
295
|
+
continue
|
|
296
|
+
token = match.group()
|
|
297
|
+
previous = tuple(history)
|
|
298
|
+
if not stack:
|
|
299
|
+
if (
|
|
300
|
+
token == "{"
|
|
301
|
+
and previous[-4:] == ("window", ".", "cgiDataNew", "=")
|
|
302
|
+
and (len(previous) < 5 or previous[-5] != ".")
|
|
303
|
+
):
|
|
304
|
+
stack.append(token)
|
|
305
|
+
else:
|
|
306
|
+
# 只接受完整的字符串属性值,不读取表达式或嵌套对象的值。
|
|
307
|
+
if (
|
|
308
|
+
stack == ["{"]
|
|
309
|
+
and token in (",", "}")
|
|
310
|
+
and len(previous) >= 4
|
|
311
|
+
and previous[-4] in ("{", ",")
|
|
312
|
+
and previous[-3].strip("\"'") == field
|
|
313
|
+
and previous[-2] == ":"
|
|
314
|
+
and previous[-1].startswith(("'", '"'))
|
|
315
|
+
):
|
|
316
|
+
value = previous[-1][1:-1]
|
|
317
|
+
if value.strip():
|
|
318
|
+
return value
|
|
319
|
+
if token in ("{", "["):
|
|
320
|
+
stack.append(token)
|
|
321
|
+
elif token in ("}", "]"):
|
|
322
|
+
if stack[-1] == {"}": "{", "]": "["}[token]:
|
|
323
|
+
stack.pop()
|
|
324
|
+
else:
|
|
325
|
+
stack.clear()
|
|
326
|
+
history.append(token)
|
|
327
|
+
return ""
|
|
328
|
+
|
|
329
|
+
|
|
232
330
|
def _extract_account_type(script_text: str, result: ArticleResult) -> None:
|
|
233
331
|
"""从 new_service_type 中提取账号类型:0/1 → 订阅号,2 → 服务号。"""
|
|
234
332
|
if result.mp_account_type:
|
|
@@ -343,7 +441,26 @@ def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None
|
|
|
343
441
|
"""将 rich_media_content 区块转换为 Markdown。"""
|
|
344
442
|
# prettify 会在行内标签之间插入空白,改变代码和引用链接的内容。
|
|
345
443
|
soup = BeautifulSoup(str(content_tag), "html.parser")
|
|
346
|
-
|
|
444
|
+
has_video = soup.select_one(
|
|
445
|
+
'mp-common-videosnap[data-type="video"], iframe.video_iframe, iframe.wx_video_iframe'
|
|
446
|
+
) is not None
|
|
447
|
+
|
|
448
|
+
# 原生公众号卡片的正文保存在属性中,转换为“名称 + 简介”的独立段落。
|
|
449
|
+
for profile in soup.find_all("mp-common-profile"):
|
|
450
|
+
nickname = profile.get("data-nickname", "").strip()
|
|
451
|
+
signature = profile.get("data-signature", "").strip()
|
|
452
|
+
if not nickname and not signature:
|
|
453
|
+
continue
|
|
454
|
+
paragraph = soup.new_tag("p")
|
|
455
|
+
if nickname:
|
|
456
|
+
name = soup.new_tag("strong")
|
|
457
|
+
name.string = nickname
|
|
458
|
+
paragraph.append(name)
|
|
459
|
+
if signature:
|
|
460
|
+
if nickname:
|
|
461
|
+
paragraph.append(":")
|
|
462
|
+
paragraph.append(signature)
|
|
463
|
+
profile.replace_with(paragraph)
|
|
347
464
|
|
|
348
465
|
# 处理 <img> 标签
|
|
349
466
|
to_remove = []
|
|
@@ -352,17 +469,13 @@ def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None
|
|
|
352
469
|
if src and src.startswith("http"):
|
|
353
470
|
normalized = _normalize_image_url(src)
|
|
354
471
|
img["src"] = normalized
|
|
355
|
-
if normalized not in seen:
|
|
356
|
-
seen.add(normalized)
|
|
357
|
-
result.images.append(normalized)
|
|
358
472
|
else:
|
|
359
473
|
to_remove.append(img)
|
|
360
474
|
for tag in to_remove:
|
|
361
475
|
tag.decompose()
|
|
362
476
|
|
|
363
|
-
#
|
|
364
|
-
#
|
|
365
|
-
# 需从外向内收集所有图片 URL,一次性替换为多个 <img> 标签
|
|
477
|
+
# SVG 可通过 background-image 或 foreignObject 内的 <img> 承载正文图片。
|
|
478
|
+
# 按文档顺序收集两种图片,再统一替换外层 SVG,避免删除内部正文图片。
|
|
366
479
|
to_remove = []
|
|
367
480
|
for svg in soup.find_all("svg", recursive=True):
|
|
368
481
|
# 跳过已被外层 SVG 处理过的嵌套 SVG(已脱离文档树)
|
|
@@ -372,11 +485,14 @@ def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None
|
|
|
372
485
|
if svg.find_parent("svg"):
|
|
373
486
|
continue
|
|
374
487
|
|
|
375
|
-
#
|
|
376
|
-
|
|
488
|
+
# 包含所有嵌套 SVG 和普通图片,保留正文中的重复图片及 alt 文本。
|
|
489
|
+
image_elements = [svg] + svg.find_all(["svg", "img"])
|
|
377
490
|
img_tags = []
|
|
378
|
-
for
|
|
379
|
-
|
|
491
|
+
for element in image_elements:
|
|
492
|
+
if element.name == "img":
|
|
493
|
+
img_tags.append(soup.new_tag("img", attrs=element.attrs.copy()))
|
|
494
|
+
continue
|
|
495
|
+
style = element.get("style", "")
|
|
380
496
|
if "background-image" not in style:
|
|
381
497
|
continue
|
|
382
498
|
m = re.search(r'url\("([^"]+)"\)', style)
|
|
@@ -384,9 +500,6 @@ def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None
|
|
|
384
500
|
continue
|
|
385
501
|
normalized = _normalize_image_url(m.group(1))
|
|
386
502
|
img_tags.append(soup.new_tag("img", src=normalized))
|
|
387
|
-
if normalized not in seen:
|
|
388
|
-
seen.add(normalized)
|
|
389
|
-
result.images.append(normalized)
|
|
390
503
|
|
|
391
504
|
if img_tags:
|
|
392
505
|
svg.replace_with(*img_tags)
|
|
@@ -395,30 +508,66 @@ def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None
|
|
|
395
508
|
for tag in to_remove:
|
|
396
509
|
tag.decompose()
|
|
397
510
|
|
|
511
|
+
# 从最终正文收集图片,使列表顺序与 Markdown 一致,并只对列表去重。
|
|
512
|
+
seen: set[str] = set()
|
|
513
|
+
for img in soup.find_all("img"):
|
|
514
|
+
src = img["src"]
|
|
515
|
+
if src not in seen:
|
|
516
|
+
seen.add(src)
|
|
517
|
+
result.images.append(src)
|
|
518
|
+
|
|
398
519
|
result.article_markdown = _to_markdown(str(soup))
|
|
520
|
+
if not result.article_markdown.strip():
|
|
521
|
+
if soup.find("mp-pay-preview-filter") is not None:
|
|
522
|
+
result.article_markdown = "这是一篇付费内容,正文需付费后查看。"
|
|
523
|
+
elif has_video:
|
|
524
|
+
result.article_markdown = "这是一篇纯视频内容。"
|
|
399
525
|
|
|
400
526
|
|
|
401
|
-
def _extract_repost_content(
|
|
402
|
-
|
|
403
|
-
|
|
527
|
+
def _extract_repost_content(
|
|
528
|
+
content_tag: Tag, scripts: list[Tag], result: ArticleResult,
|
|
529
|
+
) -> None:
|
|
530
|
+
"""读取转载附言、cgiDataNew 中的富文本正文和原文链接。"""
|
|
531
|
+
soup = BeautifulSoup(str(content_tag), "html.parser")
|
|
532
|
+
parts: list[str] = []
|
|
404
533
|
notice = soup.find("p", id="js_share_notice")
|
|
405
|
-
if
|
|
406
|
-
|
|
534
|
+
if notice:
|
|
535
|
+
m = re.search(
|
|
536
|
+
r'''innerHTML\s*=\s*("(?:\\[\s\S]|[^"\\])*"|'(?:\\[\s\S]|[^'\\])*')''',
|
|
537
|
+
str(notice),
|
|
538
|
+
)
|
|
539
|
+
if m:
|
|
540
|
+
text = _decode_text(_decode_js_string(m[1][1:-1]), preserve_newlines=True)
|
|
541
|
+
notice_markdown = _to_markdown_plain(f"<p>{text}</p>")
|
|
542
|
+
else:
|
|
543
|
+
for script in notice.find_all("script"):
|
|
544
|
+
script.decompose()
|
|
545
|
+
notice_markdown = _to_markdown_plain(str(notice))
|
|
546
|
+
if notice_markdown.strip():
|
|
547
|
+
parts.append(notice_markdown.strip())
|
|
548
|
+
|
|
549
|
+
for field in ("content_noencode", "content"):
|
|
550
|
+
body = _extract_cgi_string(scripts, field)
|
|
551
|
+
if body:
|
|
552
|
+
# 与普通富文本共用图片和 Markdown 处理,避免 data-src 变成空图片链接。
|
|
553
|
+
body_soup = BeautifulSoup(_decode_js_string(body), "html.parser")
|
|
554
|
+
_extract_rich_media_content(body_soup, result)
|
|
555
|
+
if result.article_markdown.strip():
|
|
556
|
+
parts.append(result.article_markdown.strip())
|
|
557
|
+
break
|
|
407
558
|
|
|
408
|
-
|
|
409
|
-
if not m:
|
|
559
|
+
if not parts:
|
|
410
560
|
return
|
|
411
561
|
|
|
412
|
-
text = _decode_text(m.group(1), preserve_newlines=True)
|
|
413
|
-
html_content = f"<p>{text}</p>"
|
|
414
|
-
|
|
415
562
|
share_link = soup.find("span", id="js_share_source")
|
|
416
563
|
if share_link:
|
|
417
564
|
href = share_link.get("data-url")
|
|
418
565
|
if href:
|
|
419
|
-
|
|
566
|
+
link = soup.new_tag("a", href=href)
|
|
567
|
+
link.string = "查看原文"
|
|
568
|
+
parts.append(_to_markdown_plain(str(link)))
|
|
420
569
|
|
|
421
|
-
result.article_markdown =
|
|
570
|
+
result.article_markdown = "\n\n".join(parts)
|
|
422
571
|
|
|
423
572
|
|
|
424
573
|
def _extract_plain_text_content(soup: BeautifulSoup, result: ArticleResult) -> None:
|
|
@@ -544,7 +693,7 @@ def _try_content_branch(
|
|
|
544
693
|
if branch == "rich_text":
|
|
545
694
|
_extract_rich_media_content(content, candidate)
|
|
546
695
|
elif branch == "repost":
|
|
547
|
-
_extract_repost_content(content, candidate)
|
|
696
|
+
_extract_repost_content(content, scripts, candidate)
|
|
548
697
|
elif branch in ("plain_text", "video"):
|
|
549
698
|
_extract_plain_text_content(soup, candidate)
|
|
550
699
|
elif branch == "swiper":
|
|
@@ -578,7 +727,9 @@ def _parse_html(url: str, html: str) -> ArticleResult:
|
|
|
578
727
|
|
|
579
728
|
scripts = soup.find_all("script", attrs={"type": "text/javascript"})
|
|
580
729
|
|
|
730
|
+
result.mp_user_name = _extract_user_name(soup.find_all("script"))
|
|
581
731
|
item_show_type = _extract_item_show_type(scripts)
|
|
732
|
+
result.article_item_show_type = item_show_type
|
|
582
733
|
result.article_display_type = _ARTICLE_DISPLAY_TYPES.get(
|
|
583
734
|
item_show_type, ArticleDisplayType.UNKNOWN,
|
|
584
735
|
)
|
|
@@ -79,11 +79,11 @@ def test_display_type_has_semantic_string_values(expected: str) -> None:
|
|
|
79
79
|
assert json.dumps(result.article_display_type) == json.dumps(expected)
|
|
80
80
|
serialized = json.loads(json.dumps(asdict(result)))
|
|
81
81
|
assert serialized["article_display_type"] == expected
|
|
82
|
-
assert "
|
|
83
|
-
assert not hasattr(result, "item_show_type")
|
|
82
|
+
assert serialized["article_item_show_type"] is None
|
|
84
83
|
|
|
85
84
|
|
|
86
85
|
def test_default_display_type_is_unknown() -> None:
|
|
87
86
|
result = ArticleResult()
|
|
88
87
|
assert result.article_display_type is ArticleDisplayType.UNKNOWN
|
|
88
|
+
assert result.article_item_show_type is None
|
|
89
89
|
assert not result.is_valid
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""付费受限正文的提示、元数据和预览内容回归测试。"""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from wechat_article_parser import ArticleDisplayType
|
|
6
|
+
from wechat_article_parser.parser import _parse_html
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
_URL = "https://mp.weixin.qq.com/s/n2q09-I95zwggfbXKAmbuQ"
|
|
10
|
+
_NOTICE = "这是一篇付费内容,正文需付费后查看。"
|
|
11
|
+
_FILTER = (
|
|
12
|
+
'<p class="js_pay_preview_filter" style="height: 0; overflow: hidden;">'
|
|
13
|
+
'<mp-pay-preview-filter></mp-pay-preview-filter></p>'
|
|
14
|
+
)
|
|
15
|
+
_METADATA = """
|
|
16
|
+
var biz = "MzkwMTYxMzYwNg==";
|
|
17
|
+
var mid = "2247533326";
|
|
18
|
+
var idx = "5";
|
|
19
|
+
var sn = "5ae1c1d004a9ccbf774610a8be1de3d7";
|
|
20
|
+
window.__allowLoadResFromMp = true;
|
|
21
|
+
var hd_head_img = "https://example.com/avatar";
|
|
22
|
+
var nickname = htmlDecode("真锂研究");
|
|
23
|
+
var oriCreateTime = '1791450000';
|
|
24
|
+
var user_name = "gh_61cfd6274c18";
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _html(body: str, metadata: str = _METADATA, data: str = "") -> str:
|
|
29
|
+
return (
|
|
30
|
+
'<meta property="og:title" content="付费文章标题">'
|
|
31
|
+
f'<div class="rich_media_content" id="js_content">{body}</div>'
|
|
32
|
+
'<div id="js_fullscreen_layout_padding"></div>'
|
|
33
|
+
'<script type="text/javascript">var item_show_type = "0";</script>'
|
|
34
|
+
f'<script type="text/javascript">{metadata}</script>'
|
|
35
|
+
f'<script type="text/javascript">{data}</script>'
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@pytest.mark.parametrize("metadata", [_METADATA, ""])
|
|
40
|
+
def test_paid_placeholder_keeps_metadata_and_does_not_use_unrelated_images(
|
|
41
|
+
metadata: str,
|
|
42
|
+
) -> None:
|
|
43
|
+
# 对应真实页面:正文只有付费占位符,脚本另有图片数组。
|
|
44
|
+
data = """
|
|
45
|
+
window.cgiDataNew = {
|
|
46
|
+
is_pay_subscribe: '1' * 1,
|
|
47
|
+
pay_subscribe_info: {preview_percent: '0' * 1},
|
|
48
|
+
picture_page_info_list: [{cdn_url: 'https://example.com/unrelated.jpg'}]
|
|
49
|
+
};
|
|
50
|
+
"""
|
|
51
|
+
result = _parse_html(_URL, _html(_FILTER, metadata, data))
|
|
52
|
+
|
|
53
|
+
assert result.article_markdown == _NOTICE
|
|
54
|
+
assert result.images == []
|
|
55
|
+
assert result.article_display_type is ArticleDisplayType.ARTICLE
|
|
56
|
+
assert result.article_item_show_type == 0
|
|
57
|
+
assert result.is_valid is bool(metadata)
|
|
58
|
+
if metadata:
|
|
59
|
+
assert result.mp_id == 3901613606
|
|
60
|
+
assert result.mp_name == "真锂研究"
|
|
61
|
+
assert result.mp_user_name == "gh_61cfd6274c18"
|
|
62
|
+
assert result.article_msg_id == 2247533326
|
|
63
|
+
assert result.article_idx == 5
|
|
64
|
+
assert result.article_sn == "5ae1c1d004a9ccbf774610a8be1de3d7"
|
|
65
|
+
assert result.article_publish_time == 1791450000
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@pytest.mark.parametrize("body", [
|
|
69
|
+
'<p>可见的免费预览正文。</p>',
|
|
70
|
+
'<img data-src="https://example.com/preview.jpg">',
|
|
71
|
+
])
|
|
72
|
+
def test_paid_filter_does_not_replace_visible_preview(body: str) -> None:
|
|
73
|
+
result = _parse_html(_URL, _html(body + _FILTER))
|
|
74
|
+
|
|
75
|
+
assert result.is_valid
|
|
76
|
+
assert _NOTICE not in result.article_markdown
|
|
77
|
+
if "免费预览" in body:
|
|
78
|
+
assert result.article_markdown == "可见的免费预览正文。"
|
|
79
|
+
else:
|
|
80
|
+
assert result.article_markdown == ""
|
|
81
|
+
assert result.images == ["https://example.com/preview.jpg"]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_paid_filter_takes_priority_over_empty_video_embed() -> None:
|
|
85
|
+
body = '<iframe class="wx_video_iframe"></iframe>' + _FILTER
|
|
86
|
+
result = _parse_html(_URL, _html(body))
|
|
87
|
+
|
|
88
|
+
assert result.is_valid
|
|
89
|
+
assert result.article_markdown == _NOTICE
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def test_paid_marker_outside_article_does_not_fill_empty_body() -> None:
|
|
93
|
+
result = _parse_html(_URL, _html("") + _FILTER)
|
|
94
|
+
|
|
95
|
+
assert result.article_markdown == ""
|
|
96
|
+
assert not result.is_valid
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_paid_flag_without_filter_keeps_accessible_body() -> None:
|
|
100
|
+
result = _parse_html(
|
|
101
|
+
_URL,
|
|
102
|
+
_html('<p>已提供的完整正文。</p>', data="window.cgiDataNew = {is_pay_subscribe: '1' * 1};"),
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
assert result.is_valid
|
|
106
|
+
assert result.article_markdown == "已提供的完整正文。"
|
|
@@ -16,6 +16,18 @@ TEST_URLS = [
|
|
|
16
16
|
"https://mp.weixin.qq.com/s/ySQdtsRlRmAl_skdc5HQ-A",
|
|
17
17
|
"https://mp.weixin.qq.com/s/MnkArbYQNp3tF29gujMUnQ",
|
|
18
18
|
"https://mp.weixin.qq.com/s/ncPoZnxKdvDRLM_s4DBgmA",
|
|
19
|
+
# 转载页面的静态附言及脚本正文。
|
|
20
|
+
"https://mp.weixin.qq.com/s/Pq-JkEjhoIlLvTMzWKVtOw",
|
|
21
|
+
"https://mp.weixin.qq.com/s/X51M6PfionI6UpX4zsG78Q",
|
|
22
|
+
# 视频号卡片和微信视频 iframe 的纯视频正文。
|
|
23
|
+
"https://mp.weixin.qq.com/s/wAyM7yYKDfTH21-XpS9agQ",
|
|
24
|
+
"https://mp.weixin.qq.com/s/a7P56VOipWNB0UyNaVOZYQ",
|
|
25
|
+
# 付费受限正文。
|
|
26
|
+
"https://mp.weixin.qq.com/s/n2q09-I95zwggfbXKAmbuQ",
|
|
27
|
+
# SVG foreignObject 承载的正文图片。
|
|
28
|
+
"https://mp.weixin.qq.com/s/3ixeC9whuEgpup2G1yvDZg",
|
|
29
|
+
# 纯公众号推荐卡片正文。
|
|
30
|
+
"https://mp.weixin.qq.com/s/PXZuh3fFEi3VXUUI1uBnlw",
|
|
19
31
|
]
|
|
20
32
|
|
|
21
33
|
|
|
@@ -47,6 +59,7 @@ def _assert_result(result: ArticleResult, url: str) -> None:
|
|
|
47
59
|
print(f"公众号ID: {result.mp_id}")
|
|
48
60
|
print(f"公众号名称: {result.mp_name}")
|
|
49
61
|
print(f"公众号别名: {result.mp_alias}")
|
|
62
|
+
print(f"公众号用户名: {result.mp_user_name}")
|
|
50
63
|
print(f"账号类型: {result.mp_account_type}")
|
|
51
64
|
print(
|
|
52
65
|
f"公众号图片: {result.mp_image[:80]}..."
|
|
@@ -64,6 +77,7 @@ def _assert_result(result: ArticleResult, url: str) -> None:
|
|
|
64
77
|
print(f"文章签名: {result.article_sn}")
|
|
65
78
|
print(f"文章标题: {result.article_title}")
|
|
66
79
|
print(f"展示类型: {result.article_display_type}")
|
|
80
|
+
print(f"原始展示类型: {result.article_item_show_type}")
|
|
67
81
|
print(
|
|
68
82
|
f"封面图: {result.article_cover_image[:80]}..."
|
|
69
83
|
if result.article_cover_image
|
|
@@ -82,6 +96,7 @@ def _assert_result(result: ArticleResult, url: str) -> None:
|
|
|
82
96
|
|
|
83
97
|
assert result.mp_id > 0, "mp_id should be positive"
|
|
84
98
|
assert result.mp_name, "mp_name should not be empty"
|
|
99
|
+
assert result.mp_user_name, "mp_user_name should not be empty for these reference articles"
|
|
85
100
|
assert result.article_id, "article_id should not be empty"
|
|
86
101
|
assert result.article_msg_id > 0, "article_msg_id should be positive"
|
|
87
102
|
assert result.article_idx > 0, "article_idx should be positive"
|
|
@@ -105,6 +120,7 @@ def test_fetch_all(url: str, proxy: str | None) -> None:
|
|
|
105
120
|
print(f"公众号ID: {result.mp_id}")
|
|
106
121
|
print(f"公众号名称: {result.mp_name}")
|
|
107
122
|
print(f"公众号别名: {result.mp_alias}")
|
|
123
|
+
print(f"公众号用户名: {result.mp_user_name}")
|
|
108
124
|
print(f"账号类型: {result.mp_account_type}")
|
|
109
125
|
print(f"公众号图片: {result.mp_image}")
|
|
110
126
|
print(f"公众号简介: {result.mp_description}")
|
|
@@ -114,6 +130,7 @@ def test_fetch_all(url: str, proxy: str | None) -> None:
|
|
|
114
130
|
print(f"文章签名: {result.article_sn}")
|
|
115
131
|
print(f"文章标题: {result.article_title}")
|
|
116
132
|
print(f"展示类型: {result.article_display_type}")
|
|
133
|
+
print(f"原始展示类型: {result.article_item_show_type}")
|
|
117
134
|
print(f"封面图: {result.article_cover_image}")
|
|
118
135
|
print(f"文章摘要: {result.article_description}")
|
|
119
136
|
print(f"发布时间: {result.article_publish_time}")
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""公众号推荐卡片转换及正文顺序的离线回归测试。"""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from wechat_article_parser import ArticleDisplayType
|
|
8
|
+
from wechat_article_parser.parser import _parse_html
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
_URL = "https://mp.weixin.qq.com/s/PXZuh3fFEi3VXUUI1uBnlw"
|
|
12
|
+
_METADATA = """
|
|
13
|
+
var item_show_type = "0";
|
|
14
|
+
var biz = "MzUyNTgyNjUzMw==";
|
|
15
|
+
var mid = "2247923648";
|
|
16
|
+
var idx = "2";
|
|
17
|
+
var sn = "d744c25ecdf2ac7f361d880dffb0a2ca";
|
|
18
|
+
window.__allowLoadResFromMp = true;
|
|
19
|
+
var hd_head_img = "https://example.com/publisher-avatar";
|
|
20
|
+
var nickname = htmlDecode("药融圈");
|
|
21
|
+
var oriCreateTime = '1791441362';
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _html(body: str) -> str:
|
|
26
|
+
return (
|
|
27
|
+
'<meta property="og:title" content="公众号推荐文章">'
|
|
28
|
+
f'<div class="rich_media_content">{body}</div>'
|
|
29
|
+
'<div id="js_fullscreen_layout_padding"></div>'
|
|
30
|
+
f'<script type="text/javascript">{_METADATA}</script>'
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_profile_only_article_preserves_names_signatures_and_metadata() -> None:
|
|
35
|
+
body = (
|
|
36
|
+
'<section><span><mp-common-profile data-nickname="药融圈" '
|
|
37
|
+
'data-signature="中国医药信息和人脉的链接平台" '
|
|
38
|
+
'data-headimg="https://example.com/card-avatar"></mp-common-profile></span></section>'
|
|
39
|
+
'<section><mp-common-profile data-nickname="药通社" '
|
|
40
|
+
'data-signature="政策、资讯、产业、技术"></mp-common-profile></section>'
|
|
41
|
+
)
|
|
42
|
+
result = _parse_html(
|
|
43
|
+
_URL,
|
|
44
|
+
_html(body)
|
|
45
|
+
+ '<mp-common-profile data-nickname="页外推荐" data-signature="不属于正文"></mp-common-profile>',
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
assert result.article_markdown == (
|
|
49
|
+
'**药融圈**:中国医药信息和人脉的链接平台\n\n'
|
|
50
|
+
'**药通社**:政策、资讯、产业、技术'
|
|
51
|
+
)
|
|
52
|
+
assert result.images == []
|
|
53
|
+
assert result.is_valid
|
|
54
|
+
assert result.mp_id == 3525826533
|
|
55
|
+
assert result.mp_name == "药融圈"
|
|
56
|
+
assert result.article_msg_id == 2247923648
|
|
57
|
+
assert result.article_idx == 2
|
|
58
|
+
assert result.article_sn == "d744c25ecdf2ac7f361d880dffb0a2ca"
|
|
59
|
+
assert result.article_publish_time == 1791441362
|
|
60
|
+
assert result.article_display_type is ArticleDisplayType.ARTICLE
|
|
61
|
+
assert result.article_item_show_type == 0
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def test_profile_mixed_with_text_and_images_preserves_order_without_duplicate_card() -> None:
|
|
65
|
+
result = _parse_html(_URL, _html(
|
|
66
|
+
'<p>推荐以下公众号。</p><mp-common-profile data-nickname="药融圈" '
|
|
67
|
+
'data-signature="医药信息平台">'
|
|
68
|
+
'<img src="https://example.com/card-avatar"><p>药融圈</p><p>医药信息平台</p>'
|
|
69
|
+
'</mp-common-profile>'
|
|
70
|
+
'<p><img data-src="https://mmbiz.qpic.cn/mmbiz_jpg/body/0?wx_fmt=jpeg"></p>'
|
|
71
|
+
'<p>正文结尾。</p>'
|
|
72
|
+
))
|
|
73
|
+
|
|
74
|
+
assert result.article_markdown == (
|
|
75
|
+
'推荐以下公众号。\n\n**药融圈**:医药信息平台\n\n'
|
|
76
|
+
'\n\n正文结尾。'
|
|
77
|
+
)
|
|
78
|
+
assert result.images == ["https://mmbiz.qpic.cn/mmbiz_jpg/body/640"]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@pytest.mark.parametrize(("attributes", "expected"), [
|
|
82
|
+
('data-nickname=" 药融圈 "', '**药融圈**'),
|
|
83
|
+
('data-signature=" 医药信息平台 "', '医药信息平台'),
|
|
84
|
+
('data-nickname=" " data-signature=" "', ''),
|
|
85
|
+
('', ''),
|
|
86
|
+
])
|
|
87
|
+
def test_profile_missing_fields_keeps_available_text_without_empty_placeholders(
|
|
88
|
+
attributes: str, expected: str,
|
|
89
|
+
) -> None:
|
|
90
|
+
result = _parse_html(_URL, _html(f'<mp-common-profile {attributes}></mp-common-profile>'))
|
|
91
|
+
|
|
92
|
+
assert result.article_markdown == expected
|
|
93
|
+
assert result.is_valid is bool(expected)
|
|
94
|
+
assert result.images == []
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def test_profile_attributes_keep_entities_and_escape_markdown_characters() -> None:
|
|
98
|
+
result = _parse_html(_URL, _html(
|
|
99
|
+
'<mp-common-profile data-nickname="药品&研发*圈*" '
|
|
100
|
+
'data-signature="A & B,进度 100%,保留 %20 和 <tag>。">'
|
|
101
|
+
'</mp-common-profile>'
|
|
102
|
+
))
|
|
103
|
+
|
|
104
|
+
assert result.article_markdown == r'**药品&研发\*圈\***:A & B,进度 100%,保留 %20 和 <tag>。'
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_repost_script_body_uses_same_profile_conversion() -> None:
|
|
108
|
+
body = (
|
|
109
|
+
'<mp-common-profile data-nickname="药通社" '
|
|
110
|
+
'data-signature="政策、资讯、产业、技术"></mp-common-profile>'
|
|
111
|
+
)
|
|
112
|
+
html = (
|
|
113
|
+
'<meta property="og:title" content="转载推荐文章">'
|
|
114
|
+
'<div class="original_page"><span id="js_share_source" '
|
|
115
|
+
'data-url="https://example.com/original"></span></div>'
|
|
116
|
+
f'<script type="text/javascript">{_METADATA}</script>'
|
|
117
|
+
'<script type="text/javascript">window.cgiDataNew = '
|
|
118
|
+
f'{{content_noencode: {json.dumps(body, ensure_ascii=False)}}};</script>'
|
|
119
|
+
)
|
|
120
|
+
result = _parse_html(_URL, html)
|
|
121
|
+
|
|
122
|
+
assert result.is_valid
|
|
123
|
+
assert result.article_markdown == (
|
|
124
|
+
'**药通社**:政策、资讯、产业、技术\n\n'
|
|
125
|
+
'[查看原文](https://example.com/original)'
|
|
126
|
+
)
|
|
127
|
+
assert result.images == []
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""转载页面的静态附言、脚本正文及元数据回归测试。"""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from wechat_article_parser import ArticleDisplayType
|
|
6
|
+
from wechat_article_parser.parser import _parse_html
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
_URL = "https://mp.weixin.qq.com/s/Pq-JkEjhoIlLvTMzWKVtOw"
|
|
10
|
+
_SOURCE = "https://mp.weixin.qq.com/s?__biz=original&mid=456&idx=1&sn=source"
|
|
11
|
+
_BODY = """<section>
|
|
12
|
+
<p>转载正文第一段,保留 100% 和 %20。</p>
|
|
13
|
+
<img data-src="https://mmbiz.qpic.cn/mmbiz_png/first/0?wx_fmt=png&from=appmsg">
|
|
14
|
+
<p>正文里的 <tag>、O'Reilly 和反斜杠 \\。</p>
|
|
15
|
+
<pre><code>if ready:
|
|
16
|
+
print("<tag>")
|
|
17
|
+
</code></pre>
|
|
18
|
+
<img data-src="https://mmbiz.qpic.cn/mmbiz_png/second/0?wx_fmt=png">
|
|
19
|
+
<p>转载正文最后一段。</p>
|
|
20
|
+
</section>"""
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _html(data: str = "", notice: str = "分享一篇文章。") -> str:
|
|
24
|
+
# 对应真实页面:正文容器尚未填充,正文和元数据分散在多个脚本。
|
|
25
|
+
return f"""
|
|
26
|
+
<meta property="og:title" content="转载文章标题">
|
|
27
|
+
<div class="original_page">
|
|
28
|
+
<p id="js_share_notice">{notice}</p>
|
|
29
|
+
<div id="js_share_headimg"><img src="https://example.com/avatar"></div>
|
|
30
|
+
<div id="js_content"></div>
|
|
31
|
+
<span id="js_share_source" data-url="{_SOURCE}">阅读全文</span>
|
|
32
|
+
</div>
|
|
33
|
+
<div id="js_fullscreen_layout_padding"></div>
|
|
34
|
+
<script type="text/javascript">
|
|
35
|
+
var biz = "MzAxNjI5NzI4MA==" || "";
|
|
36
|
+
var mid = "2247577355" || "";
|
|
37
|
+
var idx = "4" || "";
|
|
38
|
+
var sn = "signature" || "";
|
|
39
|
+
window.__allowLoadResFromMp = true;
|
|
40
|
+
</script>
|
|
41
|
+
<script type="text/javascript">
|
|
42
|
+
var hd_head_img = "https://example.com/publisher-avatar";
|
|
43
|
+
var nickname = htmlDecode("新浪医药");
|
|
44
|
+
var item_show_type = "0";
|
|
45
|
+
var user_name = "gh_3675b3bc7097";
|
|
46
|
+
</script>
|
|
47
|
+
<script type="text/javascript">
|
|
48
|
+
function setTime() {{ var oriCreateTime = '1791459000'; }}
|
|
49
|
+
</script>
|
|
50
|
+
<script type="text/javascript">{data}</script>
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _js_body(body: str, quote: str = "'") -> str:
|
|
55
|
+
escaped = body.translate(str.maketrans({
|
|
56
|
+
"\\": "\\\\", "'": "\\'", '"': r"\x22", "<": r"\x3c",
|
|
57
|
+
">": r"\x3e", "&": r"\x26", "\n": r"\n",
|
|
58
|
+
}))
|
|
59
|
+
return quote + escaped + quote
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@pytest.mark.parametrize("field", ["content_noencode", "content"])
|
|
63
|
+
@pytest.mark.parametrize("quote", ["'", '"'])
|
|
64
|
+
def test_static_repost_reads_script_body_and_preserves_metadata(
|
|
65
|
+
field: str, quote: str,
|
|
66
|
+
) -> None:
|
|
67
|
+
data = f"""
|
|
68
|
+
window.cgiDataNew = {{
|
|
69
|
+
reference: {{content_noencode: '嵌套的错误正文', content: '嵌套的错误正文'}},
|
|
70
|
+
{quote}{field}{quote}: {_js_body(_BODY, quote)},
|
|
71
|
+
picture_page_info_list: [{{cdn_url: 'https://example.com/wrong-image'}}]
|
|
72
|
+
}};
|
|
73
|
+
"""
|
|
74
|
+
result = _parse_html(_URL, _html(data))
|
|
75
|
+
|
|
76
|
+
assert result.is_valid
|
|
77
|
+
assert result.article_display_type is ArticleDisplayType.ARTICLE
|
|
78
|
+
assert result.article_item_show_type == 0
|
|
79
|
+
assert result.mp_id == 3016297280
|
|
80
|
+
assert result.mp_name == "新浪医药"
|
|
81
|
+
assert result.mp_user_name == "gh_3675b3bc7097"
|
|
82
|
+
assert result.article_msg_id == 2247577355
|
|
83
|
+
assert result.article_idx == 4
|
|
84
|
+
assert result.article_sn == "signature"
|
|
85
|
+
assert result.article_publish_time == 1791459000
|
|
86
|
+
assert result.images == [
|
|
87
|
+
"https://mmbiz.qpic.cn/mmbiz_png/first/640",
|
|
88
|
+
"https://mmbiz.qpic.cn/mmbiz_png/second/640",
|
|
89
|
+
]
|
|
90
|
+
markdown = result.article_markdown
|
|
91
|
+
assert markdown.startswith("分享一篇文章。")
|
|
92
|
+
assert "转载正文第一段,保留 100% 和 %20。" in markdown
|
|
93
|
+
assert "正文里的 <tag>、O'Reilly 和反斜杠" in markdown
|
|
94
|
+
assert '```\nif ready:\n print("<tag>")\n```' in markdown
|
|
95
|
+
assert "转载正文最后一段。" in markdown
|
|
96
|
+
assert markdown.endswith(f"[查看原文]({_SOURCE})")
|
|
97
|
+
assert markdown.count("![") == 2
|
|
98
|
+
assert "![]()" not in markdown
|
|
99
|
+
assert "嵌套的错误正文" not in markdown
|
|
100
|
+
assert "avatar" not in markdown
|
|
101
|
+
assert "wrong-image" not in markdown
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def test_static_notice_without_script_body_still_parses() -> None:
|
|
105
|
+
result = _parse_html(_URL, _html())
|
|
106
|
+
|
|
107
|
+
assert result.is_valid
|
|
108
|
+
assert result.article_markdown == f"分享一篇文章。\n\n[查看原文]({_SOURCE})"
|
|
109
|
+
assert result.images == []
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def test_legacy_script_notice_keeps_body_and_newlines() -> None:
|
|
113
|
+
notice = r'<script>notice.innerHTML = "旧版附言\x0a第二行";</script>'
|
|
114
|
+
result = _parse_html(_URL, _html(notice=notice))
|
|
115
|
+
|
|
116
|
+
assert result.is_valid
|
|
117
|
+
assert result.article_markdown == f"旧版附言 \n第二行\n\n[查看原文]({_SOURCE})"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_empty_unencoded_body_falls_back_to_content() -> None:
|
|
121
|
+
data = "window.cgiDataNew = {content_noencode: '', content: '<p>备用正文</p>'};"
|
|
122
|
+
result = _parse_html(_URL, _html(data, notice=""))
|
|
123
|
+
|
|
124
|
+
assert result.is_valid
|
|
125
|
+
assert result.article_markdown == f"备用正文\n\n[查看原文]({_SOURCE})"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def test_repost_ignores_body_in_comments_strings_and_other_objects() -> None:
|
|
129
|
+
data = """
|
|
130
|
+
// window.cgiDataNew = {content_noencode: '注释中的错误正文'};
|
|
131
|
+
var example = "window.cgiDataNew = {content: '字符串中的错误正文'};";
|
|
132
|
+
other.window.cgiDataNew = {content_noencode: '其他对象中的错误正文'};
|
|
133
|
+
window.cgiDataNew = {
|
|
134
|
+
reference: {content_noencode: '嵌套的错误正文'},
|
|
135
|
+
content_noencode: loadUntrustedContent()
|
|
136
|
+
};
|
|
137
|
+
"""
|
|
138
|
+
result = _parse_html(_URL, _html(data))
|
|
139
|
+
|
|
140
|
+
assert result.article_markdown == f"分享一篇文章。\n\n[查看原文]({_SOURCE})"
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""SVG 容器承载正文图片的离线回归测试。"""
|
|
2
|
+
|
|
3
|
+
from wechat_article_parser import ArticleDisplayType, ArticleResult
|
|
4
|
+
from wechat_article_parser.parser import _parse_html
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
_URL = "https://mp.weixin.qq.com/s/3ixeC9whuEgpup2G1yvDZg"
|
|
8
|
+
_METADATA = """
|
|
9
|
+
var item_show_type = "0";
|
|
10
|
+
var biz = "MjM5MjY2MTc5NQ==";
|
|
11
|
+
var mid = "2652403355";
|
|
12
|
+
var idx = "1";
|
|
13
|
+
var sn = "9c582b2554db23b592ed758e95680f0a";
|
|
14
|
+
window.__allowLoadResFromMp = true;
|
|
15
|
+
var hd_head_img = "https://example.com/avatar";
|
|
16
|
+
var nickname = htmlDecode("CFMOTO春风动力");
|
|
17
|
+
var oriCreateTime = '1791454818';
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _parse(body: str) -> ArticleResult:
|
|
22
|
+
return _parse_html(
|
|
23
|
+
_URL,
|
|
24
|
+
'<meta property="og:title" content="SVG 图片正文">'
|
|
25
|
+
f'<div class="rich_media_content">{body}</div>'
|
|
26
|
+
'<div id="js_fullscreen_layout_padding"></div>'
|
|
27
|
+
f'<script type="text/javascript">{_METADATA}</script>',
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_foreignobject_images_keep_body_order_and_rich_text_metadata() -> None:
|
|
32
|
+
# 对应真实页面:整个正文由三个 SVG 中的长图组成,没有背景图。
|
|
33
|
+
body = "".join(
|
|
34
|
+
'<svg viewBox="0 0 1080 2362"><foreignObject>'
|
|
35
|
+
f'<span><img data-src="https://mmbiz.qpic.cn/mmbiz_jpg/body-{i}/0?wx_fmt=jpeg"></span>'
|
|
36
|
+
'</foreignObject></svg>'
|
|
37
|
+
for i in range(3)
|
|
38
|
+
)
|
|
39
|
+
result = _parse(body)
|
|
40
|
+
urls = [f"https://mmbiz.qpic.cn/mmbiz_jpg/body-{i}/640" for i in range(3)]
|
|
41
|
+
|
|
42
|
+
assert result.article_markdown == "".join(f"" for url in urls)
|
|
43
|
+
assert result.images == urls
|
|
44
|
+
assert result.is_valid
|
|
45
|
+
assert result.mp_id == 2392661795
|
|
46
|
+
assert result.mp_name == "CFMOTO春风动力"
|
|
47
|
+
assert result.article_msg_id == 2652403355
|
|
48
|
+
assert result.article_idx == 1
|
|
49
|
+
assert result.article_sn == "9c582b2554db23b592ed758e95680f0a"
|
|
50
|
+
assert result.article_publish_time == 1791454818
|
|
51
|
+
assert result.article_display_type is ArticleDisplayType.ARTICLE
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_nested_svg_images_and_backgrounds_keep_source_order() -> None:
|
|
55
|
+
result = _parse(
|
|
56
|
+
'<svg style=\'background-image: url("https://example.com/first.jpg")\'>'
|
|
57
|
+
'<foreignObject><img data-src="https://example.com/second.jpg" alt="第二张">'
|
|
58
|
+
'<svg style=\'background-image: url("https://example.com/third.jpg")\'>'
|
|
59
|
+
'<foreignObject><img src="https://example.com/fourth.jpg"></foreignObject>'
|
|
60
|
+
'</svg></foreignObject></svg>'
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
assert result.article_markdown == "".join([
|
|
64
|
+
"",
|
|
65
|
+
"",
|
|
66
|
+
"",
|
|
67
|
+
"",
|
|
68
|
+
])
|
|
69
|
+
assert result.images == [
|
|
70
|
+
"https://example.com/first.jpg",
|
|
71
|
+
"https://example.com/second.jpg",
|
|
72
|
+
"https://example.com/third.jpg",
|
|
73
|
+
"https://example.com/fourth.jpg",
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def test_repeated_svg_images_remain_in_body_but_image_list_is_unique() -> None:
|
|
78
|
+
result = _parse(
|
|
79
|
+
'<img src="https://mmbiz.qpic.cn/mmbiz_jpg/body/0">'
|
|
80
|
+
'<svg><foreignObject><svg><foreignObject>'
|
|
81
|
+
'<img data-src="https://mmbiz.qpic.cn/mmbiz_jpg/body/640?wx_fmt=jpeg">'
|
|
82
|
+
'</foreignObject></svg></foreignObject></svg>'
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
url = "https://mmbiz.qpic.cn/mmbiz_jpg/body/640"
|
|
86
|
+
assert result.images == [url]
|
|
87
|
+
assert result.article_markdown.count(f"") == 2
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_decorative_svg_and_invalid_images_do_not_pollute_body() -> None:
|
|
91
|
+
result = _parse(
|
|
92
|
+
'<p>正文</p><svg><title>装饰分隔线</title><path d="M0 0 L100 0"></path></svg>'
|
|
93
|
+
'<svg><foreignObject><img><img src="data:image/png;base64,AA==">'
|
|
94
|
+
'</foreignObject></svg>'
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
assert result.article_markdown == "正文"
|
|
98
|
+
assert result.images == []
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
"""类型字段优先分发、模板细分及 HTML 兜底的离线回归测试。"""
|
|
2
2
|
|
|
3
|
+
from dataclasses import asdict
|
|
4
|
+
import json
|
|
5
|
+
|
|
3
6
|
import pytest
|
|
4
7
|
from bs4 import BeautifulSoup
|
|
5
8
|
|
|
@@ -63,6 +66,9 @@ def test_read_current_article_type(script: str, expected: int | None) -> None:
|
|
|
63
66
|
soup = BeautifulSoup(_html(script=script), "html.parser")
|
|
64
67
|
|
|
65
68
|
assert parser._extract_item_show_type(soup.find_all("script")) == expected
|
|
69
|
+
result = parser._parse_html(_URL, _html(_RICH_TEXT, script))
|
|
70
|
+
assert result.article_item_show_type == expected
|
|
71
|
+
assert json.loads(json.dumps(asdict(result)))["article_item_show_type"] == expected
|
|
66
72
|
|
|
67
73
|
|
|
68
74
|
@pytest.mark.parametrize("current_assignment", ["", "window.item_show_type = '8';"])
|
|
@@ -102,6 +108,7 @@ def test_type_zero_uses_html_to_distinguish_rich_text_and_repost(
|
|
|
102
108
|
result = parser._parse_html(_URL, _html(body, 'var item_show_type = "0";'))
|
|
103
109
|
|
|
104
110
|
assert result.article_display_type is ArticleDisplayType.ARTICLE
|
|
111
|
+
assert result.article_item_show_type == 0
|
|
105
112
|
assert expected in result.article_markdown
|
|
106
113
|
if expected == "转载附言":
|
|
107
114
|
assert "[查看原文](https://example.com/original)" in result.article_markdown
|
|
@@ -130,7 +137,7 @@ def test_known_type_takes_priority_over_other_html_templates(
|
|
|
130
137
|
assert expected in result.article_markdown
|
|
131
138
|
assert "富文本正文" not in result.article_markdown
|
|
132
139
|
assert result.article_display_type is display_type
|
|
133
|
-
assert
|
|
140
|
+
assert result.article_item_show_type == item_type
|
|
134
141
|
if item_type == 8:
|
|
135
142
|
assert result.images == ["https://mmbiz.qpic.cn/mmbiz_png/body/640"]
|
|
136
143
|
|
|
@@ -154,6 +161,7 @@ def test_known_type_without_matching_data_falls_back_to_html(item_type: int) ->
|
|
|
154
161
|
)
|
|
155
162
|
|
|
156
163
|
assert result.article_markdown == "富文本正文"
|
|
164
|
+
assert result.article_item_show_type == item_type
|
|
157
165
|
|
|
158
166
|
|
|
159
167
|
def test_type_zero_without_matching_html_falls_back_to_other_templates() -> None:
|
|
@@ -196,6 +204,7 @@ def test_unrecognized_content_still_returns_metadata() -> None:
|
|
|
196
204
|
|
|
197
205
|
assert result.article_title == "原标题"
|
|
198
206
|
assert result.mp_alias == "account"
|
|
207
|
+
assert result.article_item_show_type == 999
|
|
199
208
|
assert result.article_display_type is ArticleDisplayType.UNKNOWN
|
|
200
209
|
assert result.article_markdown == ""
|
|
201
210
|
assert not result.is_valid
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""当前公众号 user_name 的位置规则及公开返回接口回归测试。"""
|
|
2
|
+
|
|
3
|
+
from dataclasses import asdict
|
|
4
|
+
import json
|
|
5
|
+
|
|
6
|
+
import httpx
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
from wechat_article_parser import ArticleResult, parser
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
_URL = "https://mp.weixin.qq.com/s/tBqB3134QzILypNVUW1fdw"
|
|
13
|
+
# 精简自参考页面的文章元数据脚本,保留字段上下文。
|
|
14
|
+
_METADATA = """
|
|
15
|
+
var nickname = htmlDecode("爱范儿");
|
|
16
|
+
var appmsg_type = "9";
|
|
17
|
+
var ct = "1784779200";
|
|
18
|
+
var user_name = "aifaner383402";
|
|
19
|
+
var fakeid = "";
|
|
20
|
+
"""
|
|
21
|
+
_DECOYS = """
|
|
22
|
+
var ad = {biz_info: {user_name: 'gh_1e80bb81a1d2'}};
|
|
23
|
+
window.cgiDataNew = {ad: {user_name: 'nested-ad-account'}};
|
|
24
|
+
(function () { var user_name = 'local-account'; })();
|
|
25
|
+
function report() { var user_name = 'another-local-account'; }
|
|
26
|
+
var example = "var user_name = 'string-account';";
|
|
27
|
+
var template = `var user_name = 'template-account';`;
|
|
28
|
+
// var user_name = 'comment-account';
|
|
29
|
+
/* var user_name = 'block-comment-account'; */
|
|
30
|
+
var original_user_name = 'reference-account';
|
|
31
|
+
var info = {userName: window.user_name};
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _html(*scripts: str, body: str = "") -> str:
|
|
36
|
+
return body + "".join(
|
|
37
|
+
f'<script type="text/javascript">{script}</script>' for script in scripts
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@pytest.mark.parametrize("body", ["", '<div class="rich_media_content">正文</div>'])
|
|
42
|
+
def test_user_name_comes_from_global_metadata(body: str) -> None:
|
|
43
|
+
result = parser._parse_html(_URL, _html(_DECOYS, _METADATA, _DECOYS, body=body))
|
|
44
|
+
|
|
45
|
+
assert result.mp_user_name == "aifaner383402"
|
|
46
|
+
assert json.loads(json.dumps(asdict(result)))["mp_user_name"] == "aifaner383402"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@pytest.mark.parametrize(
|
|
50
|
+
("declaration", "expected"),
|
|
51
|
+
[
|
|
52
|
+
("var user_name = 'gh_another-account';", "gh_another-account"),
|
|
53
|
+
('var\nuser_name /* comment */ = "other_account";', "other_account"),
|
|
54
|
+
(r'var user_name = "gh_\x61bc";', "gh_abc"),
|
|
55
|
+
("", ""),
|
|
56
|
+
("var user_name = '';", ""),
|
|
57
|
+
("var user_name = null;", ""),
|
|
58
|
+
("var user_name = another_variable;", ""),
|
|
59
|
+
("var user_name = 'prefix' + suffix;", ""),
|
|
60
|
+
],
|
|
61
|
+
)
|
|
62
|
+
def test_user_name_is_dynamic_and_missing_values_are_empty(
|
|
63
|
+
declaration: str, expected: str,
|
|
64
|
+
) -> None:
|
|
65
|
+
result = parser._parse_html(_URL, _html(_DECOYS, declaration))
|
|
66
|
+
assert result.mp_user_name == expected
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_script_without_type_attribute() -> None:
|
|
70
|
+
result = parser._parse_html(_URL, f"<script>{_METADATA}</script>")
|
|
71
|
+
assert result.mp_user_name == "aifaner383402"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def test_default_user_name() -> None:
|
|
75
|
+
assert ArticleResult().mp_user_name == ""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@pytest.mark.parametrize("key", ["user_name", "'user_name'", '"user_name"'])
|
|
79
|
+
@pytest.mark.parametrize("body", ["", '<div class="rich_media_content">正文</div>'])
|
|
80
|
+
def test_cgi_direct_user_name_fallback(key: str, body: str) -> None:
|
|
81
|
+
data = (
|
|
82
|
+
"(function () { window.cgiDataNew = {"
|
|
83
|
+
"base_resp: {ret: '0' * 1},"
|
|
84
|
+
"ad: {user_name: 'wrong-ad'},"
|
|
85
|
+
"references: [{user_name: 'wrong-reference'}],"
|
|
86
|
+
f"{key}: 'gh_9a1919baf888'"
|
|
87
|
+
"}; })();"
|
|
88
|
+
)
|
|
89
|
+
result = parser._parse_html(_URL, _html(_DECOYS, data, body=body))
|
|
90
|
+
assert result.mp_user_name == "gh_9a1919baf888"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_global_metadata_has_priority_over_cgi_fallback() -> None:
|
|
94
|
+
data = "window.cgiDataNew = {user_name: 'cgi-account'};"
|
|
95
|
+
result = parser._parse_html(_URL, _html(data, _METADATA))
|
|
96
|
+
assert result.mp_user_name == "aifaner383402"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@pytest.mark.parametrize(
|
|
100
|
+
"data",
|
|
101
|
+
[
|
|
102
|
+
"window.cgiDataNew = {user_name: ''};",
|
|
103
|
+
"window.cgiDataNew = {user_name: 'prefix' + suffix};",
|
|
104
|
+
"other.window.cgiDataNew = {user_name: 'wrong'};",
|
|
105
|
+
"var text = \"window.cgiDataNew = {user_name: 'wrong'};\";",
|
|
106
|
+
"// window.cgiDataNew = {user_name: 'wrong'};",
|
|
107
|
+
"/* window.cgiDataNew = {user_name: 'wrong'}; */",
|
|
108
|
+
],
|
|
109
|
+
)
|
|
110
|
+
def test_cgi_fallback_ignores_decoys_and_nonliteral_values(data: str) -> None:
|
|
111
|
+
result = parser._parse_html(_URL, _html(_DECOYS, data))
|
|
112
|
+
assert result.mp_user_name == ""
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@pytest.fixture
|
|
116
|
+
def mock_article_http(monkeypatch):
|
|
117
|
+
transport = httpx.MockTransport(
|
|
118
|
+
lambda request: httpx.Response(200, text=_html(_DECOYS, _METADATA))
|
|
119
|
+
)
|
|
120
|
+
client = httpx.Client
|
|
121
|
+
async_client = httpx.AsyncClient
|
|
122
|
+
monkeypatch.setattr(
|
|
123
|
+
parser.httpx, "Client", lambda **kwargs: client(transport=transport, **kwargs),
|
|
124
|
+
)
|
|
125
|
+
monkeypatch.setattr(
|
|
126
|
+
parser.httpx, "AsyncClient", lambda **kwargs: async_client(transport=transport, **kwargs),
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def test_sync_parse_returns_user_name(mock_article_http) -> None:
|
|
131
|
+
assert parser.parse(_URL).mp_user_name == "aifaner383402"
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@pytest.mark.asyncio
|
|
135
|
+
async def test_async_parse_returns_user_name(mock_article_http) -> None:
|
|
136
|
+
result = await parser.parse_async(_URL)
|
|
137
|
+
assert result.mp_user_name == "aifaner383402"
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""纯视频号卡片正文的离线回归测试。"""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from wechat_article_parser import ArticleDisplayType
|
|
6
|
+
from wechat_article_parser.parser import _parse_html
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
_URL = "https://mp.weixin.qq.com/s/wAyM7yYKDfTH21-XpS9agQ"
|
|
10
|
+
_CARD = (
|
|
11
|
+
'<mp-common-videosnap data-type="video" data-id="video-id" '
|
|
12
|
+
'data-desc="视频描述" data-url="https://example.com/cover.jpg">'
|
|
13
|
+
'</mp-common-videosnap>'
|
|
14
|
+
)
|
|
15
|
+
_IFRAME = (
|
|
16
|
+
'<iframe class="video_iframe rich_pages wx_video_iframe" data-mpvid="wxv_123" '
|
|
17
|
+
'data-src="https://mp.weixin.qq.com/mp/readtemplate?t=pages/video_player_tmpl'
|
|
18
|
+
'&action=mpvideo&vid=wxv_123"></iframe>'
|
|
19
|
+
)
|
|
20
|
+
_METADATA = """
|
|
21
|
+
var biz = "MjM5NTAwMzE0MA==";
|
|
22
|
+
var mid = "2908209265";
|
|
23
|
+
var idx = "4";
|
|
24
|
+
var sn = "signature";
|
|
25
|
+
window.__allowLoadResFromMp = true;
|
|
26
|
+
var hd_head_img = "https://example.com/avatar";
|
|
27
|
+
var nickname = htmlDecode("卡车之家");
|
|
28
|
+
var oriCreateTime = '1791462000';
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _html(body: str, metadata: str = _METADATA) -> str:
|
|
33
|
+
return (
|
|
34
|
+
'<meta property="og:title" content="后双胎大轻客,江淮星锐EV6">'
|
|
35
|
+
f'<div class="rich_media_content">{body}</div>'
|
|
36
|
+
'<script type="text/javascript">var item_show_type = "0";</script>'
|
|
37
|
+
f'<script type="text/javascript">{metadata}</script>'
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@pytest.mark.parametrize("metadata", [_METADATA, ""])
|
|
42
|
+
@pytest.mark.parametrize("video", [_CARD, _IFRAME])
|
|
43
|
+
def test_video_only_body_gets_one_sentence_and_still_requires_metadata(
|
|
44
|
+
metadata: str, video: str,
|
|
45
|
+
) -> None:
|
|
46
|
+
result = _parse_html(
|
|
47
|
+
_URL,
|
|
48
|
+
_html(f'<section>{video}</section><p><br></p><mp-style-type></mp-style-type>', metadata),
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
assert result.article_markdown == "这是一篇纯视频内容。"
|
|
52
|
+
assert result.article_display_type is ArticleDisplayType.ARTICLE
|
|
53
|
+
assert result.article_item_show_type == 0
|
|
54
|
+
assert result.images == []
|
|
55
|
+
assert result.is_valid is bool(metadata)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@pytest.mark.parametrize("body", [
|
|
59
|
+
'<p>实际文字正文</p>',
|
|
60
|
+
'<img data-src="https://example.com/body.jpg">',
|
|
61
|
+
])
|
|
62
|
+
@pytest.mark.parametrize("video", [_CARD, _IFRAME])
|
|
63
|
+
def test_video_with_other_content_keeps_existing_markdown(body: str, video: str) -> None:
|
|
64
|
+
result = _parse_html(_URL, _html(body + video))
|
|
65
|
+
|
|
66
|
+
assert result.is_valid
|
|
67
|
+
assert "这是一篇纯视频内容。" not in result.article_markdown
|
|
68
|
+
assert "cover.jpg" not in result.article_markdown
|
|
69
|
+
if "实际文字正文" in body:
|
|
70
|
+
assert result.article_markdown == "实际文字正文"
|
|
71
|
+
else:
|
|
72
|
+
assert result.images == ["https://example.com/body.jpg"]
|
|
73
|
+
assert result.article_markdown == ""
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@pytest.mark.parametrize("body", [
|
|
77
|
+
"",
|
|
78
|
+
_CARD.replace('data-type="video"', 'data-type="image"'),
|
|
79
|
+
'<iframe src="https://example.com/unrelated"></iframe>',
|
|
80
|
+
])
|
|
81
|
+
def test_empty_or_non_video_body_does_not_get_video_notice(body: str) -> None:
|
|
82
|
+
result = _parse_html(_URL, _html(body))
|
|
83
|
+
|
|
84
|
+
assert result.article_markdown == ""
|
|
85
|
+
assert not result.is_valid
|
|
File without changes
|
|
File without changes
|
{wechat_article_parser-0.0.8 → wechat_article_parser-0.0.10}/src/wechat_article_parser/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|