wechat-article-parser 0.0.7__tar.gz → 0.0.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/PKG-INFO +5 -2
- {wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/README.md +4 -1
- {wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/pyproject.toml +1 -1
- {wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/src/wechat_article_parser/models.py +5 -1
- {wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/src/wechat_article_parser/parser.py +99 -9
- wechat_article_parser-0.0.9/tests/test_markdown.py +85 -0
- {wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/tests/test_models.py +2 -2
- {wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/tests/test_parser.py +5 -0
- {wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/tests/test_type_routing.py +10 -1
- wechat_article_parser-0.0.9/tests/test_user_name.py +137 -0
- {wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/.gitignore +0 -0
- {wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/LICENSE +0 -0
- {wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/src/wechat_article_parser/__init__.py +0 -0
- {wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/tests/conftest.py +0 -0
- {wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/tests/test_swiper_content.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: wechat-article-parser
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.9
|
|
4
4
|
Summary: WeChat MP article parser - extract metadata and content from WeChat public account articles
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -102,6 +102,7 @@ result = parse(
|
|
|
102
102
|
| `mp_id` | `int` | 公众号 ID(解码后的整数) |
|
|
103
103
|
| `mp_name` | `str` | 公众号名称 |
|
|
104
104
|
| `mp_alias` | `str` | 公众号别名 |
|
|
105
|
+
| `mp_user_name` | `str` | 公众号用户名(如 `aifaner383402` 或 `gh_9a1919baf888`),未获取到时为空字符串 |
|
|
105
106
|
| `mp_image` | `str` | 公众号头像链接 |
|
|
106
107
|
| `mp_description` | `str` | 公众号简介 |
|
|
107
108
|
| `mp_account_type` | `AccountType` | 账号类型:`AccountType.SUBSCRIPTION`(订阅号)/ `AccountType.SERVICE`(服务号)/ `AccountType.UNKNOWN`(未识别) |
|
|
@@ -115,6 +116,7 @@ result = parse(
|
|
|
115
116
|
| `article_markdown` | `str` | 文章正文的 Markdown 内容;视频分享没有文字附言时可为空 |
|
|
116
117
|
| `article_publish_time` | `int` | 发布时间(Unix 时间戳) |
|
|
117
118
|
| `article_display_type` | `ArticleDisplayType` | 文章展示类型:`article`、`video`、`image`、`text` 或 `unknown` |
|
|
119
|
+
| `article_item_show_type` | `int \| None` | 微信页面中的原始展示类型:`0` 图文、`5` 视频、`8` 图片轮播、`10` 文字;其他已提取的数值原样保留,缺失或无法解析时为 `None` |
|
|
118
120
|
| `raw_html` | `str` | 原始 HTML 网页源代码(仅在 `include_raw_html=True` 时填充,否则为空字符串) |
|
|
119
121
|
| `images` | `list[str]` | 文章中提取的所有图片链接 |
|
|
120
122
|
| `is_valid` | `bool` | 关键字段是否全部解析成功(属性) |
|
|
@@ -144,10 +146,11 @@ if result.article_display_type == "video":
|
|
|
144
146
|
print("视频分享可以没有文字正文")
|
|
145
147
|
|
|
146
148
|
print(result.article_display_type) # 例如 video
|
|
149
|
+
print(result.article_item_show_type) # 例如 5
|
|
147
150
|
```
|
|
148
151
|
|
|
149
152
|
`article_display_type` 是返回结果中的数据字段,包含在 `dataclasses.asdict(result)` 中,JSON 序列化时输出语义字符串。
|
|
150
|
-
|
|
153
|
+
`article_item_show_type` 返回微信页面中 `item_show_type` 的原始数值,也包含在 `dataclasses.asdict(result)` 中;JSON 序列化时输出数字或 `null`。
|
|
151
154
|
|
|
152
155
|
### 判断账号类型
|
|
153
156
|
|
|
@@ -87,6 +87,7 @@ result = parse(
|
|
|
87
87
|
| `mp_id` | `int` | 公众号 ID(解码后的整数) |
|
|
88
88
|
| `mp_name` | `str` | 公众号名称 |
|
|
89
89
|
| `mp_alias` | `str` | 公众号别名 |
|
|
90
|
+
| `mp_user_name` | `str` | 公众号用户名(如 `aifaner383402` 或 `gh_9a1919baf888`),未获取到时为空字符串 |
|
|
90
91
|
| `mp_image` | `str` | 公众号头像链接 |
|
|
91
92
|
| `mp_description` | `str` | 公众号简介 |
|
|
92
93
|
| `mp_account_type` | `AccountType` | 账号类型:`AccountType.SUBSCRIPTION`(订阅号)/ `AccountType.SERVICE`(服务号)/ `AccountType.UNKNOWN`(未识别) |
|
|
@@ -100,6 +101,7 @@ result = parse(
|
|
|
100
101
|
| `article_markdown` | `str` | 文章正文的 Markdown 内容;视频分享没有文字附言时可为空 |
|
|
101
102
|
| `article_publish_time` | `int` | 发布时间(Unix 时间戳) |
|
|
102
103
|
| `article_display_type` | `ArticleDisplayType` | 文章展示类型:`article`、`video`、`image`、`text` 或 `unknown` |
|
|
104
|
+
| `article_item_show_type` | `int \| None` | 微信页面中的原始展示类型:`0` 图文、`5` 视频、`8` 图片轮播、`10` 文字;其他已提取的数值原样保留,缺失或无法解析时为 `None` |
|
|
103
105
|
| `raw_html` | `str` | 原始 HTML 网页源代码(仅在 `include_raw_html=True` 时填充,否则为空字符串) |
|
|
104
106
|
| `images` | `list[str]` | 文章中提取的所有图片链接 |
|
|
105
107
|
| `is_valid` | `bool` | 关键字段是否全部解析成功(属性) |
|
|
@@ -129,10 +131,11 @@ if result.article_display_type == "video":
|
|
|
129
131
|
print("视频分享可以没有文字正文")
|
|
130
132
|
|
|
131
133
|
print(result.article_display_type) # 例如 video
|
|
134
|
+
print(result.article_item_show_type) # 例如 5
|
|
132
135
|
```
|
|
133
136
|
|
|
134
137
|
`article_display_type` 是返回结果中的数据字段,包含在 `dataclasses.asdict(result)` 中,JSON 序列化时输出语义字符串。
|
|
135
|
-
|
|
138
|
+
`article_item_show_type` 返回微信页面中 `item_show_type` 的原始数值,也包含在 `dataclasses.asdict(result)` 中;JSON 序列化时输出数字或 `null`。
|
|
136
139
|
|
|
137
140
|
### 判断账号类型
|
|
138
141
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "wechat-article-parser"
|
|
7
|
-
version = "0.0.
|
|
7
|
+
version = "0.0.9"
|
|
8
8
|
description = "WeChat MP article parser - extract metadata and content from WeChat public account articles"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
{wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/src/wechat_article_parser/models.py
RENAMED
|
@@ -68,8 +68,12 @@ class ArticleResult:
|
|
|
68
68
|
# 文章中提取的图片列表
|
|
69
69
|
images: list[str] = field(default_factory=list)
|
|
70
70
|
|
|
71
|
-
#
|
|
71
|
+
# 语义展示类型和微信页面中的原始数值。
|
|
72
72
|
article_display_type: ArticleDisplayType = ArticleDisplayType.UNKNOWN
|
|
73
|
+
article_item_show_type: int | None = None
|
|
74
|
+
|
|
75
|
+
# 当前公众号的页面全局 user_name,未提取到时为空字符串。
|
|
76
|
+
mp_user_name: str = ""
|
|
73
77
|
|
|
74
78
|
@property
|
|
75
79
|
def is_valid(self) -> bool:
|
{wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/src/wechat_article_parser/parser.py
RENAMED
|
@@ -35,12 +35,22 @@ _JS_DATA_TOKEN_RE = re.compile(
|
|
|
35
35
|
# ---------------------------------------------------------------------------
|
|
36
36
|
|
|
37
37
|
class _MarkdownConverter(MarkdownConverter):
|
|
38
|
-
def
|
|
39
|
-
|
|
38
|
+
def convert_br(self, el, text, parent_tags):
|
|
39
|
+
# 代码中的换行不需要 Markdown 硬换行所用的两个尾随空格。
|
|
40
|
+
if "pre" in parent_tags:
|
|
41
|
+
return "\n"
|
|
42
|
+
return super().convert_br(el, text, parent_tags)
|
|
40
43
|
|
|
41
|
-
def
|
|
42
|
-
|
|
43
|
-
return super().
|
|
44
|
+
def convert_pre(self, el, text, parent_tags):
|
|
45
|
+
# BeautifulSoup 已解码 ;代码中将其还原为可直接复制的普通空格。
|
|
46
|
+
return super().convert_pre(el, text.replace("\xa0", " "), parent_tags)
|
|
47
|
+
|
|
48
|
+
def convert_td(self, el, text, parent_tags):
|
|
49
|
+
# GFM 表格内的管道符即使位于行内代码中,也必须转义。
|
|
50
|
+
return super().convert_td(el, text.replace("|", r"\|"), parent_tags)
|
|
51
|
+
|
|
52
|
+
def convert_th(self, el, text, parent_tags):
|
|
53
|
+
return super().convert_th(el, text.replace("|", r"\|"), parent_tags)
|
|
44
54
|
|
|
45
55
|
|
|
46
56
|
def _to_markdown(html: str) -> str:
|
|
@@ -50,8 +60,7 @@ def _to_markdown(html: str) -> str:
|
|
|
50
60
|
def _to_markdown_plain(html: str) -> str:
|
|
51
61
|
"""用于纯文本类内容(纯文本/转载/视频/轮播/全屏)的转换。
|
|
52
62
|
|
|
53
|
-
这些内容由 <br>
|
|
54
|
-
避免 _MarkdownConverter 折叠空白时把换行一并吞掉。
|
|
63
|
+
这些内容由 <br> 承载换行,使用基础转换器保留换行。
|
|
55
64
|
"""
|
|
56
65
|
return MarkdownConverter().convert(html)
|
|
57
66
|
|
|
@@ -220,6 +229,84 @@ def _extract_item_show_type(scripts: list[Tag]) -> int | None:
|
|
|
220
229
|
return None
|
|
221
230
|
|
|
222
231
|
|
|
232
|
+
def _extract_user_name(scripts: list[Tag]) -> str:
|
|
233
|
+
"""读取全局 user_name,缺失时读取 window.cgiDataNew 的直接属性。
|
|
234
|
+
|
|
235
|
+
文章元数据脚本中该变量位于 nickname、ct 附近;不依赖行号或账号值。
|
|
236
|
+
使用词法标记跳过注释和字符串,不执行页面脚本。
|
|
237
|
+
"""
|
|
238
|
+
for script in scripts:
|
|
239
|
+
if "user_name" not in script.text:
|
|
240
|
+
continue
|
|
241
|
+
history: deque[str] = deque(maxlen=4)
|
|
242
|
+
brace_depth = 0
|
|
243
|
+
for match in _JS_DATA_TOKEN_RE.finditer(script.text):
|
|
244
|
+
if match.lastgroup == "comment":
|
|
245
|
+
continue
|
|
246
|
+
token = match.group()
|
|
247
|
+
previous = tuple(history)
|
|
248
|
+
if (
|
|
249
|
+
brace_depth == 0
|
|
250
|
+
and token == ";"
|
|
251
|
+
and previous[:3] == ("var", "user_name", "=")
|
|
252
|
+
and len(previous) == 4
|
|
253
|
+
and previous[3].startswith(("'", '"'))
|
|
254
|
+
):
|
|
255
|
+
value = _decode_text(previous[3][1:-1])
|
|
256
|
+
if value:
|
|
257
|
+
return value
|
|
258
|
+
if token == "{":
|
|
259
|
+
brace_depth += 1
|
|
260
|
+
elif token == "}":
|
|
261
|
+
brace_depth -= 1
|
|
262
|
+
history.append(token)
|
|
263
|
+
return _extract_cgi_user_name(scripts)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _extract_cgi_user_name(scripts: list[Tag]) -> str:
|
|
267
|
+
"""从当前文章的 cgiDataNew 对象中读取 user_name,排除嵌套账号。"""
|
|
268
|
+
for script in scripts:
|
|
269
|
+
if "cgiDataNew" not in script.text or "user_name" not in script.text:
|
|
270
|
+
continue
|
|
271
|
+
history: deque[str] = deque(maxlen=5)
|
|
272
|
+
stack: list[str] = []
|
|
273
|
+
for match in _JS_DATA_TOKEN_RE.finditer(script.text):
|
|
274
|
+
if match.lastgroup == "comment":
|
|
275
|
+
continue
|
|
276
|
+
token = match.group()
|
|
277
|
+
previous = tuple(history)
|
|
278
|
+
if not stack:
|
|
279
|
+
if (
|
|
280
|
+
token == "{"
|
|
281
|
+
and previous[-4:] == ("window", ".", "cgiDataNew", "=")
|
|
282
|
+
and (len(previous) < 5 or previous[-5] != ".")
|
|
283
|
+
):
|
|
284
|
+
stack.append(token)
|
|
285
|
+
else:
|
|
286
|
+
# 只接受完整的字符串属性值,不读取表达式或嵌套对象的值。
|
|
287
|
+
if (
|
|
288
|
+
stack == ["{"]
|
|
289
|
+
and token in (",", "}")
|
|
290
|
+
and len(previous) >= 4
|
|
291
|
+
and previous[-4] in ("{", ",")
|
|
292
|
+
and previous[-3].strip("\"'") == "user_name"
|
|
293
|
+
and previous[-2] == ":"
|
|
294
|
+
and previous[-1].startswith(("'", '"'))
|
|
295
|
+
):
|
|
296
|
+
value = _decode_text(previous[-1][1:-1])
|
|
297
|
+
if value:
|
|
298
|
+
return value
|
|
299
|
+
if token in ("{", "["):
|
|
300
|
+
stack.append(token)
|
|
301
|
+
elif token in ("}", "]"):
|
|
302
|
+
if stack[-1] == {"}": "{", "]": "["}[token]:
|
|
303
|
+
stack.pop()
|
|
304
|
+
else:
|
|
305
|
+
stack.clear()
|
|
306
|
+
history.append(token)
|
|
307
|
+
return ""
|
|
308
|
+
|
|
309
|
+
|
|
223
310
|
def _extract_account_type(script_text: str, result: ArticleResult) -> None:
|
|
224
311
|
"""从 new_service_type 中提取账号类型:0/1 → 订阅号,2 → 服务号。"""
|
|
225
312
|
if result.mp_account_type:
|
|
@@ -332,7 +419,8 @@ def _extract_swiper_meta(script_text: str, result: ArticleResult) -> None:
|
|
|
332
419
|
|
|
333
420
|
def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None:
|
|
334
421
|
"""将 rich_media_content 区块转换为 Markdown。"""
|
|
335
|
-
|
|
422
|
+
# prettify 会在行内标签之间插入空白,改变代码和引用链接的内容。
|
|
423
|
+
soup = BeautifulSoup(str(content_tag), "html.parser")
|
|
336
424
|
seen: set[str] = set()
|
|
337
425
|
|
|
338
426
|
# 处理 <img> 标签
|
|
@@ -385,7 +473,7 @@ def _extract_rich_media_content(content_tag: Tag, result: ArticleResult) -> None
|
|
|
385
473
|
for tag in to_remove:
|
|
386
474
|
tag.decompose()
|
|
387
475
|
|
|
388
|
-
result.article_markdown = _to_markdown(soup
|
|
476
|
+
result.article_markdown = _to_markdown(str(soup))
|
|
389
477
|
|
|
390
478
|
|
|
391
479
|
def _extract_repost_content(content_tag: Tag, result: ArticleResult) -> None:
|
|
@@ -568,7 +656,9 @@ def _parse_html(url: str, html: str) -> ArticleResult:
|
|
|
568
656
|
|
|
569
657
|
scripts = soup.find_all("script", attrs={"type": "text/javascript"})
|
|
570
658
|
|
|
659
|
+
result.mp_user_name = _extract_user_name(soup.find_all("script"))
|
|
571
660
|
item_show_type = _extract_item_show_type(scripts)
|
|
661
|
+
result.article_item_show_type = item_show_type
|
|
572
662
|
result.article_display_type = _ARTICLE_DISPLAY_TYPES.get(
|
|
573
663
|
item_show_type, ArticleDisplayType.UNKNOWN,
|
|
574
664
|
)
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""富文本 Markdown 转换的离线回归测试。"""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from wechat_article_parser.parser import _parse_html
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _convert(body: str) -> str:
|
|
9
|
+
html = f'<div class="rich_media_content">{body}</div>'
|
|
10
|
+
return _parse_html("https://mp.weixin.qq.com/s/test", html).article_markdown
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def test_highlighted_code_preserves_breaks_spaces_and_entities() -> None:
|
|
14
|
+
# 微信 mdnice 代码块:语法高亮 span 切分 token,br 换行,nbsp 占位。
|
|
15
|
+
body = (
|
|
16
|
+
'<pre data-tool="mdnice编辑器"><code>'
|
|
17
|
+
'<span><span># ======== 基础体验 ========</span></span><span><br></span>'
|
|
18
|
+
'<span><span>set</span></span><span> -g default-terminal </span>'
|
|
19
|
+
'<span><span>"tmux-256color"</span></span><span><br></span>'
|
|
20
|
+
'<span>set</span><span> -as terminal-features </span>'
|
|
21
|
+
'<span>",*:RGB"</span><span><br></span>'
|
|
22
|
+
'<span>set</span><span> -g mouse on </span>'
|
|
23
|
+
'<span># 支持鼠标操作</span><span><br></span><span><br></span>'
|
|
24
|
+
'<span> echo "<tag>" && echo  ok</span>'
|
|
25
|
+
'</code></pre>'
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
assert _convert(body) == (
|
|
29
|
+
'```\n'
|
|
30
|
+
'# ======== 基础体验 ========\n'
|
|
31
|
+
'set -g default-terminal "tmux-256color"\n'
|
|
32
|
+
'set -as terminal-features ",*:RGB"\n'
|
|
33
|
+
'set -g mouse on # 支持鼠标操作\n\n'
|
|
34
|
+
' echo "<tag>" && echo ok\n'
|
|
35
|
+
'```'
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_plain_pre_preserves_indentation_and_blank_lines() -> None:
|
|
40
|
+
assert _convert('<pre><code>if ready:\n run()\n\n done()\n</code></pre>') == (
|
|
41
|
+
'```\nif ready:\n run()\n\n done()\n```'
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@pytest.mark.parametrize("cell", ["td", "th"])
|
|
46
|
+
def test_table_pipes_in_text_and_inline_code_are_escaped(cell: str) -> None:
|
|
47
|
+
output = _convert(
|
|
48
|
+
f'<table><tr><{cell}>A | B</{cell}><{cell}><section>'
|
|
49
|
+
'<span>验证 </span><code><span>tailscale status | grep peer-relay</span></code>'
|
|
50
|
+
f'</section></{cell}></tr></table>'
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
assert r'A \| B' in output
|
|
54
|
+
assert r'`tailscale status \| grep peer-relay`' in output
|
|
55
|
+
# 每行仍然只有两个单元格(三个未转义的分隔符)。
|
|
56
|
+
assert all(line.replace(r'\|', '').count('|') == 3 for line in output.splitlines())
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_pipe_outside_table_is_unchanged() -> None:
|
|
60
|
+
assert _convert('<p><code>left | right</code></p><pre>left | right</pre>') == (
|
|
61
|
+
'`left | right`\n\n```\nleft | right\n```'
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def test_inline_spans_and_paragraph_breaks_preserve_spacing() -> None:
|
|
66
|
+
assert _convert(
|
|
67
|
+
'<p><span>hello </span><span>world</span><br><span>next line</span></p>'
|
|
68
|
+
) == 'hello world \nnext line'
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def test_bold_reference_stays_inline() -> None:
|
|
72
|
+
assert _convert(
|
|
73
|
+
'<ul><li><section><strong><span>Hermes</span><sup>'
|
|
74
|
+
'<a href="https://example.com/hermes">[1]</a></sup></strong>'
|
|
75
|
+
'<span>:Nous Research 的自主 AI Agent</span></section></li></ul>'
|
|
76
|
+
) == '* **Hermes[[1]](https://example.com/hermes)**:Nous Research 的自主 AI Agent'
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_literal_asterisks_in_source_stay_literal_and_inline() -> None:
|
|
80
|
+
# 这篇原文没有 strong 标签;不能把普通星号自动猜测为加粗语义。
|
|
81
|
+
assert _convert(
|
|
82
|
+
'<ul><li><section><span>**</span><span>Hermes</span>'
|
|
83
|
+
'<sup><span>[1]</span></sup><span>**:Nous Research 的自主 AI Agent</span>'
|
|
84
|
+
'</section></li></ul>'
|
|
85
|
+
) == r'* \*\*Hermes[1]\*\*:Nous Research 的自主 AI Agent'
|
|
@@ -79,11 +79,11 @@ def test_display_type_has_semantic_string_values(expected: str) -> None:
|
|
|
79
79
|
assert json.dumps(result.article_display_type) == json.dumps(expected)
|
|
80
80
|
serialized = json.loads(json.dumps(asdict(result)))
|
|
81
81
|
assert serialized["article_display_type"] == expected
|
|
82
|
-
assert "
|
|
83
|
-
assert not hasattr(result, "item_show_type")
|
|
82
|
+
assert serialized["article_item_show_type"] is None
|
|
84
83
|
|
|
85
84
|
|
|
86
85
|
def test_default_display_type_is_unknown() -> None:
|
|
87
86
|
result = ArticleResult()
|
|
88
87
|
assert result.article_display_type is ArticleDisplayType.UNKNOWN
|
|
88
|
+
assert result.article_item_show_type is None
|
|
89
89
|
assert not result.is_valid
|
|
@@ -47,6 +47,7 @@ def _assert_result(result: ArticleResult, url: str) -> None:
|
|
|
47
47
|
print(f"公众号ID: {result.mp_id}")
|
|
48
48
|
print(f"公众号名称: {result.mp_name}")
|
|
49
49
|
print(f"公众号别名: {result.mp_alias}")
|
|
50
|
+
print(f"公众号用户名: {result.mp_user_name}")
|
|
50
51
|
print(f"账号类型: {result.mp_account_type}")
|
|
51
52
|
print(
|
|
52
53
|
f"公众号图片: {result.mp_image[:80]}..."
|
|
@@ -64,6 +65,7 @@ def _assert_result(result: ArticleResult, url: str) -> None:
|
|
|
64
65
|
print(f"文章签名: {result.article_sn}")
|
|
65
66
|
print(f"文章标题: {result.article_title}")
|
|
66
67
|
print(f"展示类型: {result.article_display_type}")
|
|
68
|
+
print(f"原始展示类型: {result.article_item_show_type}")
|
|
67
69
|
print(
|
|
68
70
|
f"封面图: {result.article_cover_image[:80]}..."
|
|
69
71
|
if result.article_cover_image
|
|
@@ -82,6 +84,7 @@ def _assert_result(result: ArticleResult, url: str) -> None:
|
|
|
82
84
|
|
|
83
85
|
assert result.mp_id > 0, "mp_id should be positive"
|
|
84
86
|
assert result.mp_name, "mp_name should not be empty"
|
|
87
|
+
assert result.mp_user_name, "mp_user_name should not be empty for these reference articles"
|
|
85
88
|
assert result.article_id, "article_id should not be empty"
|
|
86
89
|
assert result.article_msg_id > 0, "article_msg_id should be positive"
|
|
87
90
|
assert result.article_idx > 0, "article_idx should be positive"
|
|
@@ -105,6 +108,7 @@ def test_fetch_all(url: str, proxy: str | None) -> None:
|
|
|
105
108
|
print(f"公众号ID: {result.mp_id}")
|
|
106
109
|
print(f"公众号名称: {result.mp_name}")
|
|
107
110
|
print(f"公众号别名: {result.mp_alias}")
|
|
111
|
+
print(f"公众号用户名: {result.mp_user_name}")
|
|
108
112
|
print(f"账号类型: {result.mp_account_type}")
|
|
109
113
|
print(f"公众号图片: {result.mp_image}")
|
|
110
114
|
print(f"公众号简介: {result.mp_description}")
|
|
@@ -114,6 +118,7 @@ def test_fetch_all(url: str, proxy: str | None) -> None:
|
|
|
114
118
|
print(f"文章签名: {result.article_sn}")
|
|
115
119
|
print(f"文章标题: {result.article_title}")
|
|
116
120
|
print(f"展示类型: {result.article_display_type}")
|
|
121
|
+
print(f"原始展示类型: {result.article_item_show_type}")
|
|
117
122
|
print(f"封面图: {result.article_cover_image}")
|
|
118
123
|
print(f"文章摘要: {result.article_description}")
|
|
119
124
|
print(f"发布时间: {result.article_publish_time}")
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
"""类型字段优先分发、模板细分及 HTML 兜底的离线回归测试。"""
|
|
2
2
|
|
|
3
|
+
from dataclasses import asdict
|
|
4
|
+
import json
|
|
5
|
+
|
|
3
6
|
import pytest
|
|
4
7
|
from bs4 import BeautifulSoup
|
|
5
8
|
|
|
@@ -63,6 +66,9 @@ def test_read_current_article_type(script: str, expected: int | None) -> None:
|
|
|
63
66
|
soup = BeautifulSoup(_html(script=script), "html.parser")
|
|
64
67
|
|
|
65
68
|
assert parser._extract_item_show_type(soup.find_all("script")) == expected
|
|
69
|
+
result = parser._parse_html(_URL, _html(_RICH_TEXT, script))
|
|
70
|
+
assert result.article_item_show_type == expected
|
|
71
|
+
assert json.loads(json.dumps(asdict(result)))["article_item_show_type"] == expected
|
|
66
72
|
|
|
67
73
|
|
|
68
74
|
@pytest.mark.parametrize("current_assignment", ["", "window.item_show_type = '8';"])
|
|
@@ -102,6 +108,7 @@ def test_type_zero_uses_html_to_distinguish_rich_text_and_repost(
|
|
|
102
108
|
result = parser._parse_html(_URL, _html(body, 'var item_show_type = "0";'))
|
|
103
109
|
|
|
104
110
|
assert result.article_display_type is ArticleDisplayType.ARTICLE
|
|
111
|
+
assert result.article_item_show_type == 0
|
|
105
112
|
assert expected in result.article_markdown
|
|
106
113
|
if expected == "转载附言":
|
|
107
114
|
assert "[查看原文](https://example.com/original)" in result.article_markdown
|
|
@@ -130,7 +137,7 @@ def test_known_type_takes_priority_over_other_html_templates(
|
|
|
130
137
|
assert expected in result.article_markdown
|
|
131
138
|
assert "富文本正文" not in result.article_markdown
|
|
132
139
|
assert result.article_display_type is display_type
|
|
133
|
-
assert
|
|
140
|
+
assert result.article_item_show_type == item_type
|
|
134
141
|
if item_type == 8:
|
|
135
142
|
assert result.images == ["https://mmbiz.qpic.cn/mmbiz_png/body/640"]
|
|
136
143
|
|
|
@@ -154,6 +161,7 @@ def test_known_type_without_matching_data_falls_back_to_html(item_type: int) ->
|
|
|
154
161
|
)
|
|
155
162
|
|
|
156
163
|
assert result.article_markdown == "富文本正文"
|
|
164
|
+
assert result.article_item_show_type == item_type
|
|
157
165
|
|
|
158
166
|
|
|
159
167
|
def test_type_zero_without_matching_html_falls_back_to_other_templates() -> None:
|
|
@@ -196,6 +204,7 @@ def test_unrecognized_content_still_returns_metadata() -> None:
|
|
|
196
204
|
|
|
197
205
|
assert result.article_title == "原标题"
|
|
198
206
|
assert result.mp_alias == "account"
|
|
207
|
+
assert result.article_item_show_type == 999
|
|
199
208
|
assert result.article_display_type is ArticleDisplayType.UNKNOWN
|
|
200
209
|
assert result.article_markdown == ""
|
|
201
210
|
assert not result.is_valid
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""当前公众号 user_name 的位置规则及公开返回接口回归测试。"""
|
|
2
|
+
|
|
3
|
+
from dataclasses import asdict
|
|
4
|
+
import json
|
|
5
|
+
|
|
6
|
+
import httpx
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
from wechat_article_parser import ArticleResult, parser
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
_URL = "https://mp.weixin.qq.com/s/tBqB3134QzILypNVUW1fdw"
|
|
13
|
+
# 精简自参考页面的文章元数据脚本,保留字段上下文。
|
|
14
|
+
_METADATA = """
|
|
15
|
+
var nickname = htmlDecode("爱范儿");
|
|
16
|
+
var appmsg_type = "9";
|
|
17
|
+
var ct = "1784779200";
|
|
18
|
+
var user_name = "aifaner383402";
|
|
19
|
+
var fakeid = "";
|
|
20
|
+
"""
|
|
21
|
+
_DECOYS = """
|
|
22
|
+
var ad = {biz_info: {user_name: 'gh_1e80bb81a1d2'}};
|
|
23
|
+
window.cgiDataNew = {ad: {user_name: 'nested-ad-account'}};
|
|
24
|
+
(function () { var user_name = 'local-account'; })();
|
|
25
|
+
function report() { var user_name = 'another-local-account'; }
|
|
26
|
+
var example = "var user_name = 'string-account';";
|
|
27
|
+
var template = `var user_name = 'template-account';`;
|
|
28
|
+
// var user_name = 'comment-account';
|
|
29
|
+
/* var user_name = 'block-comment-account'; */
|
|
30
|
+
var original_user_name = 'reference-account';
|
|
31
|
+
var info = {userName: window.user_name};
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _html(*scripts: str, body: str = "") -> str:
|
|
36
|
+
return body + "".join(
|
|
37
|
+
f'<script type="text/javascript">{script}</script>' for script in scripts
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@pytest.mark.parametrize("body", ["", '<div class="rich_media_content">正文</div>'])
|
|
42
|
+
def test_user_name_comes_from_global_metadata(body: str) -> None:
|
|
43
|
+
result = parser._parse_html(_URL, _html(_DECOYS, _METADATA, _DECOYS, body=body))
|
|
44
|
+
|
|
45
|
+
assert result.mp_user_name == "aifaner383402"
|
|
46
|
+
assert json.loads(json.dumps(asdict(result)))["mp_user_name"] == "aifaner383402"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@pytest.mark.parametrize(
|
|
50
|
+
("declaration", "expected"),
|
|
51
|
+
[
|
|
52
|
+
("var user_name = 'gh_another-account';", "gh_another-account"),
|
|
53
|
+
('var\nuser_name /* comment */ = "other_account";', "other_account"),
|
|
54
|
+
(r'var user_name = "gh_\x61bc";', "gh_abc"),
|
|
55
|
+
("", ""),
|
|
56
|
+
("var user_name = '';", ""),
|
|
57
|
+
("var user_name = null;", ""),
|
|
58
|
+
("var user_name = another_variable;", ""),
|
|
59
|
+
("var user_name = 'prefix' + suffix;", ""),
|
|
60
|
+
],
|
|
61
|
+
)
|
|
62
|
+
def test_user_name_is_dynamic_and_missing_values_are_empty(
|
|
63
|
+
declaration: str, expected: str,
|
|
64
|
+
) -> None:
|
|
65
|
+
result = parser._parse_html(_URL, _html(_DECOYS, declaration))
|
|
66
|
+
assert result.mp_user_name == expected
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_script_without_type_attribute() -> None:
|
|
70
|
+
result = parser._parse_html(_URL, f"<script>{_METADATA}</script>")
|
|
71
|
+
assert result.mp_user_name == "aifaner383402"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def test_default_user_name() -> None:
|
|
75
|
+
assert ArticleResult().mp_user_name == ""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@pytest.mark.parametrize("key", ["user_name", "'user_name'", '"user_name"'])
|
|
79
|
+
@pytest.mark.parametrize("body", ["", '<div class="rich_media_content">正文</div>'])
|
|
80
|
+
def test_cgi_direct_user_name_fallback(key: str, body: str) -> None:
|
|
81
|
+
data = (
|
|
82
|
+
"(function () { window.cgiDataNew = {"
|
|
83
|
+
"base_resp: {ret: '0' * 1},"
|
|
84
|
+
"ad: {user_name: 'wrong-ad'},"
|
|
85
|
+
"references: [{user_name: 'wrong-reference'}],"
|
|
86
|
+
f"{key}: 'gh_9a1919baf888'"
|
|
87
|
+
"}; })();"
|
|
88
|
+
)
|
|
89
|
+
result = parser._parse_html(_URL, _html(_DECOYS, data, body=body))
|
|
90
|
+
assert result.mp_user_name == "gh_9a1919baf888"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_global_metadata_has_priority_over_cgi_fallback() -> None:
|
|
94
|
+
data = "window.cgiDataNew = {user_name: 'cgi-account'};"
|
|
95
|
+
result = parser._parse_html(_URL, _html(data, _METADATA))
|
|
96
|
+
assert result.mp_user_name == "aifaner383402"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@pytest.mark.parametrize(
|
|
100
|
+
"data",
|
|
101
|
+
[
|
|
102
|
+
"window.cgiDataNew = {user_name: ''};",
|
|
103
|
+
"window.cgiDataNew = {user_name: 'prefix' + suffix};",
|
|
104
|
+
"other.window.cgiDataNew = {user_name: 'wrong'};",
|
|
105
|
+
"var text = \"window.cgiDataNew = {user_name: 'wrong'};\";",
|
|
106
|
+
"// window.cgiDataNew = {user_name: 'wrong'};",
|
|
107
|
+
"/* window.cgiDataNew = {user_name: 'wrong'}; */",
|
|
108
|
+
],
|
|
109
|
+
)
|
|
110
|
+
def test_cgi_fallback_ignores_decoys_and_nonliteral_values(data: str) -> None:
|
|
111
|
+
result = parser._parse_html(_URL, _html(_DECOYS, data))
|
|
112
|
+
assert result.mp_user_name == ""
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@pytest.fixture
|
|
116
|
+
def mock_article_http(monkeypatch):
|
|
117
|
+
transport = httpx.MockTransport(
|
|
118
|
+
lambda request: httpx.Response(200, text=_html(_DECOYS, _METADATA))
|
|
119
|
+
)
|
|
120
|
+
client = httpx.Client
|
|
121
|
+
async_client = httpx.AsyncClient
|
|
122
|
+
monkeypatch.setattr(
|
|
123
|
+
parser.httpx, "Client", lambda **kwargs: client(transport=transport, **kwargs),
|
|
124
|
+
)
|
|
125
|
+
monkeypatch.setattr(
|
|
126
|
+
parser.httpx, "AsyncClient", lambda **kwargs: async_client(transport=transport, **kwargs),
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def test_sync_parse_returns_user_name(mock_article_http) -> None:
|
|
131
|
+
assert parser.parse(_URL).mp_user_name == "aifaner383402"
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@pytest.mark.asyncio
|
|
135
|
+
async def test_async_parse_returns_user_name(mock_article_http) -> None:
|
|
136
|
+
result = await parser.parse_async(_URL)
|
|
137
|
+
assert result.mp_user_name == "aifaner383402"
|
|
File without changes
|
|
File without changes
|
{wechat_article_parser-0.0.7 → wechat_article_parser-0.0.9}/src/wechat_article_parser/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|